Compare commits
No commits in common. "main" and "v10.3.0" have entirely different histories.
166 changed files with 1477 additions and 16044 deletions
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
## What Is Brainy
|
||||
|
||||
@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package.
|
||||
@soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package.
|
||||
|
||||
## Core Architecture
|
||||
|
||||
|
|
|
|||
|
|
@ -32,31 +32,22 @@ jobs:
|
|||
run: |
|
||||
set -eo pipefail
|
||||
|
||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
|
||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
||||
VERSION="$(node -p "require('./package.json').version")"
|
||||
# The dist-tag follows the version: a prerelease (any hyphen —
|
||||
# 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' —
|
||||
# every consumer resolving 'latest' from this registry would otherwise
|
||||
# be handed a release candidate. Same rule scripts/release.sh applies
|
||||
# to the storefront leg.
|
||||
NPM_TAG="latest"
|
||||
case "$VERSION" in
|
||||
*-*) NPM_TAG="rc" ;;
|
||||
esac
|
||||
echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..."
|
||||
echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry..."
|
||||
|
||||
TMPRC="$(mktemp)"
|
||||
chmod 600 "$TMPRC"
|
||||
{
|
||||
echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}"
|
||||
echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}"
|
||||
echo "@soulcraft:registry=${SOURCE_NPM_REG}"
|
||||
echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}"
|
||||
} > "$TMPRC"
|
||||
|
||||
# The release script bumps package.json's version before it tags, so
|
||||
# this tag's checkout already carries the version being published —
|
||||
# nothing here re-derives it from the tag name.
|
||||
PUBLISH_OK=true
|
||||
if ! npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then
|
||||
if ! npm publish --tag latest --userconfig "$TMPRC"; then
|
||||
PUBLISH_OK=false
|
||||
fi
|
||||
|
||||
|
|
@ -64,7 +55,7 @@ jobs:
|
|||
# exit code: a benign duplicate publish (a prior run, or a mirror, already
|
||||
# landed this exact version) reports failure even though the registry
|
||||
# already holds the right content.
|
||||
LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
|
||||
LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")"
|
||||
rm -f "$TMPRC"
|
||||
|
||||
if [ "$LANDED_VERSION" != "$VERSION" ]; then
|
||||
|
|
@ -73,7 +64,7 @@ jobs:
|
|||
fi
|
||||
|
||||
if [ "$PUBLISH_OK" = true ]; then
|
||||
echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry."
|
||||
echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry."
|
||||
else
|
||||
echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
|
||||
echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead."
|
||||
fi
|
||||
|
|
|
|||
131
CHANGELOG.md
131
CHANGELOG.md
|
|
@ -2,120 +2,7 @@
|
|||
|
||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||
|
||||
### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28)
|
||||
|
||||
- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1)
|
||||
- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25)
|
||||
- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40)
|
||||
- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2)
|
||||
- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927)
|
||||
- docs: measurements in public history carry numbers, not provenance (02c61636)
|
||||
- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010)
|
||||
- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950)
|
||||
- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9)
|
||||
- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5)
|
||||
- perf(open): answer "are there any entities?" with one directory read (417ddb51)
|
||||
- perf(generations): discover generations by directory name, not by walking the log (9dd39921)
|
||||
- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc)
|
||||
- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca)
|
||||
- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f)
|
||||
- chore: keep the generated neural stamps at main's values (c1f09723)
|
||||
- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3)
|
||||
- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02)
|
||||
- feat(open): open never waits for a provider that is rebuilding itself (131daa08)
|
||||
- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f)
|
||||
- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6)
|
||||
- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b)
|
||||
- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f)
|
||||
- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c)
|
||||
- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b)
|
||||
|
||||
|
||||
### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27)
|
||||
|
||||
- Merge branch 'next/open-brainy-rename' (a58372f0)
|
||||
- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83)
|
||||
- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24)
|
||||
|
||||
|
||||
### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27)
|
||||
|
||||
- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef)
|
||||
|
||||
|
||||
### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27)
|
||||
|
||||
- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b)
|
||||
- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659)
|
||||
- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1)
|
||||
- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4)
|
||||
- Merge branch 'next/enumeration-identity-rekey' (204d74c1)
|
||||
- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16)
|
||||
- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a)
|
||||
- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab)
|
||||
- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9)
|
||||
- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245)
|
||||
- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee)
|
||||
- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb)
|
||||
- chore(release): 10.4.1-rc.1 (7870dc40)
|
||||
|
||||
|
||||
### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26)
|
||||
|
||||
- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e)
|
||||
- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8)
|
||||
|
||||
|
||||
### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26)
|
||||
|
||||
- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed)
|
||||
|
||||
|
||||
### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25)
|
||||
|
||||
- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b)
|
||||
|
||||
|
||||
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
|
||||
|
||||
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
|
||||
- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b)
|
||||
- fix(add): empty string is real data, not a missing field (258e9042)
|
||||
- feat(vfs): implement readdir's recursive option — typed since 7.30, never read (fc516da6)
|
||||
- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40)
|
||||
|
||||
|
||||
### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25)
|
||||
|
||||
- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3)
|
||||
- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97)
|
||||
- fix(storage): an unknown nested storage config can never silently land on the shared default root (ddd5e719)
|
||||
- docs(release): the 10.4.0 entry, the index-health concept doc, and the API surfaces — written from the tree, not the plan (8cced871)
|
||||
- fix(plugins): the silent-degrade doors close — a broken accelerator install can never read as absent (b9ba50fb)
|
||||
- feat(recovery): the catchup verdict is consumed; verb rows go live; the metadata rebuild goes online (18f172e0)
|
||||
- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780)
|
||||
|
||||
|
||||
### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24)
|
||||
|
||||
- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a)
|
||||
- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176)
|
||||
- test(fold-checkpoint): the ARM-AT-FLIP pin arms its crash instead of racing the pending-flush timer (4176439b)
|
||||
- fix(health): one contract for a throwing probe — heal is none, serving is not withheld; repair report gains missing/rebuilt/reason (116550eb)
|
||||
- feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete (7c8c8be3)
|
||||
- fix(delete): the null-metadata skip closes — index legs run id-keyed or narrate, never silently strand postings (607e9f54)
|
||||
- feat(repair): repairIndex returns the per-family receipt and narrates its summary (8d45f964)
|
||||
- fix(reads): the readiness gate guards every index read surface — serving empty from a not-ready provider is unrepresentable (40e7119b)
|
||||
- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1)
|
||||
|
||||
|
||||
### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18)
|
||||
|
||||
- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895)
|
||||
- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9)
|
||||
|
||||
|
||||
### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18)
|
||||
### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18)
|
||||
|
||||
- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649)
|
||||
- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28)
|
||||
|
|
@ -124,14 +11,14 @@ All notable changes to this project will be documented in this file. See [standa
|
|||
- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706)
|
||||
|
||||
|
||||
### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17)
|
||||
### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17)
|
||||
|
||||
- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f)
|
||||
- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e)
|
||||
- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838)
|
||||
|
||||
|
||||
### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13)
|
||||
### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13)
|
||||
|
||||
- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696)
|
||||
- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667)
|
||||
|
|
@ -140,7 +27,7 @@ All notable changes to this project will be documented in this file. See [standa
|
|||
- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d)
|
||||
|
||||
|
||||
### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12)
|
||||
### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12)
|
||||
|
||||
- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96)
|
||||
- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38)
|
||||
|
|
@ -172,7 +59,7 @@ All notable changes to this project will be documented in this file. See [standa
|
|||
- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8)
|
||||
|
||||
|
||||
### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04)
|
||||
### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04)
|
||||
|
||||
- docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2)
|
||||
- fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed)
|
||||
|
|
@ -207,7 +94,7 @@ All notable changes to this project will be documented in this file. See [standa
|
|||
- feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b)
|
||||
|
||||
|
||||
### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27)
|
||||
### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27)
|
||||
|
||||
- docs: the last two archived-host links point home (91ef1c8b)
|
||||
- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9)
|
||||
|
|
@ -216,19 +103,19 @@ All notable changes to this project will be documented in this file. See [standa
|
|||
- ci: run the pipeline on the forge (999d0ebb)
|
||||
|
||||
|
||||
### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03)
|
||||
### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03)
|
||||
|
||||
- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608)
|
||||
- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859)
|
||||
|
||||
|
||||
### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
|
||||
### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29)
|
||||
|
||||
- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b)
|
||||
- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82)
|
||||
|
||||
|
||||
### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
|
||||
### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24)
|
||||
|
||||
- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5)
|
||||
- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74)
|
||||
|
|
|
|||
10
CLAUDE.md
10
CLAUDE.md
|
|
@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md`
|
|||
|
||||
**Brainy's current open actions:** None. MIT open-source — no platform-specific actions.
|
||||
|
||||
**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md`
|
||||
**Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md`
|
||||
|
||||
---
|
||||
|
||||
## Project Overview
|
||||
|
||||
Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license.
|
||||
Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license.
|
||||
|
||||
## Getting Started
|
||||
|
||||
|
|
@ -91,7 +91,7 @@ test: add/update tests (patch version bump)
|
|||
|
||||
## Docs Pipeline — soulcraft.com/docs
|
||||
|
||||
Docs in `docs/**/*.md` are published with the npm package (included in `files`) and go live on soulcraft.com/docs via the docs ingest API: the release script's `scripts/push-docs.js` step POSTs every public doc to `https://soulcraft.com/api/docs/ingest` (auth: `DOCS_INGEST_SECRET` in the environment). No separate deploy step is involved (the old deploy-to-publish flow was retired in a platform change, 2026-08). Frontmatter controls what appears publicly.
|
||||
Docs in `docs/**/*.md` are published with the npm package (included in `files`) and synced to soulcraft.com/docs on every portal deploy. Frontmatter controls what appears publicly.
|
||||
|
||||
### Docs check triggers
|
||||
|
||||
|
|
@ -161,9 +161,9 @@ npm run release:major # Breaking changes (rare, manual decision)
|
|||
The script: verifies clean git state, builds, tests, bumps version, updates CHANGELOG.md, commits, tags, pushes, publishes to npm, and creates a GitHub release.
|
||||
|
||||
After a successful release, remind the user:
|
||||
> "Published. Docs are live on soulcraft.com/docs (pushed via the ingest API during the release) — spot-check a changed page with curl."
|
||||
> "Published. Deploy portal to pick up the new docs → go to the portal project and deploy."
|
||||
|
||||
There is no separate deploy step anymore. If the docs push failed (the script warns loudly), re-run `node scripts/push-docs.js` with `DOCS_INGEST_SECRET` set.
|
||||
Do NOT deploy portal from here. Portal is always deployed separately from within the portal project.
|
||||
|
||||
## Closed-Source Product Names — HARD RULE
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ may find elsewhere in the repo's history.
|
|||
|
||||
## Where the project lives
|
||||
|
||||
The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**.
|
||||
The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**.
|
||||
It's anonymously readable and cloneable — no account needed to browse, clone,
|
||||
or build.
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework.
|
|||
## Development setup
|
||||
|
||||
```bash
|
||||
git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git
|
||||
git clone https://source.soulcraft.com/soulcraft/brainy.git
|
||||
cd brainy
|
||||
npm install
|
||||
npm run build
|
||||
|
|
@ -57,17 +57,6 @@ see `package.json` for `test:integration`, `test:coverage`, and friends.
|
|||
description states a number, cite the benchmark that produced it (see
|
||||
[docs/performance-envelopes.md](docs/performance-envelopes.md) for the
|
||||
pattern). Don't state an estimate as if it were measured.
|
||||
- **Measurements carry numbers, not provenance.** Public commit messages and
|
||||
docs give the SHAPE a number was taken at and never where it was taken: no
|
||||
hostnames, no store or deployment identities, no operational anecdotes about
|
||||
someone's running system. "A 14,056-noun / 72,679-verb production-shaped
|
||||
store, measured solo under an exclusive lock" tells a reader everything the
|
||||
number depends on; the machine it ran on and whose data it was tell them
|
||||
nothing except where somebody's infrastructure lives.
|
||||
- **Documents that answer or reference a confidential specification never enter
|
||||
this repository, even summarized.** The public docs describe THIS engine and
|
||||
the published contract, and nothing else — a summary of a private document is
|
||||
still that document's contents.
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
18
README.md
18
README.md
|
|
@ -1,5 +1,5 @@
|
|||
<p align="center">
|
||||
<img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/raw/branch/main/brainy.png" alt="Brainy" width="180">
|
||||
<img src="https://source.soulcraft.com/soulcraft/brainy/raw/branch/main/brainy.png" alt="Brainy" width="180">
|
||||
</p>
|
||||
|
||||
<h1 align="center">Brainy</h1>
|
||||
|
|
@ -11,9 +11,9 @@
|
|||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://source.soulcraft.com/soulcraftlabs/-/packages/npm/brainy"><img src="https://img.shields.io/badge/package-The%20Source-2c3e50.svg" alt="Package on The Source"></a>
|
||||
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy"><img src="https://img.shields.io/badge/repo-open--brainy-2c3e50.svg" alt="Repository"></a>
|
||||
<a href="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions"><img src="https://source.soulcraft.com/soulcraftlabs/open-brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
|
||||
<a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/v/@soulcraft/brainy.svg" alt="npm version"></a>
|
||||
<a href="https://www.npmjs.com/package/@soulcraft/brainy"><img src="https://img.shields.io/npm/dm/@soulcraft/brainy.svg" alt="npm downloads"></a>
|
||||
<a href="https://source.soulcraft.com/soulcraft/brainy/actions"><img src="https://source.soulcraft.com/soulcraft/brainy/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
|
||||
<a href="https://soulcraft.com/docs"><img src="https://img.shields.io/badge/docs-soulcraft.com-blue.svg" alt="Documentation"></a>
|
||||
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License"></a>
|
||||
<a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/%3C%2F%3E-TypeScript-%230074c1.svg" alt="TypeScript"></a>
|
||||
|
|
@ -30,8 +30,6 @@
|
|||
|
||||
---
|
||||
|
||||
**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license.
|
||||
|
||||
Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together:
|
||||
|
||||
| You write | Brainy indexes it as | You query it with |
|
||||
|
|
@ -47,14 +45,12 @@ It runs **inside your process** — no server, no Docker, nothing to operate —
|
|||
## Quick start
|
||||
|
||||
```bash
|
||||
bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended
|
||||
npm install @soulcraftlabs/brainy # Node.js ≥ 22
|
||||
bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended
|
||||
npm install @soulcraft/brainy # Node.js ≥ 22
|
||||
```
|
||||
|
||||
> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read).
|
||||
|
||||
```javascript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy() // in-memory; one line swaps to disk
|
||||
await brain.init()
|
||||
|
|
|
|||
353
RELEASES.md
353
RELEASES.md
|
|
@ -1,7 +1,7 @@
|
|||
# @soulcraft/brainy — Release Notes for Consumers
|
||||
|
||||
This file is the **quick reference for downstream sessions** tracking Brainy changes.
|
||||
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases
|
||||
Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases
|
||||
|
||||
**How to use:** Brainy is the underlying data engine for downstream applications. Read this when:
|
||||
- Upgrading `@soulcraft/brainy` in your application
|
||||
|
|
@ -31,357 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the
|
|||
|
||||
---
|
||||
|
||||
## v10.4.4 — 2026-08-28
|
||||
|
||||
**A correctness and observability release.** The headline is not speed: it is that a
|
||||
restart now tells you the truth about itself, a store stops lying about how much it
|
||||
holds, and the engine stops doing work nobody asked for. There is a performance
|
||||
improvement and it is modest; it is stated exactly below rather than rounded up.
|
||||
|
||||
### The dark restart — fixed at the root
|
||||
|
||||
A service could stop cleanly, exit 0, having awaited `close()` on every store it held,
|
||||
and its next boot would announce `Overwriting stale writer lock … appears dead` for
|
||||
every one of them. Nothing had crashed. Two deployments hit this; the same defect also
|
||||
made those boots pay a crash-recovery fold they did not owe.
|
||||
|
||||
The cause was not the lock. `close()` released it correctly — when it got there. A
|
||||
failure part-way through close skipped both the release AND the clean-shutdown marker,
|
||||
and "the recorded pid is gone" reads identically for an orderly restart and a crash.
|
||||
|
||||
- `close()` is now two parts and the second is unconditional: the flush-request watcher,
|
||||
the **writer lock**, the VFS timers and the terminal `closed` flag are released whether
|
||||
the durable steps succeeded or not. The original failure is narrated with what it costs
|
||||
the next open, then rethrown.
|
||||
- Releasing the lock writes a **clean-close record** naming the lock generation it gave
|
||||
up. The next open reads that record instead of guessing: recorded → nothing to recover;
|
||||
absent → it says so, and names the recovery it is about to run. This also ends two
|
||||
long-standing false alarms — a recycled pid locking a store out of its own reopen, and
|
||||
`Re-acquiring writer lock … this is a bug` after a perfectly clean close.
|
||||
- The signal path stopped failing in a batch. One store's failing flush used to strand
|
||||
every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the
|
||||
generation store's close (the marker) is part of shutdown, the lock goes in a `finally`,
|
||||
and the handler no longer calls `process.exit()` when the host application has its own
|
||||
signal handler, a race that truncated the host's own shutdown mid-flight.
|
||||
|
||||
### The count ledger stops lying, and `counts.json` is written atomically
|
||||
|
||||
The all-tier scalars are the denominator a coverage check subtracts against. A ledger
|
||||
derived under the old rule — one entity per id DIRECTORY — counted ghost and scar
|
||||
containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for
|
||||
the life of the store. Two copies of one archive could disagree, and a downstream index
|
||||
heal reported remaining work that did not exist.
|
||||
|
||||
- Such a ledger now derives itself honestly **in the background** after the open, counting
|
||||
identity records, and persists the correction stamped. Nothing waits for it, because no
|
||||
read is served from a denominator.
|
||||
- A derivation that raced a write refuses to stamp its number: one retry on a quiet store,
|
||||
then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under
|
||||
a barrier.
|
||||
- `counts.json` is written temp+rename. A truncating write left a window in which a
|
||||
concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open
|
||||
down the full-rescan path, so the cheapest file in the store was buying the most
|
||||
expensive recovery.
|
||||
|
||||
### An open and a repair narrate themselves — on a channel a log level cannot silence
|
||||
|
||||
A store could open for three minutes and print nothing at all. The phase timings existed;
|
||||
they were written to a channel that every production-looking environment clamps away.
|
||||
|
||||
- Narration moved to an always-visible channel. An open now heartbeats the phase it is in,
|
||||
names each phase as it ends with what it was paying for, and names the expensive STEP
|
||||
inside a phase. `repairIndex()` does the same and its receipt carries a per-family
|
||||
`durationMs` — a repair that ran for half an hour with no output could only be watched
|
||||
through `top`.
|
||||
- A brain nobody has written to now does nothing: a flush over a clean store is a no-op
|
||||
and says nothing, the graph index's auto-flush asks before it acts, and the
|
||||
cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a
|
||||
directory every 500 ms per store forever, with a slow safety sweep behind it and a
|
||||
narrated fall back to polling where a filesystem cannot be watched.
|
||||
- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()`
|
||||
does not wait for it, every other family serves, and that family's doors refuse **by
|
||||
name, carrying the provider's own progress**, saying plainly that they open by
|
||||
themselves and no action is needed. Health narration dedupes by content, so an unchanged
|
||||
verdict is silent however a provider's generation counter moves.
|
||||
|
||||
### For operators — one behaviour change
|
||||
|
||||
**Four `where` operators that previously returned an empty page now raise
|
||||
`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range
|
||||
posting index cannot evaluate a substring, a pattern or an array length without reading
|
||||
every row, and it now refuses by name instead of answering with an empty result that
|
||||
looks like an answer.
|
||||
|
||||
**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and
|
||||
`excludes`. All 25 accepted operator tokens now agree between this engine and its
|
||||
accelerated counterpart.
|
||||
|
||||
### Performance — stated exactly
|
||||
|
||||
Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under
|
||||
an exclusive lock:
|
||||
|
||||
- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is
|
||||
one fix — generation discovery reads directory NAMES instead of recursively walking the
|
||||
entire generation log (−9.2 s, and it scales with history rather than row count). The
|
||||
VFS phase is **unchanged**.
|
||||
- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving
|
||||
off the critical path accounts for storage-init dropping 5,941 ms → 25 ms.
|
||||
- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own
|
||||
init is under 2 s of that phase. It is the log-authority adoption and/or the
|
||||
pending-embed log recovery, both now instrumented so the next measurement names the
|
||||
culprit outright.
|
||||
|
||||
Continuing work, named so nobody has to rediscover it: that ~38 s term; making the
|
||||
generation store's committed-range set lazy; the hydration path that substitutes
|
||||
`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix
|
||||
filter built with a `$startsWith` spelling no operator set accepts, so
|
||||
`searchFiles({ path })` throws today.
|
||||
|
||||
---
|
||||
|
||||
## v10.4.3 — 2026-08-27 (Open Brainy's first release)
|
||||
|
||||
**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for
|
||||
byte — only the name, the registry, and the pointers changed.** Install:
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy
|
||||
```
|
||||
|
||||
with the registry line in your `.npmrc` (anonymous read):
|
||||
|
||||
```
|
||||
@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/
|
||||
```
|
||||
|
||||
- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the
|
||||
npmjs republish step is retired from the release rail. Existing npmjs versions of
|
||||
`@soulcraft/brainy` stay as they are and receive no new versions.
|
||||
- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects.
|
||||
- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one
|
||||
install-line change (`@soulcraft/brainy` → `@soulcraftlabs/brainy`), which downstream
|
||||
applications make together with their native-engine bump.
|
||||
|
||||
## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector)
|
||||
|
||||
**This is the last release of the MIT engine under the `@soulcraft/brainy` name.**
|
||||
The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API,
|
||||
client library, types and protocol, an openly specified canonical format, and the TypeScript
|
||||
reference engine, scoped honestly as a single-node engine for stores up to roughly one
|
||||
million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a
|
||||
major version bump; that engine implements the same API over the same open format at
|
||||
production scale, requires a license, and refuses loudly without one. Nothing changes
|
||||
for existing installs until that major ships; the move is announced with it.
|
||||
|
||||
Six fixes, one law: a vector with no magnitude carries no information, so it must
|
||||
never reach a vector index — in any engine — and the canonical store must say so.
|
||||
|
||||
- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item
|
||||
shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted
|
||||
with an empty vector leg, never embedded, never indexed, counted as unvectored in
|
||||
the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing —
|
||||
no longer need a placeholder vector and never enter the vector leg. `vector: []`
|
||||
together with `deferEmbedding: true` is refused with a typed error (a supplied
|
||||
vector has nothing to defer). Previously `vector: []` threw a dimension error.
|
||||
- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is
|
||||
the sanctioned way to strip a vector from an existing row: canonical vector → `[]`,
|
||||
removal from the vector index, the vectored ledger decremented exactly once — and
|
||||
idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and
|
||||
it clears a pending deferred-embed marker durably so the background worker cannot
|
||||
re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the
|
||||
index from canonical rows); shedding historical vectors needs this door.
|
||||
- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on
|
||||
any write path is persisted as unvectored (`[]`) with one warning naming the row;
|
||||
the vector-index operations keep their own refusal as a second line. The engine's
|
||||
own VFS root, which used to persist a deliberate all-zero placeholder (harmless
|
||||
under cosine distance, a false attractor under a downstream engine's
|
||||
squared-euclidean serving — a production incident this week), is now created
|
||||
unvectored, and an existing store's legacy root is migrated on open by a single
|
||||
fixed-path read before the health gate runs — never a walk.
|
||||
- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the
|
||||
cursor walks behind them enumerate by the metadata record, the same key the
|
||||
canonical ledger counts by — previously the walk keyed on the vector file, so a
|
||||
row holding metadata but no vector was counted yet never yielded (a permanent
|
||||
"missing" phantom in coverage math), while an orphaned vector-only directory
|
||||
could be yielded as a phantom id. The recovery fold also never deletes an existing
|
||||
vector when it replays a metadata-only after-image (preserve-if-absent). One
|
||||
documented gap remains: a verb's endpoints live only in its vector leg, so a
|
||||
metadata-only verb is counted and loudly skipped, never fabricated — the fix is a
|
||||
canonical-format change and lands with the open format.
|
||||
- **The ledger's one-time derivation counts identity records.** Stores upgraded from
|
||||
pre-ledger versions derived their ALL-visibility scalars once by counting id
|
||||
directories, which included ghost and scar containers left by an old partial-delete
|
||||
defect — an inflated denominator whose coverage row could never reach exact. The
|
||||
derivation now counts only directories holding a metadata record, `counts.json`
|
||||
carries a derivation-rule stamp, and a ledger derived under the old rule is marked
|
||||
`suspect` at open (one O(1) field read, one warning) so the online `repairIndex()`
|
||||
path clears it with a real recount.
|
||||
- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and
|
||||
zero-norm rows (one summary line), re-pins the vector dimension from the first real
|
||||
vector after a restart (previously a restart left the pin unset, so a wrong-length
|
||||
insert became the new pin instead of being rejected), and `addItem` / `updateItem`
|
||||
throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing
|
||||
a vector-less node.
|
||||
- **Smaller:** a failing plugin activation now rethrows with the original error as
|
||||
`cause` (the originating file and line survive to the caller's log); build
|
||||
generators stamp from the repository history of their inputs instead of wall clock,
|
||||
so two builds of the same tree are byte-identical.
|
||||
|
||||
Adoption: one restart, paired with its native-engine release. The first open of an
|
||||
existing store runs the legacy-root migration (one narrated line) and, on stores that
|
||||
upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned
|
||||
recount — no rebuild in either case.
|
||||
|
||||
## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds)
|
||||
|
||||
Two production defects from the same week, fixed together as a patch to 10.4.0.
|
||||
|
||||
- **The read gate is per family.** A read now refuses only when the index family it
|
||||
actually consults is unhealthy: a metadata filter is served while the vector leg is
|
||||
rebuilding; a semantic query is refused only by the vector family; a graph
|
||||
traversal only by the graph family. Previously any unhealthy family refused every
|
||||
read on the brain — under a long vector rebuild, a production deployment's
|
||||
metadata-only reads were refused for the duration, and the retries became a write
|
||||
pump of their own.
|
||||
- **Unchanged data never re-embeds.** `update()` compares the incoming `data`
|
||||
structurally with the stored record; an update carrying identical data (a common
|
||||
shape for periodic upserts) no longer embeds again and no longer churns the vector
|
||||
leg. Previously every such update re-embedded and re-inserted, which under load
|
||||
saturated the vector index with near-identical vectors.
|
||||
|
||||
Adoption: one restart, paired with its native-engine release.
|
||||
|
||||
## v10.4.0 — 2026-08-25 (the health report has a name)
|
||||
|
||||
Three related cures, one root cause: an index deciding whether it could be trusted
|
||||
by sampling itself instead of by exact accounting. This release replaces every
|
||||
sampled self-probe with ledger-derived truth, and a read against an unhealthy index
|
||||
now refuses loudly instead of guessing.
|
||||
|
||||
- **The canonical count ledger.** Storage now tracks two scalars per family
|
||||
(nouns/verbs) on the write path: the user-facing `counted` total — unchanged,
|
||||
still what `getNounCount()` / `getVerbCount()` return — and a new ALL-visibility
|
||||
`all` total covering every tier, the real denominator a derived index's own
|
||||
coverage math needs. The unfiltered storage-level `totalCount` returned by
|
||||
`getNouns()` / `getVerbs()` is now this unclamped ALL scalar; previously it could
|
||||
only ever move up (`Math.max(scalar, scanned)`), so an inflated counter could
|
||||
never self-correct. A delete that cannot prove the record it removed actually
|
||||
existed (no canonical read, no prior image available) no longer decrements on
|
||||
faith — it marks the ledger `suspect` (narrated once per session) instead of
|
||||
silently drifting, and the next `repairIndex()` clears the flag with a real
|
||||
recount.
|
||||
- **One contract for a throwing health probe.** A provider's `validateInvariants()`
|
||||
is documented to never throw — but if one does anyway (a bug, a transient fault),
|
||||
it is now read the same way everywhere: `heal: 'none'`, the error named in the
|
||||
report, never synthesized into a rebuild trigger and never swallowed into "looks
|
||||
fine." A flaky check can no longer buy itself a rebuild. `repairIndex()`'s
|
||||
per-family receipt also gains `missing` (an exact count plus a capped id sample),
|
||||
`rebuilt` (a full rebuild ran, vs. an incremental heal), and `reason`.
|
||||
- **The named health report; reads refuse instead of rebuilding.** Any index
|
||||
provider may now expose a synchronous, O(1) `healthReport()` — composed from the
|
||||
provider's own exact ledgers, never a sample — and this is the one signal
|
||||
Brainy's read gate trusts. The first-query lazy-build path is gone: `brain.init()`
|
||||
now runs every needed rebuild to completion before it returns, always, regardless
|
||||
of dataset size. A read that lands on a provider whose health report says it
|
||||
isn't serving throws a typed error instead of triggering a rebuild mid-query —
|
||||
`GraphIndexNotReadyError`, `MetadataIndexNotReadyError`, or
|
||||
`VectorIndexNotReadyError` (all exported from `@soulcraft/brainy`), naming the
|
||||
reasons. `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] | 'all' })` is
|
||||
the new explicit operator door: it rebuilds the named family unconditionally, no
|
||||
health check consulted — reach for it when you have independent reason to
|
||||
distrust a family regardless of what it self-reports. Bare `repairIndex()` is
|
||||
unchanged in spirit: report-driven, heals only what its own checks say needs it.
|
||||
- New concept doc: [Index Health](docs/concepts/index-health.md) walks the whole
|
||||
story from a consumer's side — degraded-but-serving vs. not-ready, what
|
||||
`repairIndex()` checks and heals per family, what `suspect` counts mean.
|
||||
|
||||
**Nothing to change to adopt this.** No API removed, no signature narrowed —
|
||||
`repairIndex()` gains an optional options bag and its return value gains fields,
|
||||
both additive. The honest notes: if your code ever relied on a `find()` against a
|
||||
cold/not-yet-built index quietly triggering a rebuild and returning results a beat
|
||||
later, that behavior is gone — it now throws one of the three typed
|
||||
`*NotReadyError` classes instead (catch them if you need to distinguish "not ready
|
||||
yet" from "no results"). And `disableAutoRebuild: true` no longer defers index
|
||||
construction to the first query — a needed rebuild always runs at `open()` now;
|
||||
the flag has no effect on timing. Full manual control still lives in
|
||||
`repairIndex({ rebuild: [...] })`.
|
||||
|
||||
- **Crash-reopen catchup.** After an unclean shutdown, the metadata index now
|
||||
folds the exact fact window it missed — `find()` serves every acked write on
|
||||
reopen, closing the gap where canonical reads and counts recovered a
|
||||
crash-window write but the index kept serving its pre-crash state until the
|
||||
next full rebuild. Related root-cause fixed alongside: `close()` never
|
||||
stamped the index watermarks (only `flush()` did), so a close without a
|
||||
prior flush caused a needless full rescan verdict on the next open.
|
||||
- **Relation rows are live in the metadata index.** Previously verb rows
|
||||
entered the metadata index only during a rebuild — so a rebuilt store's
|
||||
relation postings went stale from the first `relate()` after it. Relations
|
||||
are now posted and retracted on the live write path (relate / unrelate /
|
||||
updateRelation / remove's cascade, and their `transact()` forms), in the
|
||||
same commit as the graph leg.
|
||||
- **The metadata rebuild is online.** `rebuild()` for the metadata family no
|
||||
longer clears and rebuilds in place (reads went empty for the duration): it
|
||||
builds a complete replacement beside the serving index, mirrors concurrent
|
||||
writes to both, swaps atomically, and persists once after the swap. Reads
|
||||
never observe a partial index. `repairIndex({ rebuild: ['metadata'] })` uses
|
||||
it automatically.
|
||||
- **Incremental heal is routed.** A provider invariant that asks for the
|
||||
incremental heal (`heal: 'repair'`) now routes to the provider's own
|
||||
`repair()` when it exposes one — re-posting exactly what its ledger names,
|
||||
never a store-sized rebuild — and the post-heal re-read of the report decides
|
||||
success; a repair that doesn't converge is recorded with the escalation named.
|
||||
- **The vector family joins the count ledger.** `getCanonicalCounts()` gains
|
||||
`vectors: { all }` — the count of canonical entities holding a real vector
|
||||
(deferred-embed entities count when their vector lands). And the open gate
|
||||
closes the vector leg: a store whose canonical rows hold vectors but whose
|
||||
derived vector index is empty now builds at `open()` (or refuses with the
|
||||
typed error) instead of silently serving empty vector-search results.
|
||||
- **An unknown storage config shape fails loudly.** A nested `config` object
|
||||
carrying a path-shaped key (a shape that was never supported) used to fall
|
||||
through silently to the default shared directory — every instance writing one
|
||||
store while callers believed each had its own. It now throws, naming the
|
||||
canonical `path` key.
|
||||
- **Relation index rows are JSON-safe.** Internal endpoint identifiers can no
|
||||
longer ride the metadata-index crossing (a native provider serializes it);
|
||||
they stay on the graph operations where they belong.
|
||||
- **A broken accelerator install can never read as "not installed."** The
|
||||
auto-detection free pass now requires the resolution error to name the
|
||||
accelerator package itself, exactly — a missing platform-binary sibling
|
||||
package, an inner file path, or a dependency failure is a broken install and
|
||||
`init()` throws loudly. And a plugin that declines activation is narrated on
|
||||
the always-on log channel, so `silent: true` can no longer hide a fallback
|
||||
to the default engines.
|
||||
|
||||
---
|
||||
|
||||
## v10.3.1 — 2026-08-18 (the fold that behaves)
|
||||
|
||||
Three recovery cures from one production first-boot incident (a brain's first
|
||||
process restart after a live storage-authority flip looked hung and was
|
||||
restarted three times mid-recovery). **Adopt this version before flipping
|
||||
brains with existing history** — it is the intended adoption target for
|
||||
fleets moving to the crash-safe authority.
|
||||
|
||||
- **Recovery streams.** The boot-time log fold now consumes the generation
|
||||
log one segment-batch at a time — memory stays bounded at one segment for
|
||||
any log size. Previously it materialized every fact into one array, which
|
||||
on a ~7k-fact log produced multi-GB allocation pressure and a process that
|
||||
looked wedged while it worked.
|
||||
- **Recovery narrates.** The fold announces itself before the work begins
|
||||
("recovery fold beginning — do not restart, the fold is finite") and prints
|
||||
progress every thousand facts. A visible fold gets to finish; a silent one
|
||||
gets killed by a well-meaning operator, and each kill makes the next boot
|
||||
pay the whole fold again.
|
||||
- **Bounded recovery from the flip itself.** Adopting the log authority now
|
||||
founds the recovery checkpoint at the moment of the flip (one paged
|
||||
canonical sync, bounded memory, then the stamp) — so even the FIRST unclean
|
||||
shutdown after a flip replays only the log's tail. Previously the bound
|
||||
could only establish itself at a completed crash recovery, which is exactly
|
||||
the recovery the incident kept interrupting.
|
||||
|
||||
---
|
||||
|
||||
## v10.3.0 — 2026-08-18 (the trust-and-provenance release)
|
||||
|
||||
Four consumer-driven cures. Pairs with the same native accelerator line
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ commit to backporting fixes to unsupported lines.
|
|||
|
||||
## Scope
|
||||
|
||||
This policy covers the `@soulcraftlabs/brainy` package itself — the code in
|
||||
This policy covers the `@soulcraft/brainy` package itself — the code in
|
||||
this repository. If you're evaluating a deployment that also uses
|
||||
`@soulcraft/cor`, report issues in that package the same way, to the same
|
||||
address; we'll route internally.
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
/**
|
||||
* Modern TypeScript CLI Runner
|
||||
*
|
||||
* This is the entry point after npm install @soulcraftlabs/brainy
|
||||
* This is the entry point after npm install @soulcraft/brainy
|
||||
* It runs the compiled TypeScript CLI code
|
||||
*/
|
||||
|
||||
|
|
|
|||
2
bun.lock
2
bun.lock
|
|
@ -3,7 +3,7 @@
|
|||
"configVersion": 0,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "@soulcraftlabs/brainy",
|
||||
"name": "@soulcraft/brainy",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.540.0",
|
||||
"@azure/identity": "^4.0.0",
|
||||
|
|
|
|||
|
|
@ -25,13 +25,13 @@
|
|||
|
||||
### Prerequisites
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
### Your First Neural Database
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||
|
||||
// Step 1: Create and initialize Brainy
|
||||
const brain = new Brainy({
|
||||
|
|
@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou
|
|||
### Building a Knowledge Graph
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
|
@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**.
|
|||
### Triple Intelligence in Action
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
|
@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in
|
|||
### Files as Intelligent Entities
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
|
|
@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**.
|
|||
### Production-Ready Deployment
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||
|
||||
// 1. PRODUCTION STORAGE - Filesystem with off-site snapshots
|
||||
console.log('Initializing production storage...\n')
|
||||
|
|
|
|||
|
|
@ -1217,7 +1217,7 @@ where: {
|
|||
await brain.find({ type: 'Document' })
|
||||
|
||||
// ✅ Correct: Use NounType enum
|
||||
import { NounType } from '@soulcraftlabs/brainy'
|
||||
import { NounType } from '@soulcraft/brainy'
|
||||
await brain.find({ type: NounType.Document })
|
||||
|
||||
// ❌ Error: Operator not recognized
|
||||
|
|
|
|||
|
|
@ -153,13 +153,13 @@ brainy-data/
|
|||
### Step 1: Update Brainy Package
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy@latest
|
||||
npm install @soulcraft/brainy@latest
|
||||
```
|
||||
|
||||
**Check your version:**
|
||||
```bash
|
||||
npm list @soulcraftlabs/brainy
|
||||
# Should show: @soulcraftlabs/brainy@4.0.0
|
||||
npm list @soulcraft/brainy
|
||||
# Should show: @soulcraft/brainy@4.0.0
|
||||
```
|
||||
|
||||
### Step 2: No Code Changes Required! ✅
|
||||
|
|
@ -374,7 +374,7 @@ If you encounter issues, you can rollback:
|
|||
|
||||
```bash
|
||||
# Reinstall v3
|
||||
npm install @soulcraftlabs/brainy@^3.50.0
|
||||
npm install @soulcraft/brainy@^3.50.0
|
||||
|
||||
# Restart application
|
||||
```
|
||||
|
|
@ -389,7 +389,7 @@ rm -rf ./data
|
|||
cp -r ./data-backup ./data
|
||||
|
||||
# Reinstall v3
|
||||
npm install @soulcraftlabs/brainy@^3.50.0
|
||||
npm install @soulcraft/brainy@^3.50.0
|
||||
```
|
||||
|
||||
## Common Migration Scenarios
|
||||
|
|
@ -539,7 +539,7 @@ console.log('Storage type:', status.type)
|
|||
|
||||
**Migration Checklist:**
|
||||
- ✅ Backup data
|
||||
- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`)
|
||||
- ✅ Update npm package (`npm install @soulcraft/brainy@latest`)
|
||||
- ✅ Restart application (automatic migration)
|
||||
- ✅ Verify data integrity
|
||||
- ✅ Enable lifecycle policies
|
||||
|
|
|
|||
|
|
@ -323,24 +323,58 @@ Only the graph adjacency index carries a committed scale assertion:
|
|||
- ✅ **Single-Node by Design**: One process owns one `path`; scale out at the service layer
|
||||
- ✅ **Zero Stubs**: Every line of code is production-ready
|
||||
|
||||
## Index Build at Open (10.4+)
|
||||
## Lazy Loading Performance
|
||||
|
||||
As of 10.4, `brain.init()` runs every needed index rebuild to completion before
|
||||
it returns — always, regardless of dataset size. There is no lazy,
|
||||
first-query rebuild path: a brain either finishes opening healthy, or `init()`
|
||||
fails loudly. `disableAutoRebuild` no longer defers index construction to a
|
||||
first query; it has no effect on *when* a rebuild runs. Manual control over
|
||||
rebuilds is `repairIndex({ rebuild: [...] })`. See
|
||||
[Index Health](concepts/index-health.md) for the full read-gate contract
|
||||
(providers self-report readiness via `healthReport()`; a read against a
|
||||
not-serving provider throws a typed `*NotReadyError` rather than rebuilding
|
||||
mid-query).
|
||||
Brainy supports two initialization modes for optimal performance across different use cases:
|
||||
|
||||
<!-- The pre-10.4 "Mode 2: Lazy Loading on First Query" section previously
|
||||
documented here (disableAutoRebuild deferring index construction to the
|
||||
first find() call) described a real, now-retired code path. Removed
|
||||
rather than left to mislead; the concept doc above is the current
|
||||
contract. -->
|
||||
### Mode 1: Auto-Rebuild (Default)
|
||||
|
||||
```javascript
|
||||
const brain = new Brainy()
|
||||
await brain.init() // Rebuilds indexes during init (~500ms-3s for 10K entities)
|
||||
```
|
||||
|
||||
**Performance:**
|
||||
- Init time: 500ms-3s (depends on dataset size)
|
||||
- First query: Instant (indexes already loaded)
|
||||
- Use case: Traditional applications, long-running servers
|
||||
|
||||
### Mode 2: Lazy Loading
|
||||
|
||||
```javascript
|
||||
const brain = new Brainy({ disableAutoRebuild: true })
|
||||
await brain.init() // Returns instantly (0-10ms)
|
||||
|
||||
const results = await brain.find({ limit: 10 }) // First query triggers rebuild (~50-200ms)
|
||||
const more = await brain.find({ limit: 100 }) // Subsequent queries instant (0ms check)
|
||||
```
|
||||
|
||||
**Performance:**
|
||||
- Init time: 0-10ms (instant)
|
||||
- First query: 50-200ms (includes index rebuild for 1K-10K entities)
|
||||
- Subsequent queries: 0ms check (instant)
|
||||
- Concurrent queries: Wait for same rebuild (mutex prevents duplicates)
|
||||
|
||||
**Concurrency Safety:**
|
||||
```javascript
|
||||
// 100 concurrent queries immediately after init
|
||||
await brain.init()
|
||||
|
||||
const promises = Array.from({ length: 100 }, () =>
|
||||
brain.find({ limit: 10 })
|
||||
)
|
||||
|
||||
const results = await Promise.all(promises)
|
||||
// ✅ Only 1 rebuild triggered (mutex)
|
||||
// ✅ All 100 queries return correct results
|
||||
// ✅ Total time: ~60ms (not 6000ms!)
|
||||
```
|
||||
|
||||
**Use Cases for Lazy Loading:**
|
||||
- **Serverless/Edge**: Minimize cold start time (0-10ms init)
|
||||
- **Development**: Faster restarts during development
|
||||
- **Large datasets**: Defer index loading until needed
|
||||
- **Read-heavy workloads**: Writes don't wait for index rebuild
|
||||
|
||||
## Zero Configuration Required
|
||||
|
||||
|
|
@ -350,6 +384,10 @@ Brainy is designed to be **smart enough to tune itself dynamically**. No configu
|
|||
// That's it. Brainy handles everything.
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
||||
// Or with lazy loading for serverless
|
||||
const brain = new Brainy({ disableAutoRebuild: true })
|
||||
await brain.init() // Instant (0-10ms)
|
||||
```
|
||||
|
||||
### Automatic Self-Tuning
|
||||
|
|
@ -357,6 +395,7 @@ await brain.init()
|
|||
- **Metadata Index**: Auto-builds sorted indices for range queries on first use
|
||||
- **Graph Index**: Auto-flushes every 30 seconds
|
||||
- **Default Tuning**: Research-based vector index defaults
|
||||
- **Lazy Loading**: Indices built only when needed
|
||||
- **Cache Management**: LRU caches with TTL
|
||||
|
||||
### Intelligent Defaults
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ next:
|
|||
- guides/storage-adapters
|
||||
---
|
||||
|
||||
# Plugin System
|
||||
# Plugin Development Guide
|
||||
|
||||
Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cor` provides optional native acceleration, and it's the same system available to any developer.
|
||||
|
||||
|
|
@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement
|
|||
### 1. Implement the `BrainyPlugin` interface
|
||||
|
||||
```typescript
|
||||
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin'
|
||||
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
|
||||
|
||||
const myPlugin: BrainyPlugin = {
|
||||
name: 'my-brainy-plugin', // Must be unique (typically your npm package name)
|
||||
|
|
@ -90,7 +90,7 @@ await brain.init()
|
|||
**Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`:
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import myPlugin from './my-plugin.js'
|
||||
|
||||
const brain = new Brainy()
|
||||
|
|
@ -200,30 +200,15 @@ members so a warm reopen never pays a redundant rebuild-from-canonical:
|
|||
- **`init?(): Promise<void>`** — eager cold-load. Brainy awaits it once during
|
||||
`brain.init()`, after the metadata provider's `init()` (the id-mapper hydrates first)
|
||||
and **before the rebuild gate**.
|
||||
- **`healthReport?(): HealthReport`** — the PREFERRED signal (10.4+). A named,
|
||||
synchronous, O(1) verdict derived from the provider's own exact ledgers — never a
|
||||
sample, never I/O, must never throw for a well-formed provider. Brainy's read gate
|
||||
(`assessProviderHealth()`) reads this INSTEAD of `isReady()` / size heuristics when
|
||||
present: `serving: false` refuses the read with a typed `*NotReadyError` rather than
|
||||
triggering a rebuild — a read never starts a store walk. `healthy` marks every
|
||||
*verified* invariant holding; a family named in `unledgered` counts as neither
|
||||
healthy nor broken. See `HealthReport` / `LedgerInvariantResult` /
|
||||
`InvariantSource` in `src/plugin.ts`, and
|
||||
[Index Health](concepts/index-health.md) for the consumer-facing story.
|
||||
- **`isReady?(): boolean`** — honest durability signal, the fallback when
|
||||
`healthReport()` is absent. `true` ⇔ the persisted index is loaded (or cheaply
|
||||
demand-loadable) and consistent with what was last persisted. When exposed, the
|
||||
gate defers to this signal **instead of** the `size() === 0` / `totalEntries === 0`
|
||||
heuristics — a disk-native index may report 0 resident entries while fully durable.
|
||||
Never return `true` if the durable state failed to load: the signal is honest in
|
||||
both directions, and a not-ready provider gets its rebuild even when `size() > 0`.
|
||||
- **`isReady?(): boolean`** — honest durability signal. `true` ⇔ the persisted index is
|
||||
loaded (or cheaply demand-loadable) and consistent with what was last persisted. When
|
||||
exposed, the rebuild gate defers to this signal **instead of** the `size() === 0` /
|
||||
`totalEntries === 0` heuristics — a disk-native index may report 0 resident entries
|
||||
while fully durable. Never return `true` if the durable state failed to load: the
|
||||
signal is honest in both directions, and a not-ready provider gets its rebuild even
|
||||
when `size() > 0`.
|
||||
- **`isMigrating?(): boolean`** — while `true`, the provider owns its index (background
|
||||
migration); brainy skips its rebuild entirely.
|
||||
- **`validateInvariants?(): Promise<ProviderInvariantReport>`** — the async DEEP
|
||||
diagnostic (full scans allowed), distinct from the bounded, sync `healthReport()`.
|
||||
Must never throw — a failure is `healthy: false` data, not an exception; a provider
|
||||
that throws anyway is read as a loud, unverified failure (never as "healthy") by
|
||||
every caller, never silently retried into a rebuild.
|
||||
|
||||
Providers that implement none of these keep the size/count heuristics — correct for
|
||||
engines whose `rebuild()` *is* their load path (like brainy's built-in JS vector index).
|
||||
|
|
@ -272,10 +257,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor
|
|||
#### `cache`
|
||||
**Type:** `UnifiedCache`
|
||||
|
||||
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`).
|
||||
Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`).
|
||||
|
||||
```typescript
|
||||
import type { UnifiedCache } from '@soulcraftlabs/brainy/internals'
|
||||
import type { UnifiedCache } from '@soulcraft/brainy/internals'
|
||||
|
||||
context.registerProvider('cache', myNativeCache)
|
||||
```
|
||||
|
|
@ -325,8 +310,8 @@ Plugins can register custom storage backends that users reference by name.
|
|||
### Implementing a Storage Adapter
|
||||
|
||||
```typescript
|
||||
import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin'
|
||||
import type { StorageAdapter } from '@soulcraftlabs/brainy'
|
||||
import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin'
|
||||
import type { StorageAdapter } from '@soulcraft/brainy'
|
||||
|
||||
class MyStorageAdapter implements StorageAdapter {
|
||||
async init(): Promise<void> { /* ... */ }
|
||||
|
|
@ -360,9 +345,9 @@ Brainy provides three entry points for plugin developers:
|
|||
|
||||
| Import Path | Contents | Stability |
|
||||
|-------------|----------|-----------|
|
||||
| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) |
|
||||
| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) |
|
||||
| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) |
|
||||
| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) |
|
||||
| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) |
|
||||
| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) |
|
||||
|
||||
## Diagnostics
|
||||
|
||||
|
|
@ -440,7 +425,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations
|
|||
|
||||
```typescript
|
||||
// simd-distance-plugin/src/plugin.ts
|
||||
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin'
|
||||
import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin'
|
||||
|
||||
// Hypothetical native module
|
||||
import { simdCosineDistance } from './native.js'
|
||||
|
|
@ -470,7 +455,7 @@ export default simdDistancePlugin
|
|||
"main": "./dist/plugin.js",
|
||||
"types": "./dist/plugin.d.ts",
|
||||
"peerDependencies": {
|
||||
"@soulcraftlabs/brainy": ">=7.0.0"
|
||||
"@soulcraft/brainy": ">=7.0.0"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -478,7 +463,7 @@ export default simdDistancePlugin
|
|||
Usage:
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ plugins: ['brainy-simd-distance'] })
|
||||
await brain.init()
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ After 40 API calls:
|
|||
|
||||
```typescript
|
||||
// server.ts
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// SINGLETON INSTANCE
|
||||
let brainInstance: Brainy | null = null
|
||||
|
|
@ -174,7 +174,7 @@ process.on('SIGTERM', async () => {
|
|||
|
||||
```typescript
|
||||
// server.ts - Clean Bun implementation
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brain: Brainy | null = null
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
## Quick Start
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ Examples:
|
|||
|
||||
```bash
|
||||
# 1. Deprecate wrong version on npm
|
||||
npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y"
|
||||
npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y"
|
||||
|
||||
# 2. Fix version in package.json
|
||||
# 3. Republish correct version
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
### In-Memory
|
||||
```typescript
|
||||
import Brainy from '@soulcraftlabs/brainy'
|
||||
import Brainy from '@soulcraft/brainy'
|
||||
const brain = new Brainy({ storage: { type: 'memory' } })
|
||||
```
|
||||
|
||||
|
|
@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t
|
|||
|
||||
Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single
|
||||
Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the
|
||||
open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native
|
||||
open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native
|
||||
provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`.
|
||||
|
||||
`find()` query latency, p50 / p95 (200 queries each):
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -24,7 +24,7 @@ next:
|
|||
## Quick Start
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy() // Zero config!
|
||||
await brain.init() // VFS auto-initialized!
|
||||
|
|
@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization
|
|||
|
||||
### Db API errors
|
||||
|
||||
All exported from `@soulcraftlabs/brainy`:
|
||||
All exported from `@soulcraft/brainy`:
|
||||
|
||||
| Error | Thrown by | Meaning |
|
||||
|---|---|---|
|
||||
|
|
@ -1451,34 +1451,6 @@ const count = await brain.getVerbCount()
|
|||
|
||||
---
|
||||
|
||||
### The canonical count ledger (`StorageAdapter.getCanonicalCounts()`)
|
||||
|
||||
An OPTIONAL method on the `StorageAdapter` interface (implemented by both
|
||||
built-in adapters), not a method on `Brainy` itself — relevant if you're
|
||||
writing a custom storage adapter or composing a provider's own
|
||||
`healthReport()`. O(1), no I/O. Per family (`nouns`/`verbs`):
|
||||
|
||||
```typescript
|
||||
interface CanonicalCounts {
|
||||
nouns: { counted: number; all: number }
|
||||
verbs: { counted: number; all: number }
|
||||
suspect: boolean
|
||||
}
|
||||
```
|
||||
|
||||
- `counted` mirrors `getNounCount()` / `getVerbCount()` (public + internal tiers).
|
||||
- `all` is the ALL-visibility scalar — every tier, including system/internal
|
||||
records — the denominator a derived index's own coverage math is measured
|
||||
against.
|
||||
- `suspect` is `true` when an unprovable delete has left `all` unverified since
|
||||
the last recount; `brain.repairIndex()` clears it with a real canonical walk.
|
||||
|
||||
Adapters without the ledger omit the method; treat absence as "no
|
||||
denominator," never as zero. See
|
||||
**[Index Health](../concepts/index-health.md)** for the full story.
|
||||
|
||||
---
|
||||
|
||||
### Subtype & facet APIs
|
||||
|
||||
Full guide: **[Subtypes & Facets](../guides/subtypes-and-facets.md)**.
|
||||
|
|
@ -1859,104 +1831,6 @@ const semanticOnly = await brain.getStats({ excludeVFS: true })
|
|||
|
||||
---
|
||||
|
||||
### `repairIndex(options?)` → `Promise<RepairReport>`
|
||||
|
||||
The ceremony door for index repair. Bare `repairIndex()` is report-driven: it
|
||||
prunes orphaned containers, recomputes count rollups, reconciles VFS
|
||||
containment, and rebuilds only a derived-index family whose own health check
|
||||
asks for it. Pass `options.rebuild` to force one or more families to rebuild
|
||||
UNCONDITIONALLY — no health check is consulted — when an operator has
|
||||
independent reason to reconcile a family regardless of what it self-reports.
|
||||
|
||||
```typescript
|
||||
// Report-driven: only heals what actually needs it
|
||||
const report = await brain.repairIndex()
|
||||
console.log(report.healedTotal, report.families)
|
||||
|
||||
// Explicit: force the graph adjacency to rebuild from canonical, unconditionally
|
||||
await brain.repairIndex({ rebuild: ['graph'] })
|
||||
|
||||
// Explicit: force all three derived indexes to rebuild
|
||||
await brain.repairIndex({ rebuild: 'all' })
|
||||
```
|
||||
|
||||
**`RepairReport`:**
|
||||
- `families: RepairFamilyReport[]` — one row per family checked
|
||||
- `healedTotal: number` — items healed across every family
|
||||
- `durationMs: number`
|
||||
|
||||
**`RepairFamilyReport`** (one row):
|
||||
- `family: string` — e.g. `'orphaned-containers'`, `'count-rollups'`,
|
||||
`'vfs-containment'`, `'metadata-corruption'`, `'provider:metadata'`,
|
||||
`'provider:graph'`, `'provider:vector'`
|
||||
- `checked: boolean` — was this family actually examined (`false` ⇒ see `skipped`)
|
||||
- `healed: number` — items re-posted/corrected in place (the incremental heal count)
|
||||
- `missing?: { count: number; sample: string[] }` — exact count plus a capped id
|
||||
sample when the check can name what diverged (never the full list)
|
||||
- `rebuilt?: boolean` — a full generational rebuild ran (vs. an incremental heal)
|
||||
- `detail?: string` / `reason?: string` — narration
|
||||
- `skipped?: string` — why the family wasn't checked
|
||||
|
||||
Full walkthrough — what each family checks, degraded-but-serving vs. not-ready,
|
||||
and what `suspect` counts mean — in
|
||||
**[Index Health](../concepts/index-health.md)**.
|
||||
|
||||
---
|
||||
|
||||
### Index readiness: typed errors, `healthReport()`, `disableAutoRebuild`
|
||||
|
||||
Every derived-index provider (vector, graph, metadata) may expose a named,
|
||||
synchronous, O(1) `healthReport()` composed from its own exact ledgers — the
|
||||
signal Brainy's read gate trusts over sampling or size heuristics. `init()`
|
||||
brings every provider to serving before it returns; there is no first-query
|
||||
lazy-rebuild path. A read that reaches a provider whose health report says it
|
||||
isn't serving throws instead of rebuilding mid-query:
|
||||
|
||||
| Error | Thrown by | Meaning |
|
||||
|---|---|---|
|
||||
| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | Graph adjacency isn't serving |
|
||||
| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving |
|
||||
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving |
|
||||
|
||||
All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish
|
||||
"index not ready" from a genuine empty result:
|
||||
|
||||
```typescript
|
||||
import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy'
|
||||
|
||||
try {
|
||||
const rows = await brain.find({ where: { status: 'active' } })
|
||||
} catch (err) {
|
||||
if (err instanceof MetadataIndexNotReadyError) {
|
||||
// reconcile: await brain.repairIndex(), then retry
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**`disableAutoRebuild`** no longer defers index construction to the first
|
||||
query. A needed rebuild always runs at `open()`, regardless of this flag or
|
||||
dataset size; the flag has no effect on *when* a rebuild runs. Full manual
|
||||
control lives in `repairIndex({ rebuild: [...] })`, above.
|
||||
|
||||
### `validateIndexConsistency()` → `Promise<...>`
|
||||
|
||||
The deep, async diagnostic counterpart to `healthReport()` — safe to run on a
|
||||
live brain, but does more work (a provider's `validateInvariants()` may run a
|
||||
full scan, not just read a ledger). Aggregates the JS metadata index's own
|
||||
consistency check with every derived-index provider's invariant report.
|
||||
|
||||
```typescript
|
||||
const validation = await brain.validateIndexConsistency()
|
||||
if (!validation.healthy) {
|
||||
console.log(validation.recommendation) // what to run, e.g. repairIndex()
|
||||
console.log(validation.providers) // each provider's own invariant report, when exposed
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Lifecycle
|
||||
|
||||
### Initialization
|
||||
|
|
@ -2208,7 +2082,7 @@ For the full taxonomy with all 169 types and their descriptions, see:
|
|||
- **📖 Documentation:** [Full Documentation](../)
|
||||
- **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)
|
||||
- **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions)
|
||||
- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy)
|
||||
- **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy)
|
||||
- **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy)
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with <uuid>.ack
|
|||
| **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) |
|
||||
|
||||
A pluggable index provider (the 8.0 plugin contract in
|
||||
`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the
|
||||
`@soulcraft/brainy/plugin`) may replace any of the JS implementations; the
|
||||
persisted formats above are contract-bound so JS and native implementations
|
||||
can interleave on the same directory.
|
||||
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ class TypeAwareMetadataIndex {
|
|||
**The Design**: Specify types clearly in your API calls:
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
// Add entity with explicit type
|
||||
await brain.add({
|
||||
|
|
@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation {
|
|||
**Brainy's Approach**: Extract **typed** concepts:
|
||||
|
||||
```typescript
|
||||
import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy'
|
||||
import { NaturalLanguageProcessor } from '@soulcraft/brainy'
|
||||
|
||||
const nlp = new NaturalLanguageProcessor()
|
||||
const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco")
|
||||
|
|
@ -382,7 +382,7 @@ import {
|
|||
getVerbTypes,
|
||||
BrainyTypes,
|
||||
suggestType
|
||||
} from '@soulcraftlabs/brainy'
|
||||
} from '@soulcraft/brainy'
|
||||
|
||||
// Get all available noun types
|
||||
const nounTypes = getNounTypes()
|
||||
|
|
|
|||
|
|
@ -723,14 +723,6 @@ async stats(): Promise<Statistics> {
|
|||
|
||||
### 5. Index Rebuilding (Lazy Loading Support)
|
||||
|
||||
> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is
|
||||
> RETIRED.** `disableAutoRebuild` no longer defers index construction to a
|
||||
> first query; `brain.init()` now runs every needed rebuild to completion
|
||||
> before it returns, unconditionally, and a read against a not-serving
|
||||
> provider throws a typed `*NotReadyError` instead of rebuilding mid-query.
|
||||
> See `docs/concepts/index-health.md` for the current contract. Left below
|
||||
> as historical background on the rebuild mechanics.
|
||||
|
||||
**Two modes of index loading:**
|
||||
|
||||
#### Mode 1: Auto-Rebuild on init() (default)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,5 @@
|
|||
# Initialization and Rebuild Processes
|
||||
|
||||
> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is RETIRED.**
|
||||
> `disableAutoRebuild` no longer defers index construction to a first query;
|
||||
> `brain.init()` now runs every needed rebuild to completion before it
|
||||
> returns, unconditionally. A read against a not-serving provider throws a
|
||||
> typed `*NotReadyError` instead of rebuilding mid-query. See
|
||||
> `docs/concepts/index-health.md` for the current contract; this document's
|
||||
> line-number references to `src/brainy.ts` also predate the file's current
|
||||
> size and are unreliable. Left as historical background on the rebuild
|
||||
> mechanics, not as a current API description.
|
||||
|
||||
This document explains how Brainy's four indexes (MetadataIndex, vector index, GraphAdjacencyIndex, DeletedItemsIndex) initialize and rebuild from persisted storage.
|
||||
|
||||
## Core Principle: All Indexes Are Disk-Based
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ For reference, a clean migration path:
|
|||
`isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for
|
||||
build/install artifact protection.
|
||||
5. Document the new contract in `concepts/storage-adapters.md`.
|
||||
6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by
|
||||
6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by
|
||||
plugins.
|
||||
|
||||
Estimated work: ~half a day of code, ~2 hours of doc/example updates,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ next:
|
|||
Every example on this page is written against the real Brainy 8.0 API. The setup is always the same:
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu
|
|||
- **Multi-hop Graph Traversals = Relationship Complexity**
|
||||
- **Result: Model data across virtually any industry**
|
||||
|
||||
Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name.
|
||||
Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name.
|
||||
|
||||
## The Power of Standardization: Universal Interoperability
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ constructor and `init()`.
|
|||
## Instant Start
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// That's it. No config needed.
|
||||
const brain = new Brainy()
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ await brain.find({ orderBy: 'createdAt' })
|
|||
`UnresolvableFieldError` is exported from the package root:
|
||||
|
||||
```typescript
|
||||
import { UnresolvableFieldError } from '@soulcraftlabs/brainy'
|
||||
import { UnresolvableFieldError } from '@soulcraft/brainy'
|
||||
|
||||
try {
|
||||
await brain.find({ orderBy: 'createdAt' })
|
||||
|
|
|
|||
|
|
@ -1,217 +0,0 @@
|
|||
---
|
||||
title: Index Health
|
||||
slug: concepts/index-health
|
||||
public: true
|
||||
category: concepts
|
||||
template: concept
|
||||
order: 8
|
||||
description: How Brainy knows whether a derived index can be trusted — exact accounting instead of sampling, the named health report, degraded-but-serving vs. not-ready, and what repairIndex() checks, heals, and rebuilds.
|
||||
next:
|
||||
- concepts/generation-fact-log
|
||||
- guides/inspection
|
||||
---
|
||||
|
||||
# Index Health
|
||||
|
||||
Brainy keeps one **canonical** copy of every entity and relationship, and three
|
||||
**derived** indexes built from it — vector, metadata, and graph — so `find()` can
|
||||
answer semantically, by filter, and by traversal without re-deriving the answer from
|
||||
scratch on every query. A derived index is a cache with a serving structure: it can
|
||||
be present but stale, present but only partially loaded, or fully out of sync with
|
||||
canonical after a crash. This page is about how Brainy decides whether to trust one,
|
||||
what it does when it can't, and how you reconcile the two.
|
||||
|
||||
## Exact accounting instead of sampling
|
||||
|
||||
Older health checks worked by inference: does `size()` return something greater
|
||||
than zero, does a spot-check on one known item come back correct. Both are proxies.
|
||||
A cold index can report a nonzero count while its actual serving structure never
|
||||
loaded, and a spot-check only proves the one item it happened to ask about.
|
||||
|
||||
Every derived-index provider may now expose a named, synchronous, O(1)
|
||||
`healthReport()` — composed from the provider's own **exact ledgers** (real counters
|
||||
it already maintains on the write path), never a sample or a walk. This is the one
|
||||
signal Brainy's read gate consults. A provider that doesn't yet expose one falls
|
||||
back to an honest `isReady()` boolean, and finally to a size heuristic for engines
|
||||
with neither — but wherever a `healthReport()` exists, it wins.
|
||||
|
||||
Underneath, storage itself keeps an analogous **canonical count ledger**: a
|
||||
`counted` scalar (the user-facing total — what `getNounCount()` / `getVerbCount()`
|
||||
return) and an `all` scalar (every tier, including internal records a derived
|
||||
index's own coverage math needs to compare against). This is the real denominator
|
||||
a provider's `healthReport()` measures itself by, rather than a total that can only
|
||||
ever ratchet upward. See [What `suspect` counts mean](#what-suspect-counts-mean)
|
||||
below for the one case that ledger can't stay exact through on its own.
|
||||
|
||||
## The named report
|
||||
|
||||
A `HealthReport` carries, per provider (`'vector'` / `'graph'` / `'metadata'`):
|
||||
|
||||
- **`healthy`** — `true` iff every *verified* invariant holds. An invariant whose
|
||||
family has no ledger yet is `unledgered`, never counted either way — unknown,
|
||||
not passing.
|
||||
- **`serving`** — can this provider answer a query right now. A failing invariant
|
||||
graded `heal: 'repair'` or `heal: 'none'` still leaves `serving: true` — this is
|
||||
**degraded-but-serving**: something is off (say, a stale rollup on an
|
||||
`employee` record's relationship count) but reads keep working. Only a failure
|
||||
graded `heal: 'rebuild'` flips `serving` to `false` — **not-ready** — because the
|
||||
provider itself is telling you its serving structure cannot answer correctly.
|
||||
- **`invariants`** — each checked condition, with its provenance
|
||||
(`source: 'ledger'` — an exact count; `'deep'` — a full scan, diagnostic-only;
|
||||
`'unledgered'` — not yet tracked) and, for a failing one, an exact `missing`
|
||||
count plus a capped sample of the affected ids — a verdict, never a dump.
|
||||
- **`generation`** — bumps on every ledger mutation and rebuild, so a caller can
|
||||
cache a verdict per generation instead of re-deriving it.
|
||||
|
||||
The distinction that matters day to day: `healthy: false` can be entirely benign —
|
||||
a maintenance window, a divergence `repairIndex()` will clean up on its own
|
||||
schedule. `serving: false` is not benign. It means this provider is refusing to
|
||||
answer, on its own word, right now.
|
||||
|
||||
**How a failure gets its grade — the serving law.** A provider grades `heal` by
|
||||
one question only: *could an answer be wrong?* — never *how expensive is the
|
||||
fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post
|
||||
exactly what the ledger names, reads serving throughout); it can never withhold
|
||||
serving just because healing it takes work. `serving` is withheld only by a
|
||||
small, named set of rebuild-graded conditions — the index not initialized, its
|
||||
durable state absent, a manifest naming files that are not resident, a replay
|
||||
that did not complete cleanly — the states in which an answer could genuinely be
|
||||
wrong. And a read is only ever refused by the family it actually consults: a
|
||||
metadata filter is answered by the metadata index alone, vector search by the
|
||||
vector index, traversal by the graph index — one family's refusal never blocks
|
||||
another family's reads.
|
||||
|
||||
## Reads refuse — they never rebuild
|
||||
|
||||
A query that reaches a not-serving provider does not trigger a rebuild from inside
|
||||
the read. Brainy retired that path deliberately: a rebuild kicked off by an ordinary
|
||||
`find({ where: { status: 'active' } })` call is a dark, unpredictable cost hiding
|
||||
behind a request that looks like a cheap read. Instead, the read throws a typed,
|
||||
catchable error naming the reason:
|
||||
|
||||
| Error | Thrown when | Meaning |
|
||||
|---|---|---|
|
||||
| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | The graph adjacency index isn't serving — traversal would otherwise return `[]` indistinguishable from "no relationships" |
|
||||
| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" |
|
||||
| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" |
|
||||
|
||||
All three are exported from `@soulcraftlabs/brainy`. Catch them where your application
|
||||
needs to distinguish "this index isn't ready yet" from "there's genuinely nothing
|
||||
here" — a health dashboard, a retry policy, an operator alert. The fix is always
|
||||
the same: reconcile the index, either by reopening the brain (which brings every
|
||||
provider to serving before `init()` returns — see the next section) or by calling
|
||||
`repairIndex()` explicitly.
|
||||
|
||||
```typescript
|
||||
try {
|
||||
const active = await brain.find({ where: { status: 'active' } })
|
||||
} catch (err) {
|
||||
if (err instanceof MetadataIndexNotReadyError) {
|
||||
// not a "no results" — the index itself refused; alert or retry after repair
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Rebuilds happen at open, not on first query
|
||||
|
||||
`brain.init()` runs every needed rebuild to completion **before it returns**,
|
||||
unconditionally, regardless of dataset size. There is no lazy, first-query
|
||||
rebuild path anymore — a brain either finishes opening healthy, or it fails
|
||||
open loudly. `disableAutoRebuild: true` no longer defers index construction to
|
||||
the first query: it has no effect on *when* a needed rebuild runs. Full manual
|
||||
control over rebuilds is `repairIndex({ rebuild: [...] })` (below), not this flag.
|
||||
|
||||
## `repairIndex()` — checking and healing
|
||||
|
||||
Bare `repairIndex()` is **report-driven**: it only heals what its own checks say
|
||||
actually needs it, and it always returns a full per-family receipt.
|
||||
|
||||
```typescript
|
||||
const report = await brain.repairIndex()
|
||||
report.healedTotal // total items healed across every family
|
||||
report.durationMs
|
||||
report.families // one row per family checked
|
||||
```
|
||||
|
||||
Each `RepairFamilyReport` row names what happened:
|
||||
|
||||
- **`checked`** — was this family actually examined (`false` means skipped —
|
||||
see `skipped` for why).
|
||||
- **`healed`** — items re-posted or corrected in place.
|
||||
- **`missing`** — when the check can name what diverged: an exact `count` plus a
|
||||
capped `sample` of ids.
|
||||
- **`rebuilt`** — a full generational rebuild ran (as opposed to an incremental
|
||||
heal).
|
||||
- **`detail`** / **`reason`** / **`skipped`** — the receipt's narration; a row is
|
||||
always either checked or explains why it wasn't. Nothing is silent.
|
||||
|
||||
On every call, bare `repairIndex()`:
|
||||
|
||||
1. Prunes orphaned canonical containers left by a partial delete.
|
||||
2. Recomputes the count rollups from one canonical walk (unconditional — this is
|
||||
also what clears a `suspect` ledger; see below).
|
||||
3. Reconciles VFS containment edges, if the VFS is initialized.
|
||||
4. Runs the metadata index's own corruption detection pass.
|
||||
5. Consults each of the three derived-index providers' own health check and
|
||||
rebuilds only a family whose failing invariant actually asks for it
|
||||
(`heal: 'rebuild'`) — never a provider that reports `healthy` or a lesser
|
||||
grade.
|
||||
|
||||
### The explicit rebuild door
|
||||
|
||||
`options.rebuild` skips the health check and rebuilds one or more families
|
||||
**unconditionally** — the operator override for when you have independent reason
|
||||
to distrust a family regardless of what it self-reports (a suspicious deploy, a
|
||||
storage-layer incident, a support ticket that doesn't match what the health report
|
||||
says):
|
||||
|
||||
```typescript
|
||||
// Force the graph adjacency to rebuild from canonical, no invariant consulted
|
||||
await brain.repairIndex({ rebuild: ['graph'] })
|
||||
|
||||
// Force all three derived indexes
|
||||
await brain.repairIndex({ rebuild: 'all' })
|
||||
```
|
||||
|
||||
A family named this way is recorded with `rebuilt: true` and
|
||||
`reason: 'explicit rebuild requested'`, and is skipped by the normal
|
||||
health-driven pass in the same call — it was already rebuilt unconditionally.
|
||||
|
||||
Reach for the explicit door when you need certainty regardless of self-report;
|
||||
reach for bare `repairIndex()` for routine maintenance and after any incident
|
||||
where you're not sure which family (if any) needs it.
|
||||
|
||||
## What `suspect` counts mean
|
||||
|
||||
Storage's canonical count ledger increments the ALL-visibility total on every new
|
||||
record and decrements it on every *proven* delete — one where the record was read,
|
||||
or the caller supplied its prior image. A delete that cannot prove what it removed
|
||||
existed doesn't guess: it flags the ledger `suspect` (an operator-visible
|
||||
`console.warn`, narrated once per session, not once per delete) rather than risk
|
||||
decrementing a total that was never incremented for that record in the first
|
||||
place. This is intentionally rare — it's a defensive fallback for callers on an
|
||||
unusual removal path, not a per-delete cost.
|
||||
|
||||
`suspect` is not directly exposed on any `Brainy` method today — it lives on the
|
||||
`StorageAdapter`'s optional `getCanonicalCounts()`, primarily consulted by
|
||||
`repairIndex()`'s recount step and by custom storage adapters composing their own
|
||||
`healthReport()`. What matters for an application: a `suspect` ledger is not
|
||||
incorrect, just *unverified since the last recount* — and `repairIndex()`'s
|
||||
unconditional count-rollup step (step 2, above) recomputes the ALL scalars from a
|
||||
real canonical walk on every call, clearing the flag with proof either way.
|
||||
|
||||
## Practical guidance
|
||||
|
||||
- **On a normal restart**, do nothing — `init()` brings every provider to
|
||||
serving before it returns, or fails loudly.
|
||||
- **On a `*NotReadyError`** from a live read, reconcile with `repairIndex()`
|
||||
(report-driven is almost always sufficient) and retry.
|
||||
- **After an incident** where you distrust a specific family regardless of what
|
||||
it reports healthy — a storage-layer fault, a suspicious restore — use the
|
||||
explicit door: `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] })`.
|
||||
- **To audit before trusting a report**, `brain.auditGraph()` walks every stored
|
||||
relationship and proves (or disproves) that reads return canonical truth,
|
||||
independent of what any provider self-reports — see
|
||||
[Inspecting a Live Brainy](../guides/inspection.md).
|
||||
|
|
@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from
|
|||
to call `acquireWriterLock()` at init.
|
||||
|
||||
```typescript
|
||||
import { FileSystemStorage } from '@soulcraftlabs/brainy'
|
||||
import { FileSystemStorage } from '@soulcraft/brainy'
|
||||
|
||||
export class MmapFileSystemStorage extends FileSystemStorage {
|
||||
public supportsMultiProcessLocking(): boolean {
|
||||
|
|
@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom
|
|||
network backend), extend `BaseStorage` directly:
|
||||
|
||||
```typescript
|
||||
import { BaseStorage } from '@soulcraftlabs/brainy'
|
||||
import { BaseStorage } from '@soulcraft/brainy'
|
||||
|
||||
export class MyCloudStorage extends BaseStorage {
|
||||
// BaseStorage's default no-op implementations of the multi-process
|
||||
|
|
@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`,
|
|||
`hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a
|
||||
stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM
|
||||
import (verify in your plugin's `dist/`: `import { FileSystemStorage } from
|
||||
'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype
|
||||
'@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype
|
||||
chain at runtime resolves to whatever Brainy version your consumer has
|
||||
installed.
|
||||
|
||||
|
|
@ -109,8 +109,8 @@ installed.
|
|||
the prototype chain at the consumer-app level:
|
||||
|
||||
- **Stale `node_modules`** — a lingering install from before the consumer
|
||||
upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but
|
||||
`node_modules/@soulcraftlabs/brainy` is still 7.20.x.
|
||||
upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but
|
||||
`node_modules/@soulcraft/brainy` is still 7.20.x.
|
||||
- **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy
|
||||
version older than the package.json range, and `bun install` honors the
|
||||
lockfile.
|
||||
|
|
@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint:
|
|||
methods on its prototype chain. Writer locking and the flush-request RPC are
|
||||
disabled for this directory. Likely fix: clean install (`rm -rf node_modules
|
||||
bun.lockb && bun install`) or rebuild your container image to refresh
|
||||
`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md.
|
||||
`@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md.
|
||||
```
|
||||
|
||||
## Authoring a new storage adapter — minimum checklist
|
||||
|
|
@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh
|
|||
install time — fix install, not your plugin.
|
||||
|
||||
6. **Pin your peer dep generously.** `"peerDependencies": {
|
||||
"@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin
|
||||
"@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin
|
||||
to an exact patch unless you're tracking a known regression.
|
||||
|
||||
## Future direction
|
||||
|
|
@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change.
|
|||
heartbeat semantics, what the lock protects.
|
||||
- [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the
|
||||
read-only mode.
|
||||
- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the
|
||||
- `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the
|
||||
authoritative type signatures for every method this page references.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ they share a single scan.
|
|||
## Quick Start
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any
|
|||
|
||||
Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server:
|
||||
|
||||
- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'`
|
||||
- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'`
|
||||
- **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node
|
||||
- **Cleaner code**: No browser polyfills, no conditional client/server imports
|
||||
- **Better DX**: One instance shared across your server routes
|
||||
|
|
@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers
|
|||
### Install Brainy
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
### Basic Integration
|
||||
|
||||
```javascript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// Run on the server (API route, server component, backend service)
|
||||
// new Brainy() auto-detects filesystem persistence on Node
|
||||
|
|
@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod
|
|||
|
||||
```javascript
|
||||
// lib/brain.server.js
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brainPromise
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests:
|
|||
|
||||
```javascript
|
||||
// server/brain.js (server-only module)
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brainPromise
|
||||
|
||||
|
|
@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun):
|
|||
|
||||
```typescript
|
||||
// server: api/search
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy() // auto-detects filesystem persistence on Node
|
||||
await brain.init()
|
||||
|
|
@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or
|
|||
|
||||
```javascript
|
||||
// lib/brain.server.js (imported only by server code)
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brainPromise
|
||||
|
||||
|
|
@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul
|
|||
|
||||
```javascript
|
||||
// src/lib/server/brain.js (server-only — note the .server suffix)
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brainPromise
|
||||
|
||||
|
|
@ -432,7 +432,7 @@ import { defineConfig } from 'vite'
|
|||
|
||||
export default defineConfig({
|
||||
ssr: {
|
||||
external: ['@soulcraftlabs/brainy']
|
||||
external: ['@soulcraft/brainy']
|
||||
}
|
||||
})
|
||||
```
|
||||
|
|
@ -440,7 +440,7 @@ export default defineConfig({
|
|||
```javascript
|
||||
// rollup.config.js (server bundle)
|
||||
export default {
|
||||
external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto']
|
||||
external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto']
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -466,7 +466,7 @@ export async function load({ url }) {
|
|||
|
||||
```javascript
|
||||
// For build-time usage (runs in Node during the build)
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
export async function generateStaticProps() {
|
||||
const brain = new Brainy({
|
||||
|
|
@ -513,7 +513,7 @@ export async function generateStaticProps() {
|
|||
|
||||
### Issue: Large client bundle size
|
||||
**Cause**: A client module is pulling in Brainy.
|
||||
**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle.
|
||||
**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle.
|
||||
|
||||
### Issue: SSR hydration mismatch
|
||||
**Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup.
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING:
|
|||
## The Ultimate Simplicity
|
||||
|
||||
```javascript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats**
|
|||
### Basic Progress Tracking
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import * as fs from 'fs'
|
||||
|
||||
const brain = await Brainy.create()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
## Basic Import
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -187,7 +187,7 @@ await brain.import(file, {
|
|||
## Complete Example
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import * as fs from 'fs'
|
||||
|
||||
async function importCatalog() {
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI.
|
|||
## Programmatic inspection
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const reader = await Brainy.openReadOnly({
|
||||
storage: { type: 'filesystem', path: '/data/brain' }
|
||||
|
|
|
|||
|
|
@ -21,21 +21,21 @@ next:
|
|||
## Install
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
Or with your preferred package manager:
|
||||
|
||||
```bash
|
||||
bun add @soulcraftlabs/brainy
|
||||
yarn add @soulcraftlabs/brainy
|
||||
pnpm add @soulcraftlabs/brainy
|
||||
bun add @soulcraft/brainy
|
||||
yarn add @soulcraft/brainy
|
||||
pnpm add @soulcraft/brainy
|
||||
```
|
||||
|
||||
## Verify
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -52,7 +52,7 @@ npm install @soulcraft/cor
|
|||
```
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ plugins: ['@soulcraft/cor'] })
|
||||
await brain.init() // native providers registered during init
|
||||
|
|
@ -71,7 +71,7 @@ remains available on npm if you need it.
|
|||
Brainy ships with full TypeScript types. No `@types/` package needed:
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ const results = await brain.search("query")
|
|||
**New diagnostics for capacity planning and performance tuning.**
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')}
|
|||
### Step 1: Update Package
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy@latest
|
||||
npm install @soulcraft/brainy@latest
|
||||
```
|
||||
|
||||
### Step 2: Restart Your Application
|
||||
|
|
@ -134,7 +134,7 @@ npm run start
|
|||
### Check Adaptive Sizing is Working
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -218,7 +218,7 @@ For debugging or compatibility testing:
|
|||
If you need to rollback to v3.35.0:
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy@3.35.0
|
||||
npm install @soulcraft/brainy@3.35.0
|
||||
```
|
||||
|
||||
**Note:** We don't anticipate any issues, but rollback is straightforward if needed.
|
||||
|
|
@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) {
|
|||
|
||||
## Next Steps
|
||||
|
||||
1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest`
|
||||
1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest`
|
||||
2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements
|
||||
3. 🎯 **Tune:** Adjust based on recommendations (if needed)
|
||||
4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings.
|
|||
|
||||
```bash
|
||||
# Bun as a runtime — supported and recommended
|
||||
bun add @soulcraftlabs/brainy
|
||||
bun add @soulcraft/brainy
|
||||
bun run server.ts
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ If you read raw stored records (fact-log scanners, export tooling), use
|
|||
the exported shape-aware splitters — they handle both record eras:
|
||||
|
||||
```typescript
|
||||
import { splitNounMetadataRecord } from '@soulcraftlabs/brainy'
|
||||
import { splitNounMetadataRecord } from '@soulcraft/brainy'
|
||||
const { reserved, custom } = splitNounMetadataRecord(rawRecord)
|
||||
// reserved = engine fields · custom = the user's bag, ANY names
|
||||
```
|
||||
|
|
@ -88,7 +88,7 @@ const { reserved, custom } = splitNounMetadataRecord(rawRecord)
|
|||
Feature detection (never version-sniff):
|
||||
|
||||
```typescript
|
||||
import * as brainy from '@soulcraftlabs/brainy'
|
||||
import * as brainy from '@soulcraft/brainy'
|
||||
const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1'
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou
|
|||
```bash
|
||||
npx create-next-app@latest my-brainy-app
|
||||
cd my-brainy-app
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
### Basic Setup
|
||||
|
|
@ -18,7 +18,7 @@ npm install @soulcraftlabs/brainy
|
|||
// app/components/BrainyProvider.jsx
|
||||
'use client'
|
||||
import { createContext, useContext, useEffect, useState } from 'react'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const BrainyContext = createContext()
|
||||
|
||||
|
|
@ -271,7 +271,7 @@ export default function SearchPage() {
|
|||
|
||||
```javascript
|
||||
// app/api/search/route.js (App Router)
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brain = null
|
||||
|
||||
|
|
@ -332,7 +332,7 @@ export async function GET() {
|
|||
|
||||
```javascript
|
||||
// pages/api/search.js (Pages Router)
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brain = null
|
||||
|
||||
|
|
@ -374,7 +374,7 @@ export default async function handler(req, res) {
|
|||
|
||||
```javascript
|
||||
// app/api/data/route.js
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brain = null
|
||||
|
||||
|
|
@ -418,7 +418,7 @@ export async function POST(request) {
|
|||
```jsx
|
||||
// app/actions/brainy.js
|
||||
'use server'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brain = null
|
||||
|
||||
|
|
@ -630,7 +630,7 @@ CMD ["npm", "start"]
|
|||
/** @type {import('next').NextConfig} */
|
||||
const nextConfig = {
|
||||
experimental: {
|
||||
serverComponentsExternalPackages: ['@soulcraftlabs/brainy']
|
||||
serverComponentsExternalPackages: ['@soulcraft/brainy']
|
||||
},
|
||||
webpack: (config, { isServer }) => {
|
||||
if (!isServer) {
|
||||
|
|
@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) {
|
|||
// app/contexts/BrainyContext.jsx
|
||||
'use client'
|
||||
import { createContext, useContext, useReducer, useEffect } from 'react'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const BrainyContext = createContext()
|
||||
|
||||
|
|
@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider'
|
|||
import { Search } from '../app/components/Search'
|
||||
|
||||
// Mock Brainy
|
||||
jest.mock('@soulcraftlabs/brainy', () => ({
|
||||
jest.mock('@soulcraft/brainy', () => ({
|
||||
Brainy: jest.fn().mockImplementation(() => ({
|
||||
init: jest.fn().mockResolvedValue(undefined),
|
||||
find: jest.fn().mockResolvedValue([
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin
|
|||
Every distributed-job scheduler eventually wants this exact loop:
|
||||
|
||||
```ts
|
||||
import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, RevisionConflictError } from '@soulcraft/brainy'
|
||||
|
||||
const LOCK_ID = '...uuid for this job slot...'
|
||||
|
||||
|
|
@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API
|
|||
It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved:
|
||||
|
||||
```ts
|
||||
import { GenerationConflictError } from '@soulcraftlabs/brainy'
|
||||
import { GenerationConflictError } from '@soulcraft/brainy'
|
||||
|
||||
async function addIfMissingByEmail(email: string, data: string) {
|
||||
for (let attempt = 0; attempt < 5; attempt++) {
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ Get Brainy running in under a minute.
|
|||
## 1. Install
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
## 2. Initialize
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType, VerbType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -67,7 +67,7 @@ await brain.relate({
|
|||
## 5. Query with Triple Intelligence
|
||||
|
||||
```typescript
|
||||
import type { Result } from '@soulcraftlabs/brainy'
|
||||
import type { Result } from '@soulcraft/brainy'
|
||||
|
||||
// All three search paradigms in one call
|
||||
const results: Result[] = await brain.find({
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
### One Interface for Everything
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = await Brainy.create()
|
||||
|
||||
|
|
@ -78,7 +78,7 @@ interface ImportProgress {
|
|||
|
||||
```typescript
|
||||
import { useState } from 'react'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
function UniversalImportProgress({ file }: { file: File }) {
|
||||
const [progress, setProgress] = useState({
|
||||
|
|
@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) {
|
|||
|
||||
```typescript
|
||||
import ora from 'ora'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
async function importWithProgress(filePath: string) {
|
||||
const spinner = ora('Starting import...').start()
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map).
|
|||
## Quick start
|
||||
|
||||
```ts
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// Filesystem (recommended for any persistent workload):
|
||||
const brain = new Brainy({
|
||||
|
|
@ -134,7 +134,7 @@ config; the `type` is optional.
|
|||
If you want to skip the factory:
|
||||
|
||||
```ts
|
||||
import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy'
|
||||
import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy'
|
||||
|
||||
const fsStorage = new FileSystemStorage('./brainy-data')
|
||||
const memStorage = new MemoryStorage()
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ Three layers solve this:
|
|||
### Write
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -240,7 +240,7 @@ await brain.migrateField({
|
|||
A realistic adoption sequence for a brain that started without these primitives:
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } })
|
||||
await brain.init()
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you.
|
|||
|
||||
## TL;DR
|
||||
|
||||
- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.**
|
||||
- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.**
|
||||
If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**,
|
||||
with no operator action.
|
||||
- Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**.
|
||||
|
|
@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and
|
|||
open it.**
|
||||
|
||||
```ts
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// Opening the store is all that is required — recovery runs during init().
|
||||
const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } })
|
||||
|
|
@ -182,5 +182,5 @@ and opening each store is sufficient.
|
|||
The recovery is copy-only, so no rollback of the recovery itself is ever needed.
|
||||
If you need to roll back the **whole** 7→8 upgrade, restore the directory from
|
||||
your pre-upgrade backup (retained automatically while recovery is incomplete, or
|
||||
your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old
|
||||
your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old
|
||||
branch layout in place, so a directory-level restore is the rollback path.
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N
|
|||
npm create vue@latest my-brainy-app
|
||||
cd my-brainy-app
|
||||
npm install
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
### Basic Setup
|
||||
|
|
@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun
|
|||
|
||||
```javascript
|
||||
// server/utils/brain.js (server-only — Nitro never bundles this into the client)
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
let brainPromise
|
||||
|
||||
|
|
@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue'
|
|||
export default defineConfig({
|
||||
plugins: [vue()],
|
||||
ssr: {
|
||||
external: ['@soulcraftlabs/brainy']
|
||||
external: ['@soulcraft/brainy']
|
||||
}
|
||||
})
|
||||
```
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c
|
|||
### Method 1: Brain Instance (Recommended)
|
||||
|
||||
```typescript
|
||||
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', {
|
|||
import {
|
||||
SmartExtractor,
|
||||
SmartRelationshipExtractor
|
||||
} from '@soulcraftlabs/brainy'
|
||||
} from '@soulcraft/brainy'
|
||||
// Or use subpath imports:
|
||||
import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor'
|
||||
import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, {
|
|||
**Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration.
|
||||
|
||||
```typescript
|
||||
import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy'
|
||||
import { SmartExtractor, FormatContext } from '@soulcraft/brainy'
|
||||
|
||||
const extractor = new SmartExtractor(brain, {
|
||||
minConfidence: 0.7, // Threshold
|
||||
|
|
@ -229,7 +229,7 @@ interface ExtractionResult {
|
|||
**Relationship type classifier.** Determines verb/relationship types between entities.
|
||||
|
||||
```typescript
|
||||
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy'
|
||||
import { SmartRelationshipExtractor } from '@soulcraft/brainy'
|
||||
|
||||
const relExtractor = new SmartRelationshipExtractor(brain, {
|
||||
minConfidence: 0.6,
|
||||
|
|
@ -286,7 +286,7 @@ const rel = await relExtractor.infer(
|
|||
**Full extraction orchestrator.** Handles candidate detection, classification, and deduplication.
|
||||
|
||||
```typescript
|
||||
import { NeuralEntityExtractor } from '@soulcraftlabs/brainy'
|
||||
import { NeuralEntityExtractor } from '@soulcraft/brainy'
|
||||
|
||||
const extractor = new NeuralEntityExtractor(brain)
|
||||
|
||||
|
|
@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location)
|
|||
### Example 2: Excel Data Classification
|
||||
|
||||
```typescript
|
||||
import { SmartExtractor } from '@soulcraftlabs/brainy'
|
||||
import { SmartExtractor } from '@soulcraft/brainy'
|
||||
|
||||
const extractor = new SmartExtractor(brain)
|
||||
|
||||
|
|
@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) {
|
|||
### Example 3: Relationship Extraction
|
||||
|
||||
```typescript
|
||||
import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy'
|
||||
import { SmartRelationshipExtractor } from '@soulcraft/brainy'
|
||||
|
||||
const relExtractor = new SmartRelationshipExtractor(brain)
|
||||
|
||||
|
|
|
|||
|
|
@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing })
|
|||
### Basic Add Operation
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { NounType } from '@soulcraftlabs/brainy/types'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import { NounType } from '@soulcraft/brainy/types'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked
|
|||
|
||||
```typescript
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
describe('Transaction Tests', () => {
|
||||
it('should rollback on failure', async () => {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati
|
|||
### Basic Usage
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brainy = new Brainy()
|
||||
await brainy.init()
|
||||
|
|
|
|||
|
|
@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low):
|
|||
### Step 1: Create the Strategy Class
|
||||
|
||||
```typescript
|
||||
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs'
|
||||
import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs'
|
||||
|
||||
export class PriorityProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'priority'
|
||||
|
|
@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy {
|
|||
### Step 2: Register the Strategy
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import { PriorityProjection } from './PriorityProjection'
|
||||
|
||||
const brain = new Brainy()
|
||||
|
|
@ -537,7 +537,7 @@ Use the projection's resolve cache:
|
|||
|
||||
```typescript
|
||||
import { describe, it, expect, beforeAll } from 'vitest'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import { PriorityProjection } from './PriorityProjection'
|
||||
|
||||
describe('PriorityProjection', () => {
|
||||
|
|
@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) {
|
|||
3. Use appropriate limits: Don't fetch more than needed
|
||||
|
||||
### Type errors
|
||||
1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'`
|
||||
1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'`
|
||||
2. Use `as VFSEntity` when mapping results
|
||||
3. Check BaseProjectionStrategy import
|
||||
|
||||
|
|
|
|||
|
|
@ -14,11 +14,11 @@ A file explorer that:
|
|||
## ⚡ Step 1: Basic Setup (1 minute)
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// ✅ CORRECT: Use filesystem storage for production
|
||||
const brain = new Brainy({
|
||||
|
|
@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns:
|
|||
|
||||
```tsx
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
export function FileExplorer() {
|
||||
const [brain, setBrain] = useState(null)
|
||||
|
|
@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next:
|
|||
### "Module not found" errors
|
||||
```bash
|
||||
# Make sure you're using the right import
|
||||
npm ls @soulcraftlabs/brainy # Check version
|
||||
npm install @soulcraftlabs/brainy@latest # Update if needed
|
||||
npm ls @soulcraft/brainy # Check version
|
||||
npm install @soulcraft/brainy@latest # Update if needed
|
||||
```
|
||||
|
||||
### "VFS not initialized" errors
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne
|
|||
## Quick Start
|
||||
|
||||
```javascript
|
||||
import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs'
|
||||
import { VirtualFileSystem } from '@soulcraft/brainy/vfs'
|
||||
|
||||
// Initialize the VFS
|
||||
const vfs = new VirtualFileSystem({
|
||||
|
|
@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system:
|
|||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @soulcraftlabs/brainy
|
||||
npm install @soulcraft/brainy
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows.
|
|||
|
||||
```typescript
|
||||
// Planned (research phase)
|
||||
import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse'
|
||||
import { mountVFS } from '@soulcraft/brainy/vfs/fuse'
|
||||
|
||||
await mountVFS(vfs, {
|
||||
mountPoint: '/mnt/brainy',
|
||||
|
|
@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested
|
|||
### Express.js Static Middleware
|
||||
```typescript
|
||||
// Wanted: Community contribution
|
||||
import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express'
|
||||
import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express'
|
||||
|
||||
app.use('/files', createStaticMiddleware(vfs, {
|
||||
index: ['index.html', 'index.md'],
|
||||
|
|
@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, {
|
|||
### VSCode Extension
|
||||
```typescript
|
||||
// Wanted: Community contribution
|
||||
import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode'
|
||||
import { VFSProvider } from '@soulcraft/brainy/vfs/vscode'
|
||||
|
||||
const provider = new VFSProvider(vfs)
|
||||
vscode.workspace.registerFileSystemProvider('brainy', provider)
|
||||
|
|
|
|||
|
|
@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true
|
|||
Create your own semantic dimensions:
|
||||
|
||||
```typescript
|
||||
import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic'
|
||||
import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic'
|
||||
|
||||
class PriorityProjection extends BaseProjectionStrategy {
|
||||
readonly name = 'priority'
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha
|
|||
## Quick Start
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// Initialize Brainy
|
||||
const brain = new Brainy({
|
||||
|
|
@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123')
|
|||
VFS uses standard POSIX-style errors:
|
||||
|
||||
```typescript
|
||||
import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy'
|
||||
import { VFSError, VFSErrorCode } from '@soulcraft/brainy'
|
||||
|
||||
try {
|
||||
await vfs.readFile('/nonexistent.txt')
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities:
|
|||
#### GitBridge Usage
|
||||
```javascript
|
||||
// Import and instantiate GitBridge
|
||||
import { GitBridge } from '@soulcraftlabs/brainy'
|
||||
import { GitBridge } from '@soulcraft/brainy'
|
||||
const gitBridge = new GitBridge(vfs, brain)
|
||||
|
||||
// Export VFS to Git repository structure
|
||||
|
|
@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare
|
|||
## Complete Example
|
||||
|
||||
```javascript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
async function vfsExample() {
|
||||
// Initialize
|
||||
|
|
|
|||
|
|
@ -196,5 +196,5 @@ await brain.relate({
|
|||
Always import and use the type enums:
|
||||
|
||||
```javascript
|
||||
import { NounType, VerbType } from '@soulcraftlabs/brainy'
|
||||
import { NounType, VerbType } from '@soulcraft/brainy'
|
||||
```
|
||||
|
|
@ -5,7 +5,7 @@
|
|||
The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed!
|
||||
|
||||
```javascript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
// Create and initialize Brainy
|
||||
const brain = new Brainy({
|
||||
|
|
@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e
|
|||
## Complete Example
|
||||
|
||||
```javascript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
async function useVFS() {
|
||||
// Initialize Brainy
|
||||
|
|
@ -100,7 +100,7 @@ useVFS().catch(console.error)
|
|||
## TypeScript Usage
|
||||
|
||||
```typescript
|
||||
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
|
||||
|
||||
class FileManager {
|
||||
private brain: Brainy
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues:
|
|||
### Method 1: Use `getDirectChildren()` (Recommended)
|
||||
|
||||
```typescript
|
||||
import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy()
|
||||
await brain.init()
|
||||
|
|
@ -97,7 +97,7 @@ Here's a complete example using React:
|
|||
|
||||
```tsx
|
||||
import React, { useState, useEffect } from 'react'
|
||||
import { VirtualFileSystem } from '@soulcraftlabs/brainy'
|
||||
import { VirtualFileSystem } from '@soulcraft/brainy'
|
||||
|
||||
interface FileNode {
|
||||
name: string
|
||||
|
|
@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) {
|
|||
If you must build trees manually from flat lists, use the `VFSTreeUtils`:
|
||||
|
||||
```typescript
|
||||
import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs'
|
||||
import { VFSTreeUtils } from '@soulcraft/brainy/vfs'
|
||||
|
||||
// Get all entities somehow
|
||||
const allEntities = await vfs.getDescendants('/root')
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
* the Bluesky firehose with Brainy's distributed architecture
|
||||
*/
|
||||
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
import { WebSocket } from 'ws'
|
||||
|
||||
// =====================================================
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
* ts-node examples/monitor-cache-performance.ts
|
||||
*/
|
||||
|
||||
import { Brainy, NounType } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, NounType } from '@soulcraft/brainy'
|
||||
|
||||
// ANSI color codes for pretty output
|
||||
const colors = {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu
|
|||
## Quick Start
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ integrations: true })
|
||||
await brain.init()
|
||||
|
|
@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature.
|
|||
### Minimal (in-memory):
|
||||
|
||||
```typescript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ integrations: true })
|
||||
await brain.init()
|
||||
|
|
@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions())
|
|||
|
||||
```typescript
|
||||
import express from 'express'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const app = express()
|
||||
const brain = new Brainy({
|
||||
|
|
@ -232,7 +232,7 @@ app.listen(3000, () => {
|
|||
|
||||
```typescript
|
||||
import { Hono } from 'hono'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const app = new Hono()
|
||||
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings.
|
|||
The simplest way to enable all integrations:
|
||||
|
||||
```javascript
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const brain = new Brainy({ integrations: true })
|
||||
await brain.init()
|
||||
|
|
@ -112,7 +112,7 @@ With Express:
|
|||
|
||||
```javascript
|
||||
import express from 'express'
|
||||
import { Brainy } from '@soulcraftlabs/brainy'
|
||||
import { Brainy } from '@soulcraft/brainy'
|
||||
|
||||
const app = express()
|
||||
const brain = new Brainy({ integrations: true })
|
||||
|
|
|
|||
8
package-lock.json
generated
8
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "@soulcraftlabs/brainy",
|
||||
"version": "10.4.4",
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "10.3.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@soulcraftlabs/brainy",
|
||||
"version": "10.4.4",
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "10.3.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@msgpack/msgpack": "^3.1.2",
|
||||
|
|
|
|||
14
package.json
14
package.json
|
|
@ -1,7 +1,6 @@
|
|||
{
|
||||
"name": "@soulcraftlabs/brainy",
|
||||
"version": "10.4.4",
|
||||
"brainyContract": 1,
|
||||
"name": "@soulcraft/brainy",
|
||||
"version": "10.3.0",
|
||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||
"main": "dist/index.js",
|
||||
"module": "dist/index.js",
|
||||
|
|
@ -127,16 +126,15 @@
|
|||
"license": "MIT",
|
||||
"private": false,
|
||||
"publishConfig": {
|
||||
"access": "public",
|
||||
"registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
|
||||
"access": "public"
|
||||
},
|
||||
"homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy",
|
||||
"homepage": "https://source.soulcraft.com/soulcraft/brainy",
|
||||
"bugs": {
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues"
|
||||
"url": "https://source.soulcraft.com/soulcraft/brainy/issues"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git"
|
||||
"url": "git+https://source.soulcraft.com/soulcraft/brainy.git"
|
||||
},
|
||||
"files": [
|
||||
"dist/**/*.js",
|
||||
|
|
|
|||
|
|
@ -1,76 +0,0 @@
|
|||
{
|
||||
"product": "brainy",
|
||||
"entries": [
|
||||
{
|
||||
"version": "11.0.5",
|
||||
"date": "2026-09-02",
|
||||
"headline": "Graph-first finds in production, and opens that stop rescanning history",
|
||||
"items": [
|
||||
"find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.",
|
||||
"related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).",
|
||||
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.4",
|
||||
"date": "2026-09-01",
|
||||
"headline": "Closes in milliseconds, index rebuilds without the disk-sync storm",
|
||||
"items": [
|
||||
"close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.",
|
||||
"The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.",
|
||||
"A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.3",
|
||||
"date": "2026-09-01",
|
||||
"headline": "The embedding upgrade ceremony runs on every brain",
|
||||
"items": [
|
||||
"A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.",
|
||||
"A one-fix release; nothing else changed."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.2",
|
||||
"date": "2026-08-31",
|
||||
"headline": "One embedding quality everywhere, 3–4× faster imports",
|
||||
"items": [
|
||||
"Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.",
|
||||
"Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.",
|
||||
"The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.1",
|
||||
"date": "2026-08-31",
|
||||
"headline": "Deletes inside transactions are safe",
|
||||
"items": [
|
||||
"Deleting relations inside a transact() no longer corrupts index bookkeeping.",
|
||||
"A store that deletes its last relation keeps serving instead of refusing."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.0",
|
||||
"date": "2026-08-28",
|
||||
"headline": "One install, one engine — Brainy",
|
||||
"items": [
|
||||
"The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.",
|
||||
"A missing native build refuses loudly with its cures named; nothing falls back silently.",
|
||||
"Stores open in place — no migration."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
}
|
||||
],
|
||||
"history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md."
|
||||
}
|
||||
|
|
@ -1,110 +0,0 @@
|
|||
{
|
||||
"product": "open-brainy",
|
||||
"entries": [
|
||||
{
|
||||
"version": "10.4.9",
|
||||
"date": "2026-09-02",
|
||||
"headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history",
|
||||
"items": [
|
||||
"find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).",
|
||||
"related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.",
|
||||
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.7",
|
||||
"date": "2026-09-01",
|
||||
"headline": "Count ledgers can no longer race themselves",
|
||||
"items": [
|
||||
"Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.",
|
||||
"Atomic writes carry a per-process sequence, so two processes' temp files can never collide."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.6",
|
||||
"date": "2026-08-31",
|
||||
"headline": "Transactions cross the index seam safely",
|
||||
"items": [
|
||||
"Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.",
|
||||
"Fixes a class of transaction failures on stores with integer-mapped relation endpoints."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.5",
|
||||
"date": "2026-08-31",
|
||||
"headline": "Recovery tells the truth, docs live at home",
|
||||
"items": [
|
||||
"A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.",
|
||||
"A sealed segment declares only the generations it actually holds.",
|
||||
"The engine's documentation now publishes from its own repository."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.4",
|
||||
"date": "2026-08-28",
|
||||
"headline": "Faster opens, quieter idle",
|
||||
"items": [
|
||||
"Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.",
|
||||
"The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.",
|
||||
"A slow open now names the exact step it is in, so operators see what is being paid and why."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.3",
|
||||
"date": "2026-08-27",
|
||||
"headline": "Open Brainy, under its own name",
|
||||
"items": [
|
||||
"The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.",
|
||||
"No code changes; your imports change once and everything else stays put."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.2",
|
||||
"date": "2026-08-27",
|
||||
"headline": "Vectors that lie are refused, counts that drift are caught",
|
||||
"items": [
|
||||
"A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.",
|
||||
"The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.",
|
||||
"Plugin activation failures keep their original error as cause, so the real frame reaches your logs."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.1",
|
||||
"date": "2026-08-26",
|
||||
"headline": "Writes that change nothing cost nothing",
|
||||
"items": [
|
||||
"The read gate is per index family, and a write carrying unchanged data never re-embeds.",
|
||||
"The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.0",
|
||||
"date": "2026-08-26",
|
||||
"headline": "Repair routing, the vector ledger, and honest empties",
|
||||
"items": [
|
||||
"Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.",
|
||||
"An empty string is real data, not a missing field.",
|
||||
"The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0",
|
||||
"thumb": null
|
||||
}
|
||||
],
|
||||
"history": "Earlier releases are recorded in CHANGELOG.md in this repository."
|
||||
}
|
||||
|
|
@ -10,7 +10,6 @@ import { TransformerEmbedding } from '../src/utils/embedding.js'
|
|||
import * as fs from 'fs/promises'
|
||||
import * as path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import { resolveDeterministicStamp } from './lib/deterministicStamp.js'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
|
|
@ -99,21 +98,12 @@ async function buildEmbeddedPatterns() {
|
|||
const uint8 = new Uint8Array(buffer)
|
||||
const base64 = Buffer.from(uint8).toString('base64')
|
||||
|
||||
// Deterministic stamp: derived from the git commit time of this
|
||||
// generator's inputs, never from wall-clock time — two builds of the
|
||||
// same source tree must produce byte-identical output.
|
||||
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts')
|
||||
const generatedStamp = resolveDeterministicStamp(
|
||||
[path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath],
|
||||
outputPath
|
||||
)
|
||||
|
||||
// Generate TypeScript file with everything embedded
|
||||
const tsContent = `/**
|
||||
* 🧠 BRAINY EMBEDDED PATTERNS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: ${generatedStamp}
|
||||
* Generated: ${new Date().toISOString()}
|
||||
* Patterns: ${libraryData.patterns.length}
|
||||
* Coverage: 94-98% of all queries
|
||||
*
|
||||
|
|
@ -207,6 +197,7 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length}
|
|||
`
|
||||
|
||||
// Write the TypeScript file
|
||||
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts')
|
||||
await fs.writeFile(outputPath, tsContent)
|
||||
|
||||
// Report statistics
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import * as fs from 'fs/promises'
|
|||
import * as path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import { NounType, VerbType } from '../src/types/graphTypes.js'
|
||||
import { resolveDeterministicStamp } from './lib/deterministicStamp.js'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
|
|
@ -374,24 +373,12 @@ async function buildTypeEmbeddings() {
|
|||
const uint8 = new Uint8Array(buffer)
|
||||
const base64 = Buffer.from(uint8).toString('base64')
|
||||
|
||||
// Deterministic stamp: derived from the git commit time of this
|
||||
// generator's inputs, never from wall-clock time — two builds of the
|
||||
// same source tree must produce byte-identical output.
|
||||
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts')
|
||||
const generatedStamp = resolveDeterministicStamp(
|
||||
[
|
||||
path.join(__dirname, 'buildTypeEmbeddings.ts'),
|
||||
path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts')
|
||||
],
|
||||
outputPath
|
||||
)
|
||||
|
||||
// Generate TypeScript file
|
||||
const tsContent = `/**
|
||||
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: ${generatedStamp}
|
||||
* Generated: ${new Date().toISOString()}
|
||||
* Noun Types: ${nounTypes.length}
|
||||
* Verb Types: ${verbTypes.length}
|
||||
*
|
||||
|
|
@ -408,7 +395,7 @@ export const TYPE_METADATA = {
|
|||
verbTypes: ${verbTypes.length},
|
||||
totalTypes: ${totalTypes},
|
||||
embeddingDimensions: ${embeddingDim},
|
||||
generatedAt: "${generatedStamp}",
|
||||
generatedAt: "${new Date().toISOString()}",
|
||||
sizeBytes: {
|
||||
embeddings: ${buffer.byteLength},
|
||||
base64: ${base64.length}
|
||||
|
|
@ -507,6 +494,7 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n
|
|||
`
|
||||
|
||||
// Write the TypeScript file
|
||||
const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts')
|
||||
await fs.writeFile(outputPath, tsContent)
|
||||
|
||||
// Report statistics
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Emit this build's API-contract manifest to docs/api-contract.json.
|
||||
*
|
||||
* WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the
|
||||
* code the first time somebody adds one. This reads the surface the build
|
||||
* actually exposes — the prototype's own methods and accessors, the exported
|
||||
* error classes, the `where` operator sets, the field-addressing vocabulary,
|
||||
* the health verdicts — so a diff between two engines' manifests is a diff
|
||||
* between two engines, never between two authors.
|
||||
*
|
||||
* Requirement marking (required / optional per door) is NOT derivable from the
|
||||
* surface — it is a commitment, recorded with the contract's owner rather than
|
||||
* here. This manifest carries the surface; the promise lives with the contract.
|
||||
*
|
||||
* Usage: node scripts/emit-contract-manifest.mjs [--check]
|
||||
* --check exits non-zero when the committed manifest is stale.
|
||||
*/
|
||||
|
||||
import { writeFileSync, readFileSync, existsSync } from 'node:fs'
|
||||
import { join, dirname } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const OUT = join(ROOT, 'docs', 'api-contract.json')
|
||||
|
||||
const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js'))
|
||||
const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js'))
|
||||
const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js'))
|
||||
const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js'))
|
||||
|
||||
/** Every own method and accessor on the class's prototype, minus the private ones. */
|
||||
function surfaceOf(ctor) {
|
||||
const doors = []
|
||||
for (const name of Object.getOwnPropertyNames(ctor.prototype)) {
|
||||
if (name === 'constructor' || name.startsWith('_')) continue
|
||||
const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name)
|
||||
if (!descriptor) continue
|
||||
if (typeof descriptor.value === 'function') {
|
||||
doors.push({ name, kind: 'method', arity: descriptor.value.length })
|
||||
} else if (descriptor.get) {
|
||||
doors.push({ name, kind: 'accessor' })
|
||||
}
|
||||
}
|
||||
return doors.sort((a, b) => a.name.localeCompare(b.name))
|
||||
}
|
||||
|
||||
const errors = Object.entries(errorsModule)
|
||||
.filter(([name, value]) => typeof value === 'function' && /Error$/.test(name))
|
||||
.map(([name]) => name)
|
||||
.sort()
|
||||
|
||||
// The operator sets, read from the engine's own refusal message so the
|
||||
// manifest can never disagree with the validator.
|
||||
const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8')
|
||||
const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set<string>\(\[([\s\S]*?)\]\)/)
|
||||
if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess')
|
||||
const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort()
|
||||
|
||||
const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8')
|
||||
const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) =>
|
||||
// Proven by the refusal path: these are the tokens with no case in the
|
||||
// index's operator switch, so they fall to its default and are refused.
|
||||
!new RegExp(`case '${op}':`).test(indexSource)
|
||||
)
|
||||
const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op))
|
||||
|
||||
const manifest = {
|
||||
contractVersion: versionModule.contractVersion(),
|
||||
engine: '@soulcraftlabs/brainy',
|
||||
compatibility: {
|
||||
minor:
|
||||
'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms',
|
||||
major:
|
||||
'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused'
|
||||
},
|
||||
doors: surfaceOf(Brainy),
|
||||
errors,
|
||||
operators: {
|
||||
accepted,
|
||||
servedOnIndexPath: servedOnIndex,
|
||||
refusedByIndexPath: refusedByIndex,
|
||||
combinators: ['allOf', 'anyOf', 'not']
|
||||
},
|
||||
fieldAddressing: {
|
||||
systemKeyPrefix: 'system.',
|
||||
systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(),
|
||||
systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(),
|
||||
plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort()
|
||||
},
|
||||
health: {
|
||||
verdicts: ['pass', 'warn', 'fail'],
|
||||
healKinds: ['none', 'repair', 'rebuild'],
|
||||
servingWithholdingInvariants: [
|
||||
'index-initialized',
|
||||
'durable-state-present',
|
||||
'manifest-residency',
|
||||
'replay-clean',
|
||||
'strand-latch'
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
const rendered = `${JSON.stringify(manifest, null, 2)}\n`
|
||||
|
||||
if (process.argv.includes('--check')) {
|
||||
if (!existsSync(OUT)) {
|
||||
console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`)
|
||||
process.exit(1)
|
||||
}
|
||||
if (readFileSync(OUT, 'utf-8') !== rendered) {
|
||||
console.error(
|
||||
`docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` +
|
||||
`the addition (minor = additive; a removal is a contract major).`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
writeFileSync(OUT, rendered)
|
||||
console.log(
|
||||
`Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` +
|
||||
`${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` +
|
||||
`${manifest.operators.accepted.length} operators ` +
|
||||
`(${manifest.operators.refusedByIndexPath.length} refused by the index path).`
|
||||
)
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
# Gate Guards
|
||||
|
||||
Two standalone scripts that stand between a test/build gate and a false
|
||||
verdict: one refuses to let the gate start on a noisy machine, the other
|
||||
refuses to let a truncated or crashed vitest run be read as green.
|
||||
|
||||
## Why these exist
|
||||
|
||||
Both guards exist because of the 2026-08-13 lost-day ledger: a gate ran on
|
||||
a machine under load, and separately a vitest worker pool died mid-suite
|
||||
while still printing a plausible-looking summary line, and in both cases
|
||||
the bad result was trusted and acted on for the better part of a day before
|
||||
anyone noticed. Neither failure mode announces itself — a loaded machine
|
||||
still finishes and reports numbers, and a truncated test run still prints a
|
||||
`Test Files` / `Tests` line — so both guards check the evidence explicitly
|
||||
rather than trusting that a gate finishing means the gate was valid.
|
||||
|
||||
## gate-preflight.sh
|
||||
|
||||
Run before any gate lane starts. Exits 1 the moment the machine isn't
|
||||
gate-clean, with one `FATAL:` line per violation naming the exact offender
|
||||
(the pid and command, the path, the measured value). Prints one `OK:` line
|
||||
per check that passes. `WARNING:` lines mark checks that were skipped, not
|
||||
failures.
|
||||
|
||||
Checks:
|
||||
|
||||
| # | Check | Default threshold | Override |
|
||||
|---|-------|--------------------|----------|
|
||||
| a | 1-minute load average | `nproc / 2` | `GATE_MAX_LOAD` |
|
||||
| b | any non-allowlisted process over 50% of one core | 50% | `GATE_ALLOW_REGEX` (extra pattern matched against the process's args) |
|
||||
| c | cpu0 scaling governor must be `performance` | — | none (warns and skips if the sysfs path is absent) |
|
||||
| d | free space on `/` and `/tmp` | 10G each | `GATE_SKIP_DISK_CHECK=1` to skip entirely |
|
||||
|
||||
The allowlist for check (b) is always: this script's own process tree
|
||||
(its ancestors and its direct child processes), `sshd`, `systemd`, and
|
||||
kernel threads (recognizable by args wrapped in brackets, e.g.
|
||||
`[kworker/0:1]`). `GATE_ALLOW_REGEX` extends it — it does not replace it.
|
||||
|
||||
## vitest-verdict-check.sh
|
||||
|
||||
Run after every vitest lane, against that lane's captured log. Fails
|
||||
loudly, quoting the exact line or string that tripped it, when the log's
|
||||
own summary can't be trusted:
|
||||
|
||||
- no `Test Files` (or, in `--count-tests` mode, `Tests`) summary line is
|
||||
present at all
|
||||
- the parenthesized total in that line doesn't match what was expected
|
||||
- fewer files/tests are accounted for (passed + failed + skipped) than the
|
||||
total claims — a truncated run
|
||||
- the log contains `Unhandled Error` or `Timeout calling` anywhere — a dead
|
||||
worker pool, regardless of what the summary line claims
|
||||
|
||||
```
|
||||
vitest-verdict-check.sh <log-file> <expected-file-count>
|
||||
vitest-verdict-check.sh --count-tests <log-file> <minimum-test-count>
|
||||
```
|
||||
|
||||
The first form checks `Test Files` for an exact match. The second checks
|
||||
`Tests` for a minimum (a floor, not an exact count, since the total number
|
||||
of individual tests moves more often than the number of test files).
|
||||
|
||||
## Wiring into a CI lane
|
||||
|
||||
```sh
|
||||
# Before any lane that will report a verdict:
|
||||
scripts/gate/gate-preflight.sh || exit 1
|
||||
|
||||
# Run the suite, capturing its output:
|
||||
npx vitest run tests/unit 2>&1 | tee /tmp/unit.log
|
||||
|
||||
# After every vitest lane, check the log against the actual file count:
|
||||
EXPECTED_FILES=$(ls tests/unit/**/*.test.ts | wc -l)
|
||||
scripts/gate/vitest-verdict-check.sh /tmp/unit.log "$EXPECTED_FILES" || exit 1
|
||||
```
|
||||
|
||||
## Exit-code contract
|
||||
|
||||
| Script | Exit 0 | Exit 1 |
|
||||
|--------|--------|--------|
|
||||
| `gate-preflight.sh` | machine is gate-clean | one or more `FATAL:` violations printed |
|
||||
| `vitest-verdict-check.sh` | log's summary is trustworthy and matches | usage error, missing/unreadable log, or one or more `FATAL:` violations printed |
|
||||
|
||||
Non-zero from either script means: do not trust the gate that was about to
|
||||
run, or the result of the one that just ran.
|
||||
|
|
@ -1,206 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Brainy Gate Preflight
|
||||
# Refuses to let a test/build gate run on a machine that isn't clean enough
|
||||
# to trust the numbers it produces. See scripts/gate/README.md for why (the
|
||||
# 2026-08-13 lost-day ledger).
|
||||
#
|
||||
# Checks: 1-minute load average, any non-allowlisted process pinning a core,
|
||||
# the cpu0 scaling governor, and free space on / and /tmp.
|
||||
#
|
||||
# Exit 0 and print one OK line per passing check when the machine is clean.
|
||||
# Exit 1 and print one FATAL line per violation, naming the offender, when
|
||||
# it is not.
|
||||
#
|
||||
# Known trap: a helper function whose last executed statement is a `while`
|
||||
# (or any command whose own exit status happens to be nonzero) hands that
|
||||
# status back as the function's return value. Called as a plain statement,
|
||||
# that silently kills this script under `set -e`. Every helper below ends
|
||||
# on an explicit `return 0` as its own statement, never on a loop or test.
|
||||
#
|
||||
# The same failure mode hides in plainer-looking lines too: `var=$(cmd)` is
|
||||
# a bare assignment, so `set -e` DOES treat a nonzero `cmd` (or, under
|
||||
# `pipefail`, a nonzero stage anywhere in `cmd`'s pipeline) as a failure of
|
||||
# that statement and kills the script right there — even mid-loop, even
|
||||
# when the "failure" is routine (a process that exited before a second
|
||||
# lookup, a path that doesn't exist). Every such assignment below is paired
|
||||
# with an explicit `|| var=""` fallback so a routine miss degrades to an
|
||||
# empty value instead of an exit.
|
||||
|
||||
VIOLATIONS=0
|
||||
ANCESTOR_PIDS=""
|
||||
|
||||
fatal() {
|
||||
echo "FATAL: $1"
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
}
|
||||
|
||||
ok() {
|
||||
echo "OK: $1"
|
||||
}
|
||||
|
||||
# Walks this process's parent chain up to pid 1, then takes one snapshot of
|
||||
# its direct children (the ps/read pipeline in check_processes), and
|
||||
# records both in ANCESTOR_PIDS — so the process-scan below can recognize
|
||||
# its own tree (the shell/terminal/session that launched it, plus its own
|
||||
# helper commands) instead of flagging it. Children are captured once, up
|
||||
# front, rather than re-queried per row later, so a helper command that has
|
||||
# already exited by the time it's looked up can't be mistaken for a miss.
|
||||
build_ancestor_pids() {
|
||||
local pid="$$"
|
||||
local ppid child
|
||||
ANCESTOR_PIDS=" $pid "
|
||||
while [ "$pid" != "1" ]; do
|
||||
ppid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') || ppid=""
|
||||
if [ -z "$ppid" ]; then
|
||||
break
|
||||
fi
|
||||
ANCESTOR_PIDS="${ANCESTOR_PIDS}${ppid} "
|
||||
pid="$ppid"
|
||||
done
|
||||
|
||||
while IFS= read -r child; do
|
||||
[ -z "$child" ] && continue
|
||||
ANCESTOR_PIDS="${ANCESTOR_PIDS}${child} "
|
||||
done < <(ps --ppid "$$" -o pid= 2>/dev/null || true)
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# (a) 1-minute load average vs. threshold (default: nproc / 2).
|
||||
check_load() {
|
||||
local max_load="${GATE_MAX_LOAD:-}"
|
||||
if [ -z "$max_load" ]; then
|
||||
max_load=$(( $(nproc) / 2 ))
|
||||
if [ "$max_load" -lt 1 ]; then
|
||||
max_load=1
|
||||
fi
|
||||
fi
|
||||
|
||||
local load_1m
|
||||
load_1m=$(cut -d' ' -f1 /proc/loadavg)
|
||||
|
||||
if awk -v l="$load_1m" -v m="$max_load" 'BEGIN { exit !(l > m) }'; then
|
||||
fatal "1-minute load average ${load_1m} exceeds threshold ${max_load} (GATE_MAX_LOAD=${max_load})"
|
||||
else
|
||||
ok "1-minute load average ${load_1m} is within threshold ${max_load}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# (b) any process outside the allowlist pinning more than half a core.
|
||||
# Parsed with `read` into named fields, not an awk/cut chain — a fixed-column
|
||||
# awk/cut split on `ps` output duplicated fields the first time this was
|
||||
# tried, because process args vary in word count. `read` with a fixed list
|
||||
# of variables dumps everything left over into the last one (args), which
|
||||
# handles that correctly.
|
||||
check_processes() {
|
||||
local max_pcpu=50
|
||||
local extra_regex="${GATE_ALLOW_REGEX:-}"
|
||||
local violation_found=0
|
||||
local line pcpu pid args pcpu_int
|
||||
|
||||
while IFS= read -r line; do
|
||||
[ -z "$line" ] && continue
|
||||
read -r pcpu pid args <<< "$line"
|
||||
|
||||
# Kernel threads report their comm in brackets, e.g. "[kworker/0:1]".
|
||||
case "$args" in
|
||||
\[*\]) continue ;;
|
||||
esac
|
||||
|
||||
# This script's own tree: its ancestors (shell, terminal, session) and
|
||||
# its direct children, both captured once by build_ancestor_pids.
|
||||
case " $ANCESTOR_PIDS " in
|
||||
*" $pid "*) continue ;;
|
||||
esac
|
||||
|
||||
case "$args" in
|
||||
*sshd*|*systemd*) continue ;;
|
||||
esac
|
||||
|
||||
if [ -n "$extra_regex" ] && [[ "$args" =~ $extra_regex ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
pcpu_int="${pcpu%.*}"
|
||||
if [ -z "$pcpu_int" ]; then
|
||||
pcpu_int=0
|
||||
fi
|
||||
if [ "$pcpu_int" -gt "$max_pcpu" ]; then
|
||||
fatal "pid ${pid} ('${args}') is using ${pcpu}% of one core"
|
||||
violation_found=1
|
||||
fi
|
||||
done < <(ps -eo pcpu,pid,args --sort=-pcpu | tail -n +2)
|
||||
|
||||
if [ "$violation_found" -eq 0 ]; then
|
||||
ok "no process outside the allowlist exceeds ${max_pcpu}% of one core"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# (c) cpu0 scaling governor must be "performance". Skipped with a warning
|
||||
# (not a violation) when the sysfs path doesn't exist on this machine.
|
||||
check_governor() {
|
||||
local gov_path="/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"
|
||||
if [ ! -r "$gov_path" ]; then
|
||||
echo "WARNING: ${gov_path} not present; skipping governor check"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local governor
|
||||
governor=$(cat "$gov_path" 2>/dev/null) || governor=""
|
||||
if [ "$governor" != "performance" ]; then
|
||||
fatal "cpu0 governor is '${governor}', not 'performance'"
|
||||
else
|
||||
ok "cpu0 governor is 'performance'"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# (d) free-space floors on / and /tmp (default 10G each). Skip entirely via
|
||||
# GATE_SKIP_DISK_CHECK=1.
|
||||
check_disk() {
|
||||
if [ "${GATE_SKIP_DISK_CHECK:-0}" = "1" ]; then
|
||||
echo "WARNING: disk free-space check skipped (GATE_SKIP_DISK_CHECK=1)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local floor_gb=10
|
||||
local floor_bytes=$((floor_gb * 1024 * 1024 * 1024))
|
||||
local path avail_bytes avail_gb
|
||||
|
||||
for path in / /tmp; do
|
||||
avail_bytes=$(df --output=avail -B1 "$path" 2>/dev/null | tail -n 1 | tr -d ' ') || avail_bytes=""
|
||||
if [ -z "$avail_bytes" ]; then
|
||||
echo "WARNING: could not determine free space on ${path}; skipping"
|
||||
continue
|
||||
fi
|
||||
if [ "$avail_bytes" -lt "$floor_bytes" ]; then
|
||||
avail_gb=$((avail_bytes / 1024 / 1024 / 1024))
|
||||
fatal "${path} has only ${avail_gb}G free, below the ${floor_gb}G floor"
|
||||
else
|
||||
ok "${path} has enough free space (floor ${floor_gb}G)"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "Brainy gate preflight"
|
||||
echo "----------------------"
|
||||
|
||||
build_ancestor_pids
|
||||
check_load
|
||||
check_processes
|
||||
check_governor
|
||||
check_disk
|
||||
|
||||
echo "----------------------"
|
||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
||||
echo "FATAL: gate preflight failed with ${VIOLATIONS} violation(s) — machine is not gate-clean"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "gate preflight passed — machine is gate-clean"
|
||||
exit 0
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Brainy Vitest Verdict Check
|
||||
# Confirms a vitest run's own summary line is trustworthy before anything
|
||||
# downstream treats a green run as green. See scripts/gate/README.md for why
|
||||
# (the 2026-08-13 lost-day ledger).
|
||||
#
|
||||
# Usage:
|
||||
# vitest-verdict-check.sh <log-file> <expected-file-count>
|
||||
# vitest-verdict-check.sh --count-tests <log-file> <minimum-test-count>
|
||||
#
|
||||
# The first form checks the "Test Files" summary line's total against an
|
||||
# exact expected count. The second checks the "Tests" summary line's total
|
||||
# against a minimum. Both also fail on any sign the worker pool died
|
||||
# mid-run, whether or not a summary line still made it into the log.
|
||||
#
|
||||
# Exit 0 and print one OK line per passing check when the log is clean.
|
||||
# Exit 1 and print one FATAL line per violation, quoting the exact line or
|
||||
# string that tripped it, when it is not.
|
||||
#
|
||||
# Known trap (shared with gate-preflight.sh): every helper below ends on an
|
||||
# explicit `return 0` as its own statement, never on a loop or test, so a
|
||||
# helper's last command can never hand its own exit status back as the
|
||||
# function's under `set -e`. The same applies to `var=$(cmd)` assignments
|
||||
# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that
|
||||
# legitimately finds nothing (exit 1) would otherwise kill the script
|
||||
# instead of just leaving the variable empty — every such assignment below
|
||||
# is paired with an explicit `|| true` inside the substitution.
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <log-file> <expected-file-count>"
|
||||
echo " $0 --count-tests <log-file> <minimum-test-count>"
|
||||
exit 1
|
||||
}
|
||||
|
||||
MODE="files"
|
||||
if [ "${1:-}" = "--count-tests" ]; then
|
||||
MODE="tests"
|
||||
shift
|
||||
fi
|
||||
|
||||
LOG_FILE="${1:-}"
|
||||
THRESHOLD="${2:-}"
|
||||
|
||||
if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
echo "FATAL: log file '${LOG_FILE}' does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then
|
||||
echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VIOLATIONS=0
|
||||
|
||||
fatal() {
|
||||
echo "FATAL: $1"
|
||||
VIOLATIONS=$((VIOLATIONS + 1))
|
||||
}
|
||||
|
||||
ok() {
|
||||
echo "OK: $1"
|
||||
}
|
||||
|
||||
# Vitest colorizes its summary with ANSI escapes; strip them before parsing
|
||||
# anything, or the color codes end up embedded in the fields we grep for.
|
||||
CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")"
|
||||
|
||||
# Worker-pool death: if either string appears, the run's own summary line —
|
||||
# even if present and even if its numbers look fine — cannot be trusted,
|
||||
# because the process died mid-suite and vitest's own accounting is what
|
||||
# died with it.
|
||||
check_worker_death() {
|
||||
if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then
|
||||
fatal "log contains 'Unhandled Error' — worker pool died mid-run"
|
||||
fi
|
||||
if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then
|
||||
fatal "log contains 'Timeout calling' — worker pool died mid-run"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Shared shape between the "Test Files" and "Tests" summary lines:
|
||||
# <label> <n> passed | <n> failed | <n> skipped (<total>)
|
||||
# `compare` is "eq" (total must equal threshold) or "min" (total must be at
|
||||
# least threshold).
|
||||
check_summary_line() {
|
||||
local label="$1"
|
||||
local threshold="$2"
|
||||
local compare="$3"
|
||||
local summary_line total accounted n
|
||||
|
||||
summary_line=$(echo "$CLEAN_LOG" | grep -E "^[[:space:]]*${label}[[:space:]]+" | tail -n 1 || true)
|
||||
|
||||
if [ -z "$summary_line" ]; then
|
||||
fatal "no '${label}' summary line found in ${LOG_FILE}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
total=$(echo "$summary_line" | grep -oE '\([0-9]+\)' | tr -d '()' | tail -n 1 || true)
|
||||
if [ -z "$total" ]; then
|
||||
fatal "'${label}' summary line has no parenthesized total: \"${summary_line}\""
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$compare" = "eq" ]; then
|
||||
if [ "$total" -ne "$threshold" ]; then
|
||||
fatal "'${label}' total is ${total}, expected ${threshold}: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' total matches expected ${threshold}"
|
||||
fi
|
||||
else
|
||||
if [ "$total" -lt "$threshold" ]; then
|
||||
fatal "'${label}' total is ${total}, below minimum ${threshold}: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' total ${total} meets minimum ${threshold}"
|
||||
fi
|
||||
fi
|
||||
|
||||
accounted=0
|
||||
for n in $(echo "$summary_line" | grep -oE '[0-9]+ (passed|failed|skipped)' | grep -oE '^[0-9]+'); do
|
||||
accounted=$((accounted + n))
|
||||
done
|
||||
|
||||
if [ "$accounted" -lt "$total" ]; then
|
||||
fatal "'${label}' line accounts for only ${accounted} of ${total} — truncated run: \"${summary_line}\""
|
||||
else
|
||||
ok "'${label}' line accounts for all ${total}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
echo "Brainy vitest verdict check: ${LOG_FILE}"
|
||||
echo "----------------------"
|
||||
|
||||
check_worker_death
|
||||
|
||||
if [ "$MODE" = "files" ]; then
|
||||
check_summary_line "Test Files" "$THRESHOLD" "eq"
|
||||
else
|
||||
check_summary_line "Tests" "$THRESHOLD" "min"
|
||||
fi
|
||||
|
||||
echo "----------------------"
|
||||
if [ "$VIOLATIONS" -gt 0 ]; then
|
||||
echo "FATAL: vitest verdict check failed with ${VIOLATIONS} violation(s) for ${LOG_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "vitest verdict check passed for ${LOG_FILE}"
|
||||
exit 0
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
/**
|
||||
* Deterministic generation-stamp resolution for Brainy's build-time code
|
||||
* generators.
|
||||
*
|
||||
* Two builds of the same source tree must produce byte-identical output.
|
||||
* A wall-clock stamp (`new Date()`) breaks that guarantee, so every
|
||||
* generator that writes a "Generated:" header or a `generatedAt` field
|
||||
* into its output must resolve the stamp through this module instead.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. The newest git commit timestamp among the generator's input files
|
||||
* (the generator script itself always counts as an input).
|
||||
* 2. If git metadata is unavailable (for example, building from a
|
||||
* published npm tarball with no `.git` directory), the stamp already
|
||||
* recorded in the previously generated output file.
|
||||
* 3. If neither is available, the fixed epoch string
|
||||
* `1970-01-01T00:00:00.000Z`.
|
||||
*
|
||||
* Every fallback logs a line to stderr — deterministic degradation is
|
||||
* loud, never a silent divergence.
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'child_process'
|
||||
import * as fs from 'fs'
|
||||
|
||||
const EPOCH_STAMP = '1970-01-01T00:00:00.000Z'
|
||||
const STAMP_PATTERN = /\*\s*Generated:\s*(\S+)/
|
||||
|
||||
/**
|
||||
* Resolve the deterministic stamp for a generator run.
|
||||
*
|
||||
* @param inputPaths Absolute paths to every file whose content determines
|
||||
* the generator's output, including the generator script itself.
|
||||
* @param previousOutputPath Absolute path to the previously generated
|
||||
* file, used for the existing-stamp fallback when git is unavailable.
|
||||
* @returns An ISO-8601 timestamp string that is deterministic for a given
|
||||
* source tree.
|
||||
*/
|
||||
export function resolveDeterministicStamp(
|
||||
inputPaths: string[],
|
||||
previousOutputPath: string
|
||||
): string {
|
||||
const gitStamp = newestGitCommitTimestamp(inputPaths)
|
||||
if (gitStamp) {
|
||||
return gitStamp
|
||||
}
|
||||
|
||||
const existingStamp = readExistingStamp(previousOutputPath)
|
||||
if (existingStamp) {
|
||||
process.stderr.write(
|
||||
`[deterministic-stamp] no git commit history found for generator inputs; ` +
|
||||
`reusing existing stamp from ${previousOutputPath}: ${existingStamp}\n`
|
||||
)
|
||||
return existingStamp
|
||||
}
|
||||
|
||||
process.stderr.write(
|
||||
`[deterministic-stamp] no git commit history and no previous output at ` +
|
||||
`${previousOutputPath}; falling back to fixed epoch stamp ${EPOCH_STAMP}\n`
|
||||
)
|
||||
return EPOCH_STAMP
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the newest git commit timestamp among the given input paths.
|
||||
* Returns null if git is unavailable, the tree is not a git repository,
|
||||
* or none of the inputs have any commit history yet.
|
||||
*/
|
||||
function newestGitCommitTimestamp(inputPaths: string[]): string | null {
|
||||
let newest: string | null = null
|
||||
|
||||
for (const inputPath of inputPaths) {
|
||||
if (!fs.existsSync(inputPath)) {
|
||||
continue
|
||||
}
|
||||
|
||||
let out: string
|
||||
try {
|
||||
out = execFileSync(
|
||||
'git',
|
||||
['log', '-1', '--format=%cI', '--', inputPath],
|
||||
{ stdio: ['ignore', 'pipe', 'ignore'] }
|
||||
)
|
||||
.toString()
|
||||
.trim()
|
||||
} catch {
|
||||
// git missing, not a repository, or no permissions — handled by the
|
||||
// caller's fallback chain.
|
||||
continue
|
||||
}
|
||||
|
||||
if (!out) {
|
||||
// Path exists but has no commit history yet (e.g. newly created,
|
||||
// uncommitted file).
|
||||
continue
|
||||
}
|
||||
|
||||
if (!newest || new Date(out).getTime() > new Date(newest).getTime()) {
|
||||
newest = out
|
||||
}
|
||||
}
|
||||
|
||||
return newest
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the `* Generated: <ISO timestamp>` header out of a previously
|
||||
* generated file, if one exists.
|
||||
*/
|
||||
function readExistingStamp(outputPath: string): string | null {
|
||||
if (!fs.existsSync(outputPath)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(outputPath, 'utf-8')
|
||||
const match = content.match(STAMP_PATTERN)
|
||||
return match ? match[1] : null
|
||||
}
|
||||
|
|
@ -15,12 +15,6 @@ NC='\033[0m' # No Color
|
|||
RELEASE_TYPE="${1:-patch}" # patch, minor, or major
|
||||
SKIP_TESTS=false
|
||||
DRY_RUN=false
|
||||
# --source-only is now a no-op: The Source is the one registry, so every
|
||||
# release already ships Source-only — tag, CI's publish to The Source, the
|
||||
# release page, and the docs push, with no separate storefront leg to skip.
|
||||
# The flag is still accepted (for backward-compatible invocations) and just
|
||||
# prints a notice; it no longer changes behavior.
|
||||
SOURCE_ONLY=false
|
||||
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
|
|
@ -30,9 +24,6 @@ for arg in "$@"; do
|
|||
--dry-run)
|
||||
DRY_RUN=true
|
||||
;;
|
||||
--source-only)
|
||||
SOURCE_ONLY=true
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
|
@ -109,7 +100,7 @@ else
|
|||
;;
|
||||
*)
|
||||
echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}"
|
||||
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run] [--source-only (no-op; The Source is the one registry)]"
|
||||
echo "Usage: ./scripts/release.sh [patch|minor|major|<explicit-version>] [--dry-run]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
@ -128,9 +119,6 @@ echo -e "${BLUE}New version: ${NEW_VERSION}${NC}"
|
|||
if [ "$PRERELEASE" = true ]; then
|
||||
echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}"
|
||||
fi
|
||||
if [ "$SOURCE_ONLY" = true ]; then
|
||||
echo -e "${YELLOW}⚠️ The Source is the one registry; --source-only is implied${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
|
|
@ -154,7 +142,7 @@ else
|
|||
fi
|
||||
|
||||
# Create new changelog entry
|
||||
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d))
|
||||
CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d))
|
||||
|
||||
${COMMITS}
|
||||
"
|
||||
|
|
@ -205,9 +193,9 @@ echo -e "${GREEN}✅ Pushed to origin${NC}\n"
|
|||
# .forgejo/workflows/publish-source.yml, which builds and publishes on The
|
||||
# Source's own runner (datacenter-side: seconds, not the laptop's WAN timing
|
||||
# out on an 87MB tarball PUT). The laptop holds no home-registry publish
|
||||
# credential anymore; it only waits for CI's result before continuing on to
|
||||
# the release page and the docs push.
|
||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/"
|
||||
# credential anymore; it only waits for CI's result before trusting the
|
||||
# home/npmjs pair enough to publish the storefront leg.
|
||||
SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/"
|
||||
SOURCE_POLL_INTERVAL_S=15
|
||||
SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml
|
||||
# backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0);
|
||||
|
|
@ -215,7 +203,7 @@ SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequen
|
|||
echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}"
|
||||
SOURCE_LANDED=false
|
||||
for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do
|
||||
LANDED_VERSION=$(npm view "@soulcraftlabs/brainy@${NEW_VERSION}" version "--@soulcraftlabs:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "")
|
||||
LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "")
|
||||
if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then
|
||||
SOURCE_LANDED=true
|
||||
break
|
||||
|
|
@ -228,8 +216,50 @@ if [ "$SOURCE_LANDED" = true ]; then
|
|||
echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n"
|
||||
else
|
||||
echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}"
|
||||
echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraftlabs/brainy@${NEW_VERSION} never became visible on the${NC}"
|
||||
echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting.${NC}"
|
||||
echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}"
|
||||
echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}"
|
||||
# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download
|
||||
# the tarball The Source serves and publish that file, never a fresh local pack
|
||||
# (a local rebuild can differ byte-wise, and the fleet verifies the pair by
|
||||
# shasum across registries).
|
||||
STOREFRONT_TMP="$(mktemp -d)"
|
||||
(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null)
|
||||
SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)"
|
||||
echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}"
|
||||
npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/"
|
||||
rm -rf "$STOREFRONT_TMP"
|
||||
# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish.
|
||||
npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true
|
||||
# Verify the pair is byte-identical by registry-reported shasum — divergence
|
||||
# here means the storefront leg must be treated as failed, loudly. RETRIED
|
||||
# with raw curl: npmjs metadata propagates with a lag measured in minutes,
|
||||
# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a
|
||||
# raw curl of the registry document already confirmed byte-identity. The
|
||||
# probe now reads the registry JSON directly (no npm cache in the path) and
|
||||
# gives propagation up to 5 minutes before calling the pair divergent.
|
||||
NPMJS_VERIFY_ATTEMPTS=20
|
||||
NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace
|
||||
SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable")
|
||||
PAIR_IDENTICAL=false
|
||||
for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do
|
||||
NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \
|
||||
| node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \
|
||||
|| echo "")
|
||||
if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then
|
||||
PAIR_IDENTICAL=true
|
||||
break
|
||||
fi
|
||||
echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}"
|
||||
sleep "$NPMJS_VERIFY_INTERVAL_S"
|
||||
done
|
||||
if [ "$PAIR_IDENTICAL" = true ]; then
|
||||
echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n"
|
||||
else
|
||||
echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -237,7 +267,7 @@ fi
|
|||
# and RELEASES.md are the record; this just gives The Source's UI a release page).
|
||||
echo -e "${BLUE}🔟 Creating release page on The Source...${NC}"
|
||||
if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then
|
||||
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \
|
||||
if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \
|
||||
-H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then
|
||||
echo -e "${GREEN}✅ Release page created on The Source${NC}\n"
|
||||
|
|
@ -248,15 +278,21 @@ else
|
|||
echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n"
|
||||
fi
|
||||
|
||||
# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled):
|
||||
# soulcraft.com/docs carries the paid product's documentation only. This
|
||||
# engine's documentation home is THIS repository — README and docs/ — and the
|
||||
# site serves 301s for the slugs this rail used to push. The push script stays
|
||||
# in the tree for history; the rail no longer calls it.
|
||||
echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}"
|
||||
# Step 12: Push public docs to the soulcraft.com docs ingest door
|
||||
# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when
|
||||
# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
|
||||
# that already happened) when a push errors, so the docs site never
|
||||
# silently trails npm.
|
||||
echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
|
||||
if node scripts/push-docs.js; then
|
||||
echo -e "${GREEN}✅ Docs push step done${NC}\n"
|
||||
else
|
||||
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo ""
|
||||
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}"
|
||||
echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}"
|
||||
echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}"
|
||||
|
|
|
|||
2999
src/brainy.ts
2999
src/brainy.ts
File diff suppressed because it is too large
Load diff
100
src/coreTypes.ts
100
src/coreTypes.ts
|
|
@ -792,30 +792,6 @@ export interface DerivedFamilyDeclaration {
|
|||
rebuildable?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The canonical count ledger a storage adapter maintains on its
|
||||
* write path: per family, the user-facing `counted` scalar and the
|
||||
* ALL-visibility `all` scalar (every tier — the coverage-ledger denominator).
|
||||
* See {@link StorageAdapter.getCanonicalCounts}.
|
||||
*/
|
||||
export interface CanonicalCounts {
|
||||
nouns: { counted: number; all: number }
|
||||
verbs: { counted: number; all: number }
|
||||
/**
|
||||
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
||||
* coverage denominator a vector index's node-count ledger is measured
|
||||
* against (`nodeCount === vectors.all` is the whole-store coverage
|
||||
* verdict for the vector leg, the vector-side mirror of `nouns.all` for
|
||||
* metadata/graph). A deferred-embed noun (`add({ deferEmbedding: true })`)
|
||||
* counts only once its vector actually LANDS — its canonical record exists
|
||||
* (counted in `nouns.all`) with an empty vector until then, so it is
|
||||
* deliberately NOT counted here in the interim.
|
||||
*/
|
||||
vectors: { all: number }
|
||||
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
||||
suspect: boolean
|
||||
}
|
||||
|
||||
export interface StorageAdapter {
|
||||
init(): Promise<void>
|
||||
|
||||
|
|
@ -830,68 +806,14 @@ export interface StorageAdapter {
|
|||
* Save noun metadata separately
|
||||
* @param id Noun ID
|
||||
* @param metadata Noun metadata
|
||||
* @param hasVector - OPTIONAL vectored-noun ledger hint: `true` when this
|
||||
* write is a FRESH insert (`isNew`) whose vector is a real, non-empty
|
||||
* array — the caller already knows this for free (the insert's own
|
||||
* `vector` local), so the increment rides the SAME isNew gate that
|
||||
* already protects `totalNounCountAll` from double-counting on HNSW
|
||||
* neighbor-link re-saves (`saveNoun_internal` re-runs on every link
|
||||
* change; this metadata seam does not). Absent/`false` ⇒ no ledger
|
||||
* action. A deferred-embed insert passes `false` (its vector lands
|
||||
* later — see {@link StorageAdapter.noteVectorLanded}).
|
||||
*/
|
||||
saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void>
|
||||
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>
|
||||
|
||||
/**
|
||||
* Delete noun metadata
|
||||
* @param id Noun ID
|
||||
* @param priorRecord - OPTIONAL already-known metadata (the caller's
|
||||
* pre-delete read) — see {@link StorageAdapter.deleteNoun}.
|
||||
* @param hadVector - OPTIONAL vectored-noun ledger hint: `true`/`false`
|
||||
* when the caller already knows (read as a side effect of ITS OWN delete
|
||||
* flow — e.g. `remove()`'s pre-read for the vector-index removal — never
|
||||
* a read added FOR this ledger), `undefined` when genuinely unknown. A
|
||||
* known `true` decrements the vectored-noun ledger; a known `false` is a
|
||||
* no-op (it was never counted); `undefined` marks the ledger SUSPECT
|
||||
* rather than guessing — the delete path must never add a canonical read
|
||||
* to answer this question.
|
||||
*/
|
||||
deleteNounMetadata(id: string, priorRecord?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
||||
|
||||
/**
|
||||
* OPTIONAL narrow ledger hook: record that a canonical noun's vector just
|
||||
* LANDED for the first time. Exists ONLY for the deferred-embedding
|
||||
* lifecycle — the landing commit (`system:embed-landing`) carries a vector
|
||||
* write with no accompanying metadata operation, so the normal
|
||||
* `saveNounMetadata(..., hasVector)` seam never fires for it. Callers MUST
|
||||
* call this only when the noun held NO real vector before this write (the
|
||||
* deferred-embed worker already holds that fact for free, from its own
|
||||
* pre-embed read — never an added read). A backend without vectored-noun
|
||||
* tracking is a no-op via this method's absence (feature-detected).
|
||||
* @param id - The noun whose vector just landed.
|
||||
*/
|
||||
noteVectorLanded?(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}:
|
||||
* record that a canonical noun's vector was just REMOVED — rewritten from
|
||||
* a real (non-empty) vector to the "unvectored" empty-array shape. Exists
|
||||
* for the ONE sanctioned reverse migration this engine supports: the VFS
|
||||
* root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and
|
||||
* `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix
|
||||
* store's all-zero placeholder root vector to `[]` and must decrement
|
||||
* `vectors.all` through this hook so the coverage ledger never drifts.
|
||||
* NOT a general-purpose "I removed a vector" callback — ordinary
|
||||
* application data has no sanctioned path from vectored back to
|
||||
* unvectored (`update()` refuses an empty vector as a dimension
|
||||
* mismatch by design). Callers MUST call this only when the noun held a
|
||||
* REAL vector immediately before this write (the caller already holds
|
||||
* that fact for free, from its own pre-write read — never an added read).
|
||||
* A backend without vectored-noun tracking is a no-op via this method's
|
||||
* absence (feature-detected).
|
||||
* @param id - The noun whose vector was just removed.
|
||||
*/
|
||||
noteVectorUnlanded?(id: string): Promise<void>
|
||||
deleteNounMetadata(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Get noun with metadata combined
|
||||
|
|
@ -940,11 +862,8 @@ export interface StorageAdapter {
|
|||
* REQUIRE re-reading the record being removed: when the internal read
|
||||
* returns `null` (replace race, or a ghost left by an earlier version) the
|
||||
* decrement falls back to this record instead of being silently skipped.
|
||||
* @param hadVector OPTIONAL vectored-noun ledger hint — see
|
||||
* {@link StorageAdapter.deleteNounMetadata}'s `hadVector` param, which
|
||||
* this forwards to unchanged.
|
||||
*/
|
||||
deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
||||
deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void>
|
||||
|
||||
/**
|
||||
* Save verb - Pure HNSW verb with core fields only
|
||||
|
|
@ -1374,19 +1293,6 @@ export interface StorageAdapter {
|
|||
*/
|
||||
getVerbCount(): Promise<number>
|
||||
|
||||
/**
|
||||
* The canonical count ledger — O(1), no I/O. `counted` mirrors
|
||||
* `getNounCount()` / `getVerbCount()` (public + internal tiers); `all` is
|
||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||
* against — the denominator a derived-index provider's coverage ledger
|
||||
* subtracts from. `suspect` is `true` when an unprovable delete has left
|
||||
* `all` unverified since the last sanctioned recount (`repairIndex()`).
|
||||
* Optional: adapters without the ledger omit it; a consumer treats absence
|
||||
* as "no denominator", never as zero.
|
||||
* @returns Both scalars per family plus the suspect flag.
|
||||
*/
|
||||
getCanonicalCounts?(): Promise<CanonicalCounts>
|
||||
|
||||
/**
|
||||
* OPTIONAL — create a pre-upgrade backup of the whole store and return its
|
||||
* location, or `null` when there is nothing to back up (empty store). On the
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
* speculative `with()` overlay; the canonical storage walk only ever answers
|
||||
* "what is live right now."
|
||||
*
|
||||
* All are exported from the package root (`@soulcraftlabs/brainy`).
|
||||
* All are exported from the package root (`@soulcraft/brainy`).
|
||||
*/
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -770,43 +770,6 @@ export class FactLog {
|
|||
* segments directly; the torn tail's invalid suffix is ignored exactly
|
||||
* like open() would).
|
||||
*/
|
||||
/**
|
||||
* STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold:
|
||||
* yields facts above the bound one SEGMENT at a time, ascending, without
|
||||
* ever materializing the whole log (a production first-boot fold OOM-class
|
||||
* allocation storm came from exactly that — GBs of decoded after-images in
|
||||
* one array while the process looked hung). Memory is one segment's worth.
|
||||
* Works manifest-direct (safe before {@link FactLog.open}). Ordering is
|
||||
* structural (segments rotate in order; appends are ordered within one) and
|
||||
* ASSERTED — a violation aborts loudly, never a silent misordered replay.
|
||||
*/
|
||||
async *streamFactsAbove(committedGeneration: number): AsyncGenerator<CommitFact[], void> {
|
||||
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
||||
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return
|
||||
if (stored.formatVersion !== FACTS_FORMAT_VERSION) return
|
||||
const files = [...stored.segments.map((s) => s.file)]
|
||||
if (stored.tailSegment) files.push(stored.tailSegment)
|
||||
let lastGen = committedGeneration
|
||||
for (const file of files) {
|
||||
const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
|
||||
if (bytes === null) continue
|
||||
const { facts } = parseSegment(file, bytes)
|
||||
const batch: CommitFact[] = []
|
||||
for (const f of facts) {
|
||||
if (f.generation <= committedGeneration) continue
|
||||
if (f.generation <= lastGen) {
|
||||
throw new Error(
|
||||
`fact log: streamFactsAbove found non-ascending generations ` +
|
||||
`(${f.generation} after ${lastGen} in ${file}) — refusing to replay out of order`
|
||||
)
|
||||
}
|
||||
lastGen = f.generation
|
||||
batch.push(f)
|
||||
}
|
||||
if (batch.length > 0) yield batch
|
||||
}
|
||||
}
|
||||
|
||||
async peekFactsAbove(committedGeneration: number): Promise<CommitFact[]> {
|
||||
const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
|
||||
if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return []
|
||||
|
|
|
|||
|
|
@ -12,11 +12,9 @@
|
|||
* the verified surface is a small set of rollup invariants (entity/
|
||||
* relationship counts) plus `sourceGeneration`.
|
||||
*
|
||||
* `sourceGeneration` is the COMMITTED generation of the source-of-truth log
|
||||
* this projection reflects — never the allocated counter, which names a
|
||||
* generation that may never commit (see {@link StampVerdict.torn}) — so
|
||||
* open-time coherence becomes a COMPARISON (stamp vs committed head), not a
|
||||
* walk:
|
||||
* `sourceGeneration` is the generation of the source-of-truth log this
|
||||
* projection reflects — open-time coherence becomes a COMPARISON (stamp vs
|
||||
* log head), not a walk:
|
||||
*
|
||||
* - equal + invariants hold → coherent, serve.
|
||||
* - behind → the projection missed the tail (crash between commit and stamp);
|
||||
|
|
@ -26,9 +24,6 @@
|
|||
* - invariants FAIL at equal generation → genuine incoherence: loud, and the
|
||||
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
|
||||
* canonical walk) heals it.
|
||||
* - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log
|
||||
* tail's. TERMINAL, never a wait — the generation the stamp names does not
|
||||
* exist to arrive.
|
||||
*
|
||||
* Stamps are JSON on purpose — every incident gets debugged by reading a
|
||||
* stamp in a terminal.
|
||||
|
|
@ -75,12 +70,6 @@ export type StampVerdict =
|
|||
| { state: 'coherent' }
|
||||
| { state: 'absent' } // legacy store — first stamp writes at the next flush
|
||||
| { state: 'behind'; stampSource: number; head: number }
|
||||
/**
|
||||
* TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the
|
||||
* store's committed watermark can no longer show. TERMINAL — there is no
|
||||
* generation to wait for, so the open demotes (or refuses) and never spins.
|
||||
*/
|
||||
| { state: 'torn'; stampSource: number; head: number }
|
||||
| { state: 'incoherent'; failures: string[] }
|
||||
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
|
||||
|
||||
|
|
@ -129,15 +118,12 @@ export function verifyFamilyStamp(
|
|||
): StampVerdict {
|
||||
if (stamp === null) return { state: 'absent' }
|
||||
if (stamp.sourceGeneration > head) {
|
||||
// A stamp AHEAD of committed truth witnesses a generation the store can no
|
||||
// longer show: the stamp's fsync survived a crash that the log tail did
|
||||
// not. This is the TORN GENERATION-LOG TAIL — its own class, never folded
|
||||
// in with `incoherent` (a count that drifted at a generation both sides
|
||||
// agree on), because the two have opposite cures: incoherence is recounted,
|
||||
// a tear is DEMOTED. It is also terminal by construction — there is no
|
||||
// generation the open can wait for, because the one the stamp names is
|
||||
// gone.
|
||||
return { state: 'torn', stampSource: stamp.sourceGeneration, head }
|
||||
// A stamp AHEAD of the log claims state that never committed — the
|
||||
// projection was stamped against truth that a crash rolled back.
|
||||
return {
|
||||
state: 'incoherent',
|
||||
failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
|
||||
}
|
||||
}
|
||||
if (stamp.sourceGeneration < head) {
|
||||
return { state: 'behind', stampSource: stamp.sourceGeneration, head }
|
||||
|
|
|
|||
|
|
@ -147,60 +147,6 @@ export class GenerationSegmentStore {
|
|||
return this.coveringSegment(gen) !== null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description True when `meta` declares more generations than it holds
|
||||
* frames — a segment sealed by a writer that folded across a hole. The
|
||||
* manifest records `frames` at fold time, so this is an O(1) comparison
|
||||
* against the declared span and needs no I/O.
|
||||
*/
|
||||
private isSparse(meta: SegmentMeta): boolean {
|
||||
return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The generations this tier ACTUALLY holds, as coalesced
|
||||
* ascending intervals — not what the segments declare.
|
||||
*
|
||||
* Dense segments (every one a current writer produces) contribute their
|
||||
* declared range with no I/O. A SPARSE segment — one sealed before the
|
||||
* density law was enforced, whose declared range spans generations it has
|
||||
* no frame for — has its real generation list read from its sidecar and
|
||||
* contributed instead, with the discrepancy narrated once.
|
||||
*
|
||||
* This is what keeps a store that already carries the damage from wedging.
|
||||
* `open()` seeds `committedRanges` from these intervals, so a hole is never
|
||||
* re-admitted as a committed generation, and the auto-compaction pass that
|
||||
* used to fail on every run with "packed history is damaged" simply never
|
||||
* asks for the missing frame.
|
||||
*
|
||||
* @returns Ascending, non-overlapping `[first, last]` intervals.
|
||||
*/
|
||||
async actualRanges(): Promise<Array<[number, number]>> {
|
||||
const out: Array<[number, number]> = []
|
||||
for (const meta of this.manifest.segments) {
|
||||
if (!this.isSparse(meta)) {
|
||||
out.push([meta.firstGeneration, meta.lastGeneration])
|
||||
continue
|
||||
}
|
||||
const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames
|
||||
prodLog.warn(
|
||||
`[GenerationSegments] sealed segment ${meta.file} declares generations ` +
|
||||
`${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` +
|
||||
`frame(s) — ${missing} generation(s) in that span were never folded into it. ` +
|
||||
`Serving the frames it actually holds; the declared span is not treated as ` +
|
||||
`committed history. (Written by a pre-density-law writer that folded across a ` +
|
||||
`gap; the segment itself is intact and no record is lost.)`
|
||||
)
|
||||
const idx = await this.sidecarFor(meta)
|
||||
for (const [gen] of idx.generations) {
|
||||
const last = out[out.length - 1]
|
||||
if (last !== undefined && gen === last[1] + 1) last[1] = gen
|
||||
else out.push([gen, gen])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold consecutive generations into ONE new sealed segment + sidecar and
|
||||
* append it to the manifest atomically. Caller guarantees: `gens` is
|
||||
|
|
@ -218,38 +164,6 @@ export class GenerationSegmentStore {
|
|||
throw new Error('[GenerationSegments] fold() input must be strictly ascending')
|
||||
}
|
||||
}
|
||||
// THE DENSITY LAW, MADE MECHANICAL.
|
||||
//
|
||||
// A sealed segment declares a CONTIGUOUS range [firstGeneration,
|
||||
// lastGeneration] and every reader treats that range as containment:
|
||||
// `coveringSegment` is an interval test, `hasGeneration` returns true for
|
||||
// anything inside it, and `open()` seeds committedRanges from it. So a
|
||||
// segment folded from a SPARSE input silently claims generations it does
|
||||
// not hold, and the first read of one of those holes throws
|
||||
// "inside sealed segment ... but has no frame — packed history is damaged".
|
||||
//
|
||||
// That is exactly how the damage was produced. `repackHistory` skipped
|
||||
// generations mid-batch — ones absent from committedRanges, ones still in
|
||||
// the pending buffer, ones whose tx.json would not read — and handed the
|
||||
// survivors here, where the range was computed from the first and last of
|
||||
// them. Worse, the mis-declared range was then merged back into
|
||||
// committedRanges at the next open, which is what turned a quiet hole into
|
||||
// a repeating auto-compaction failure on every subsequent run.
|
||||
//
|
||||
// Callers now split at discontinuities; this refusal is what keeps any
|
||||
// future caller from reintroducing the class. A refusal here loses
|
||||
// nothing — the generations stay in the live tier, readable, and the next
|
||||
// pass folds them correctly.
|
||||
for (let i = 1; i < gens.length; i++) {
|
||||
if (gens[i].generation !== gens[i - 1].generation + 1) {
|
||||
throw new Error(
|
||||
`[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` +
|
||||
`${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` +
|
||||
`generation(s). A sealed segment declares a dense range, so folding a sparse ` +
|
||||
`batch would claim generations it does not hold. Split the batch at the gap.`
|
||||
)
|
||||
}
|
||||
}
|
||||
const last = this.manifest.segments[this.manifest.segments.length - 1]
|
||||
if (last && gens[0].generation <= last.lastGeneration) {
|
||||
throw new Error(
|
||||
|
|
@ -450,37 +364,12 @@ export class GenerationSegmentStore {
|
|||
return this.decodeFrame(payload)
|
||||
}
|
||||
}
|
||||
// Inside the covering range but with no frame. Two very different causes,
|
||||
// and conflating them is what made this class wedge every maintenance pass
|
||||
// on the affected stores.
|
||||
//
|
||||
// (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than
|
||||
// the span it declares. That segment was sealed by a writer that
|
||||
// folded across a hole (the class this file's density law now bars).
|
||||
// The segment is INTACT and nothing is lost; it simply never held this
|
||||
// generation. Answering "not packed" is the honest answer, and it lets
|
||||
// the caller's two-tier read decide what a genuinely absent generation
|
||||
// means, instead of every compaction pass dying on a repeating throw.
|
||||
// `actualRanges()` keeps such holes out of committedRanges at open, so
|
||||
// in a healed store nobody asks this question in the first place.
|
||||
//
|
||||
// (2) A DENSE SEGMENT missing a frame it says it has — the manifest and
|
||||
// the sidecar disagree about a segment that claims to be complete.
|
||||
// That IS damage, and it stays loud.
|
||||
if (this.isSparse(meta)) {
|
||||
prodLog.warn(
|
||||
`[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` +
|
||||
`declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` +
|
||||
`holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` +
|
||||
`-generation span — it was sealed across a gap and never held this generation. ` +
|
||||
`Reporting it as unpacked rather than as damage; no record is lost.`
|
||||
)
|
||||
return null
|
||||
}
|
||||
// In the covering range but not present: the packed tier is dense by
|
||||
// construction (fold packs every generation it is handed, including
|
||||
// record-less ones) — absence inside a sealed range is damage.
|
||||
throw new Error(
|
||||
`[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
|
||||
`range but has no frame, and that segment declares a complete ${meta.frames}-frame ` +
|
||||
`span — the manifest and the sidecar disagree; packed history is damaged`
|
||||
`range but has no frame — packed history is damaged`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -96,35 +96,6 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json'
|
|||
/** Storage-root-relative prefix of the per-generation record directories. */
|
||||
export const GENERATIONS_PREFIX = '_generations'
|
||||
|
||||
/**
|
||||
* @description Split an ascending list of fold candidates into maximal
|
||||
* CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`.
|
||||
*
|
||||
* A sealed segment declares one dense range `[firstGeneration,
|
||||
* lastGeneration]`, and every reader treats that range as containment. So a
|
||||
* batch with a hole in it must never become one segment: it would claim a
|
||||
* generation it does not hold, and the first read of that hole reports the
|
||||
* packed history as damaged. One run, one segment — the ranges then describe
|
||||
* exactly what the segments contain.
|
||||
*
|
||||
* @param gens - Fold candidates, strictly ascending by generation.
|
||||
* @returns One array per contiguous run, in ascending order. Empty in, empty out.
|
||||
*/
|
||||
export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] {
|
||||
const runs: FoldGeneration[][] = []
|
||||
let run: FoldGeneration[] = []
|
||||
for (const g of gens) {
|
||||
const prev = run[run.length - 1]
|
||||
if (prev !== undefined && g.generation !== prev.generation + 1) {
|
||||
runs.push(run)
|
||||
run = []
|
||||
}
|
||||
run.push(g)
|
||||
}
|
||||
if (run.length > 0) runs.push(run)
|
||||
return runs
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Phases of the {@link GenerationStore.commitTransaction} commit
|
||||
* protocol at which a test-only fault injector can simulate a process crash.
|
||||
|
|
@ -566,30 +537,13 @@ export class GenerationStore {
|
|||
this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon')
|
||||
this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed)
|
||||
|
||||
// Discover existing generation record directories — BY DIRECTORY NAME.
|
||||
// This used to call listRawObjects(), which recurses the whole
|
||||
// `_generations/` tree and returns every file in every generation, to
|
||||
// extract a set of integers the top-level directory names already spell.
|
||||
// MEASURED on a real store with an 11 GB generation history: the phase
|
||||
// this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with
|
||||
// no fold to blame — this walk is what it was doing. An adapter without
|
||||
// the one-level door falls back to the recursive listing, unchanged.
|
||||
const seenGens = new Set<number>()
|
||||
const oneLevel = (
|
||||
this.storage as { listRawPrefixes?: (prefix: string) => Promise<string[]> }
|
||||
).listRawPrefixes
|
||||
if (typeof oneLevel === 'function') {
|
||||
for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) {
|
||||
const gen = Number(name)
|
||||
if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen)
|
||||
}
|
||||
} else {
|
||||
// Discover existing generation record directories.
|
||||
const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX)
|
||||
const seenGens = new Set<number>()
|
||||
for (const p of recordPaths) {
|
||||
const gen = parseGenerationFromPath(p)
|
||||
if (gen !== null) seenGens.add(gen)
|
||||
}
|
||||
}
|
||||
|
||||
let rolledBack = 0
|
||||
// Coalesce the ascending on-disk committed gens into interval form: each
|
||||
|
|
@ -683,73 +637,22 @@ export class GenerationStore {
|
|||
this.foldCheckpointChainValid = checkpoint !== null || this.committed === 0
|
||||
this.foldCheckpoint = foldBound
|
||||
if (uncleanOpen) this.foldCheckpointChainValid = true
|
||||
// THE FOLD STREAMS AND NARRATES. A production first boot after a live
|
||||
// flip folded ~7k facts by materializing them all (GBs of decoded
|
||||
// after-images, a GC storm, a starved write lane) in SILENCE — the
|
||||
// operator restarted the process three times mid-fold, each restart
|
||||
// making the next boot unclean again. Two laws from that day: the
|
||||
// fold consumes the log one segment-batch at a time (memory = one
|
||||
// segment, any log size), and it announces itself BEFORE the work
|
||||
// with progress lines DURING it — an operator who can see a fold
|
||||
// converging lets it finish.
|
||||
const foldKind = uncleanOpen
|
||||
? foldBound > 0
|
||||
? `BOUNDED fold above checkpoint ${foldBound}`
|
||||
: 'WHOLE-LOG fold'
|
||||
: 'above-manifest replay'
|
||||
const factsToReplay = uncleanOpen
|
||||
? await this.factLog.peekFactsAbove(foldBound)
|
||||
: orphans
|
||||
if (factsToReplay.length > 0) {
|
||||
let replayed = 0
|
||||
const foldStartedAt = Date.now()
|
||||
const replayFact = async (fact: CommitFact): Promise<void> => {
|
||||
for (const fact of factsToReplay) {
|
||||
for (const op of fact.ops) {
|
||||
let image: { metadata: unknown | null; vector: unknown | null }
|
||||
if (op.record === null) {
|
||||
// A genuine tombstone (both legs absent) — the fold removes
|
||||
// both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw
|
||||
// exact-restore contract.
|
||||
image = { metadata: null, vector: null }
|
||||
} else if (
|
||||
op.record.metadata !== null &&
|
||||
(op.record.vector === null || op.record.vector === undefined)
|
||||
) {
|
||||
// PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
|
||||
// half): a metadata-only after-image must never DELETE an
|
||||
// existing vector leg through the fold. `writeNounRaw`/
|
||||
// `writeVerbRaw` are exact-restore primitives — a `vector:
|
||||
// null` there means "delete", which is exactly right for
|
||||
// `rollBackUncommittedGeneration`'s before-image restore (a
|
||||
// transaction abort legitimately un-writes a vector the failed
|
||||
// transaction added). It is NOT right here: this fold replays
|
||||
// AFTER-IMAGES, and re-applying an already-intact record must
|
||||
// be byte-safe (this module's own invariant, see the log-authority
|
||||
// comment above) — silently erasing a landed vector because one
|
||||
// replayed fact's vector leg came back null is the exact defect
|
||||
// that left metadata-counted, never-enumerated rows in a
|
||||
// production store (confirmed root cause: the enumeration walk
|
||||
// used to key on the vector leg, so a preserved-but-then-deleted
|
||||
// vector made the row invisible while the ledger still counted
|
||||
// it by metadata). A genuine "unvector" has its own sanctioned,
|
||||
// ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
|
||||
// never this raw primitive, and never the fold.
|
||||
const current =
|
||||
op.kind === 'verb'
|
||||
? await this.storage.readVerbRaw(op.id)
|
||||
: await this.storage.readNounRaw(op.id)
|
||||
image = { metadata: op.record.metadata, vector: current.vector ?? null }
|
||||
} else {
|
||||
image = { metadata: op.record.metadata, vector: op.record.vector }
|
||||
}
|
||||
const image =
|
||||
op.record === null
|
||||
? { metadata: null, vector: null }
|
||||
: { metadata: op.record.metadata, vector: op.record.vector }
|
||||
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
|
||||
else await this.storage.writeNounRaw(op.id, image)
|
||||
this.noteCheckpointDirty(op.kind, op.id)
|
||||
}
|
||||
replayed++
|
||||
if (replayed % 1000 === 0) {
|
||||
prodLog.narrate(
|
||||
`[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` +
|
||||
`in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` +
|
||||
`do not restart, the fold is finite`
|
||||
)
|
||||
}
|
||||
if (fact.generation > this.committed) {
|
||||
this.committed = fact.generation
|
||||
this.appendCommittedGen(fact.generation)
|
||||
|
|
@ -761,20 +664,6 @@ export class GenerationStore {
|
|||
})
|
||||
}
|
||||
}
|
||||
if (uncleanOpen) {
|
||||
prodLog.narrate(
|
||||
`[GenerationStore] log-authority recovery: ${foldKind} beginning ` +
|
||||
`(unclean shutdown detected) — streaming replay, bounded memory, ` +
|
||||
`progress every 1000 facts. Do not restart the process; a restart ` +
|
||||
`re-pays the whole fold.`
|
||||
)
|
||||
for await (const batch of this.factLog.streamFactsAbove(foldBound)) {
|
||||
for (const fact of batch) await replayFact(fact)
|
||||
}
|
||||
} else {
|
||||
for (const fact of orphans) await replayFact(fact)
|
||||
}
|
||||
if (replayed > 0) {
|
||||
if (this.counter < this.committed) this.counter = this.committed
|
||||
await this.persistCounterUnlocked()
|
||||
const manifest: GenerationManifest = {
|
||||
|
|
@ -785,10 +674,15 @@ export class GenerationStore {
|
|||
}
|
||||
await this.storage.writeRawObject(MANIFEST_PATH, manifest)
|
||||
await this.storage.syncRawObjects([MANIFEST_PATH])
|
||||
prodLog.narrate(
|
||||
prodLog.warn(
|
||||
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
|
||||
`canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` +
|
||||
`${this.committed}) — an acked write is never lost`
|
||||
`canonical (${
|
||||
uncleanOpen
|
||||
? foldBound > 0
|
||||
? `BOUNDED fold above checkpoint ${foldBound} — unclean shutdown`
|
||||
: 'WHOLE-LOG fold — unclean shutdown'
|
||||
: 'above-manifest'
|
||||
}; committed at ${this.committed}) — an acked write is never lost`
|
||||
)
|
||||
}
|
||||
// A recovery fold re-applied (and the barrier below re-syncs) every
|
||||
|
|
@ -813,15 +707,9 @@ export class GenerationStore {
|
|||
if (storageSupportsFactLog(this.storage)) {
|
||||
this.segments = new GenerationSegmentStore(this.storage)
|
||||
await this.segments.open()
|
||||
// ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law
|
||||
// writer can declare a span wider than the frames it holds; seeding
|
||||
// committedRanges from the declared span re-admits those holes as
|
||||
// committed generations, and every later maintenance pass then asks for a
|
||||
// frame that was never written. `actualRanges()` reads the real
|
||||
// generation list from the sidecar for exactly those segments (and does
|
||||
// no I/O for the dense ones, which is all of them on a healthy store).
|
||||
const packedRanges = (await this.segments.actualRanges())
|
||||
.map((r): [number, number] => [r[0], Math.min(r[1], this.committed)])
|
||||
const packedRanges = this.segments
|
||||
.segments()
|
||||
.map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
|
||||
.filter(([lo, hi]) => lo <= hi)
|
||||
if (packedRanges.length > 0) {
|
||||
// Merge packed (older) + live (newer) interval sets — both ascending;
|
||||
|
|
@ -1009,44 +897,6 @@ export class GenerationStore {
|
|||
this.authorityIsLog = true
|
||||
}
|
||||
|
||||
/** Whether the fold-checkpoint chain is armed (a bounded fold is possible). */
|
||||
foldCheckpointChainArmed(): boolean {
|
||||
return this.foldCheckpointChainValid
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Stamp the fold checkpoint after the caller has completed a
|
||||
* FULL canonical barrier (every live row's canonical bytes fsynced, paged —
|
||||
* the adoption path does this right after a non-fresh flip). The stamp
|
||||
* asserts total coverage, so it may ONLY be called when the barrier walked
|
||||
* everything; stamp-after-data is the caller's ordering to keep. Arms the
|
||||
* chain: the brain's first unclean boot folds (checkpoint, head] instead of
|
||||
* the whole log — a production first boot after a live flip paid a full-log
|
||||
* fold through three mid-fold restarts because the chain could previously
|
||||
* only arm at a crash.
|
||||
*/
|
||||
async stampFoldCheckpointAfterFullBarrier(): Promise<void> {
|
||||
return this.withMutex(async () => {
|
||||
if (!this.authorityIsLog || !this.factLog) {
|
||||
throw new Error(
|
||||
'stampFoldCheckpointAfterFullBarrier: only a log-authority brain stamps a fold checkpoint'
|
||||
)
|
||||
}
|
||||
this.foldCheckpointChainValid = true
|
||||
// The full barrier supersedes any accumulated partial set.
|
||||
this.checkpointDirtyNouns = new Set()
|
||||
this.checkpointDirtyVerbs = new Set()
|
||||
const target = this.committed
|
||||
await this.storage.writeRawObject(FOLD_CHECKPOINT_PATH, { generation: target })
|
||||
await this.storage.syncRawObjects([FOLD_CHECKPOINT_PATH])
|
||||
this.foldCheckpoint = target
|
||||
prodLog.info(
|
||||
`[GenerationStore] fold checkpoint founded at generation ${target} — ` +
|
||||
`crash recovery is bounded from this moment`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Adoption-time chain bootstrap, abort — called when an
|
||||
* adoption attempt throws or refuses after phase 1. Disarms the chain and
|
||||
|
|
@ -3156,26 +3006,13 @@ export class GenerationStore {
|
|||
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
|
||||
}
|
||||
if (foldInput.length === 0) continue
|
||||
// SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three
|
||||
// filters above punch holes in it: a generation missing from
|
||||
// committedRanges never appears, one still in the pending buffer is
|
||||
// skipped, and one whose tx.json will not read is skipped. A sealed
|
||||
// segment declares a DENSE range, so folding across such a hole makes
|
||||
// the segment claim a generation it does not hold; the next open
|
||||
// merges that mis-declared range into committedRanges, and every
|
||||
// subsequent auto-compaction pass then asks for the missing frame and
|
||||
// fails with "packed history is damaged". Fold each contiguous RUN as
|
||||
// its own segment instead — same bytes, honest ranges.
|
||||
for (const run of contiguousRuns(foldInput)) {
|
||||
if (deadline !== undefined && Date.now() >= deadline) break
|
||||
await segments.fold(run)
|
||||
await segments.fold(foldInput)
|
||||
segmentsCreated++
|
||||
// Segment + manifest durable → the live copies retire.
|
||||
for (const g of run) {
|
||||
for (const g of foldInput) {
|
||||
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
|
||||
}
|
||||
folded += run.length
|
||||
}
|
||||
folded += foldInput.length
|
||||
}
|
||||
if (folded > 0) {
|
||||
prodLog.info(
|
||||
|
|
|
|||
|
|
@ -450,21 +450,6 @@ export interface GenerationStorage {
|
|||
deleteRawObject(path: string): Promise<void>
|
||||
/** List raw object paths under a prefix (normalized, `.gz`-stripped). */
|
||||
listRawObjects(prefix: string): Promise<string[]>
|
||||
/**
|
||||
* OPTIONAL: the IMMEDIATE child directory names under a prefix — one level,
|
||||
* no recursion, no file paths.
|
||||
*
|
||||
* Why it exists: discovering which generations are on disk needs only the
|
||||
* top-level directory NAMES under `_generations/`, but the only door for it
|
||||
* was `listRawObjects`, which recurses the whole tree and returns every file
|
||||
* in every generation. On a store with a long history that is a full walk of
|
||||
* the entire generation log, paid on EVERY open, to learn a set of integers
|
||||
* the directory names already spell out.
|
||||
*
|
||||
* An adapter without this door keeps working — the caller falls back to the
|
||||
* recursive listing.
|
||||
*/
|
||||
listRawPrefixes?(prefix: string): Promise<string[]>
|
||||
/** Remove every object under a prefix (and the directory itself on disk). */
|
||||
removeRawPrefix(prefix: string): Promise<void>
|
||||
/** Durability barrier: fsync the given object paths (no-op in memory). */
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ async function loadBunAssets(): Promise<ModelAssets> {
|
|||
}
|
||||
|
||||
// Strategy 2: node_modules path relative to CWD (for installed packages)
|
||||
const nmPath = './node_modules/@soulcraftlabs/brainy/assets/models/all-MiniLM-L6-v2'
|
||||
const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2'
|
||||
pathsToTry.push([
|
||||
`${nmPath}/model.safetensors`,
|
||||
`${nmPath}/tokenizer.json`,
|
||||
|
|
@ -168,9 +168,9 @@ async function loadBunAssets(): Promise<ModelAssets> {
|
|||
// If all strategies fail, provide helpful error message
|
||||
throw new Error(
|
||||
'Could not load model assets. For bun --compile, ensure model files are accessible:\n' +
|
||||
' Option 1: Keep node_modules/@soulcraftlabs/brainy/assets/ alongside your binary\n' +
|
||||
' Option 1: Keep node_modules/@soulcraft/brainy/assets/ alongside your binary\n' +
|
||||
' Option 2: Copy assets/ folder to your working directory\n' +
|
||||
' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraftlabs/brainy/assets/**/*"'
|
||||
' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraft/brainy/assets/**/*"'
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -190,7 +190,7 @@ async function loadNodeAssets(): Promise<ModelAssets> {
|
|||
if (!fs.existsSync(assetsDir)) {
|
||||
throw new Error(
|
||||
`Model assets not found: ${assetsDir}\n` +
|
||||
`Ensure @soulcraftlabs/brainy is installed correctly.`
|
||||
`Ensure @soulcraft/brainy is installed correctly.`
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
* - {@link RelationNotFoundError} — a referenced relationship (verb) does
|
||||
* not exist.
|
||||
*
|
||||
* Both are exported from the package root (`@soulcraftlabs/brainy`).
|
||||
* Both are exported from the package root (`@soulcraft/brainy`).
|
||||
*/
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1052,17 +1052,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
*/
|
||||
private startAutoFlush(): void {
|
||||
this.flushTimer = setInterval(async () => {
|
||||
// NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an
|
||||
// index nobody has written to since the last flush has nothing to
|
||||
// write, and calling into the trees (and their logging) on a cadence
|
||||
// over a quiet store is exactly the idle cost this law exists to
|
||||
// remove.
|
||||
if (
|
||||
!this.lsmTreeVerbsBySource.hasPendingWrites() &&
|
||||
!this.lsmTreeVerbsByTarget.hasPendingWrites()
|
||||
) {
|
||||
return
|
||||
}
|
||||
await this.flush()
|
||||
}, this.config.flushInterval)
|
||||
// Background maintenance must never keep the host process alive —
|
||||
|
|
|
|||
|
|
@ -687,17 +687,6 @@ export class LSMTree {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Whether this tree holds anything a flush would write —
|
||||
* the MemTable is non-empty. Synchronous and O(1), so a background cadence
|
||||
* can ask before it does anything at all: the engine does no periodic work
|
||||
* without a cause.
|
||||
* @returns true when a flush would write; false when it would be a no-op.
|
||||
*/
|
||||
hasPendingWrites(): boolean {
|
||||
return !this.memTable.isEmpty()
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.stopCompactionTimer()
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
Vector,
|
||||
VectorDocument
|
||||
} from '../coreTypes.js'
|
||||
import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js'
|
||||
import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
|
||||
import type { BaseStorage } from '../storage/baseStorage.js'
|
||||
import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
|
|
@ -64,34 +64,6 @@ export class HnswFlushError extends Error {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link
|
||||
* JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0
|
||||
* vector is the sanctioned "unvectored" shape for a canonical noun record
|
||||
* (class-J: a VFS-system row, a deferred embed not yet landed, or any other
|
||||
* legitimately-vector-less row) — but it is NEVER a legal INDEX insert. The
|
||||
* index itself has no concept of "unvectored"; deciding that a row is
|
||||
* unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job
|
||||
* (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem.
|
||||
* A length-0 vector reaching this point is a caller bug: silently accepting
|
||||
* it would pin `this.dimension = 0` on an empty index (poisoning every real
|
||||
* insert thereafter with a dimension mismatch) or store a vector-less node
|
||||
* that a distance calculation can never safely compare against. Loud errors,
|
||||
* never quiet losses — this throws instead of either.
|
||||
*/
|
||||
export class EmptyVectorIndexError extends Error {
|
||||
constructor(public readonly id: string, operation: 'addItem' | 'updateItem') {
|
||||
super(
|
||||
`${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` +
|
||||
`sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` +
|
||||
`insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` +
|
||||
`themselves (unvectored = nothing to index, not an error at that layer); reaching ` +
|
||||
`here with one is a caller bug.`
|
||||
)
|
||||
this.name = 'EmptyVectorIndexError'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls
|
||||
* on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native
|
||||
|
|
@ -608,15 +580,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
throw new Error('Vector is undefined or null')
|
||||
}
|
||||
|
||||
// THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc):
|
||||
// an empty vector is the sanctioned "unvectored" shape at the canonical
|
||||
// layer, never a legal index member. Refusing here — loudly, before the
|
||||
// dimension pin below — means no future fill/rebuild/load path can ever
|
||||
// poison `this.dimension` to 0 or park a vector-less node in the graph.
|
||||
if (vector.length === 0) {
|
||||
throw new EmptyVectorIndexError(id, 'addItem')
|
||||
}
|
||||
|
||||
// Set dimension on first insert
|
||||
if (this.dimension === null) {
|
||||
this.dimension = vector.length
|
||||
|
|
@ -991,13 +954,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
return
|
||||
}
|
||||
|
||||
// Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an
|
||||
// in-place relink must never rewrite an already-indexed node down to the
|
||||
// unvectored shape or poison the pinned dimension.
|
||||
if (vector.length === 0) {
|
||||
throw new EmptyVectorIndexError(id, 'updateItem')
|
||||
}
|
||||
|
||||
if (this.dimension === null) {
|
||||
this.dimension = vector.length
|
||||
} else if (vector.length !== this.dimension) {
|
||||
|
|
@ -1599,15 +1555,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
}
|
||||
|
||||
const loaded = await this.storage.getNounVector(noun.id)
|
||||
// `loaded` is a length-0 array (not null/undefined) for a canonical row
|
||||
// that is legitimately unvectored — `![]` is FALSE (an empty array is
|
||||
// truthy), so the bare `!loaded` check below would silently accept it
|
||||
// as "found" and hand a dimension-0 vector to a distance calculation.
|
||||
// A node only reaches this lazy-load path because it is a MEMBER of
|
||||
// the live index (rebuild() now refuses to admit unvectored rows — see
|
||||
// its JSDoc), so an empty vector here is never legitimate: treat it
|
||||
// exactly like "not found", loudly.
|
||||
if (!loaded || loaded.length === 0) {
|
||||
if (!loaded) {
|
||||
throw new Error(`Vector not found for noun ${noun.id}`)
|
||||
}
|
||||
|
||||
|
|
@ -1817,56 +1765,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
|
||||
totalCount = result.totalCount || result.items.length
|
||||
|
||||
// UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical
|
||||
// noun whose vector leg is `[]` (a VFS-root-style system row, a
|
||||
// deferred embed not yet landed, or a best-effort fallback for an
|
||||
// unreadable vector leg) is a normal, enumerable, countable row — it
|
||||
// is simply not indexed. `storage.getVectorIndexData()` derives its
|
||||
// {level, connections} answer straight from the noun's OWN record, so
|
||||
// it returns non-null for every existing noun regardless of whether
|
||||
// that noun ever actually reached `addItem()` — it cannot be used to
|
||||
// decide indexability. `nounData.vector.length === 0` is the one
|
||||
// truthful signal (mirrors the `noun.vector.length > 0` guards in
|
||||
// {@link getVectorSafe} / {@link getVectorSync}): skip here, counted
|
||||
// once in a summary line, never per-row spam.
|
||||
let skippedUnvectored = 0
|
||||
|
||||
// Process all nouns at once
|
||||
for (const nounData of result.items) {
|
||||
try {
|
||||
if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) {
|
||||
skippedUnvectored++
|
||||
continue
|
||||
}
|
||||
// THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm
|
||||
// vector (a pre-10.4.2 row the canonical write has not yet
|
||||
// normalized) must never enter the index either, mirroring the
|
||||
// belt AddToVectorIndexOperation enforces on the live write path.
|
||||
// Only the canonical vector is authoritative here — persisted
|
||||
// HNSW graph metadata (level/connections) can outlive an unvector.
|
||||
if (isZeroNormVector(nounData.vector)) {
|
||||
prodLog.warn(
|
||||
`[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` +
|
||||
`zero-norm (a zero-norm vector is not a vector and never crosses an ` +
|
||||
`engine boundary)`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// Restore the pinned dimension from the first real vector this
|
||||
// rebuild loads. `addItem`/`updateItem` only pin `this.dimension`
|
||||
// on a LIVE insert — a fresh rebuild from storage never goes
|
||||
// through either, so without this the pin stays `null` across a
|
||||
// restart. A `null` pin means the very next insert (correct OR
|
||||
// wrong length) silently BECOMES the new pin instead of being
|
||||
// checked against the store's real dimension — the wrong-length
|
||||
// case then fails much later and less clearly, inside a distance
|
||||
// calculation against an already-loaded node, instead of here,
|
||||
// immediately, with a named expected-vs-got mismatch.
|
||||
if (this.dimension === null) {
|
||||
this.dimension = nounData.vector.length
|
||||
}
|
||||
|
||||
// Load HNSW graph data for this entity
|
||||
const hnswData = await this.storage.getVectorIndexData(nounData.id)
|
||||
|
||||
|
|
@ -1914,10 +1815,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
options.onProgress(loadedCount, totalCount)
|
||||
}
|
||||
|
||||
prodLog.info(
|
||||
`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` +
|
||||
(skippedUnvectored > 0 ? ` — ${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '')
|
||||
)
|
||||
prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`)
|
||||
}
|
||||
|
||||
// Step 5: CRITICAL - Recover entry point if missing)
|
||||
|
|
|
|||
14
src/index.ts
14
src/index.ts
|
|
@ -80,9 +80,7 @@ export type {
|
|||
AggregationOp,
|
||||
TimeWindowGranularity,
|
||||
GroupByDimension,
|
||||
AggregationProvider,
|
||||
RepairReport,
|
||||
RepairFamilyReport,
|
||||
AggregationProvider
|
||||
} from './types/brainy.types.js'
|
||||
|
||||
// Read-barrier contract (waitForIndexed): the leg names, the options, and
|
||||
|
|
@ -184,7 +182,6 @@ export {
|
|||
|
||||
// Export version utilities
|
||||
export { getBrainyVersion } from './utils/version.js'
|
||||
export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js'
|
||||
|
||||
// Export plugin system
|
||||
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'
|
||||
|
|
@ -270,10 +267,6 @@ export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.j
|
|||
export { isVersionedIndexProvider } from './plugin.js'
|
||||
export type { VersionedIndexProvider } from './plugin.js'
|
||||
export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js'
|
||||
// The named, synchronous, O(1) health-report contract (the read gate's ONLY
|
||||
// source of truth for "can I serve right now") — see HealthReport's
|
||||
// derivation laws in plugin.ts.
|
||||
export type { HealthReport, LedgerInvariantResult, InvariantSource } from './plugin.js'
|
||||
// Optional provider self-report of outstanding background maintenance work
|
||||
// (compaction, deferred writes, etc.) — the payload type for
|
||||
// brain.maintenanceDebt(). See the measure-only-what-you-track contract on
|
||||
|
|
@ -390,10 +383,7 @@ import type {
|
|||
HNSWVerb,
|
||||
HNSWConfig,
|
||||
StorageAdapter,
|
||||
DerivedFamilyDeclaration,
|
||||
// The canonical count ledger a storage adapter maintains (counted + ALL-visibility
|
||||
// scalars per family, the coverage-ledger denominators) — see StorageAdapter.getCanonicalCounts.
|
||||
CanonicalCounts
|
||||
DerivedFamilyDeclaration
|
||||
} from './coreTypes.js'
|
||||
|
||||
// Export vector index implementation (the JS HNSW path)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
*
|
||||
* @example Enable integrations (recommended)
|
||||
* ```typescript
|
||||
* import { Brainy } from '@soulcraftlabs/brainy'
|
||||
* import { Brainy } from '@soulcraft/brainy'
|
||||
*
|
||||
* const brain = new Brainy({ integrations: true })
|
||||
* await brain.init()
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ The `BrainyMCPService` has been refactored to separate the core functionality fr
|
|||
### In Any Environment (Browser, Node.js, Server)
|
||||
|
||||
```typescript
|
||||
import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy'
|
||||
|
||||
// Create a Brainy instance
|
||||
const brainyData = new Brainy()
|
||||
|
|
@ -81,7 +81,7 @@ const toolResponse = await toolset.handleRequest({
|
|||
### In Browser Environment (Core Functionality Only)
|
||||
|
||||
```typescript
|
||||
import { Brainy, BrainyMCPService } from '@soulcraftlabs/brainy'
|
||||
import { Brainy, BrainyMCPService } from '@soulcraft/brainy'
|
||||
|
||||
// Create a Brainy instance
|
||||
const brainyData = new Brainy()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* 🧠 BRAINY EMBEDDED PATTERNS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: 2025-09-29T10:10:00-07:00
|
||||
* Generated: 2026-07-02T21:43:26.976Z
|
||||
* Patterns: 220
|
||||
* Coverage: 94-98% of all queries
|
||||
*
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: 2026-06-29T10:04:19-07:00
|
||||
* Generated: 2026-02-09T16:59:48.867Z
|
||||
* Noun Types: 42
|
||||
* Verb Types: 127
|
||||
*
|
||||
|
|
@ -19,7 +19,7 @@ export const TYPE_METADATA = {
|
|||
verbTypes: 127,
|
||||
totalTypes: 169,
|
||||
embeddingDimensions: 384,
|
||||
generatedAt: "2026-06-29T10:04:19-07:00",
|
||||
generatedAt: "2026-02-09T16:59:48.867Z",
|
||||
sizeBytes: {
|
||||
embeddings: 259584,
|
||||
base64: 346112
|
||||
|
|
|
|||
117
src/plugin.ts
117
src/plugin.ts
|
|
@ -9,7 +9,6 @@
|
|||
* registered manually via `brain.use()` — there is no implicit detection.
|
||||
*/
|
||||
|
||||
import { prodLog } from './utils/logger.js'
|
||||
import type {
|
||||
StorageAdapter,
|
||||
Vector,
|
||||
|
|
@ -22,7 +21,7 @@ import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js'
|
|||
|
||||
// Re-export the provider contracts that already live closer to their
|
||||
// implementations so a plugin author (Cor) can import the *entire*
|
||||
// provider surface from one stable entrypoint: `@soulcraftlabs/brainy/plugin`.
|
||||
// provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`.
|
||||
export type { ColumnStoreProvider } from './indexes/columnStore/types.js'
|
||||
export type {
|
||||
AggregationProvider,
|
||||
|
|
@ -41,7 +40,7 @@ export interface BrainyPlugin {
|
|||
name: string
|
||||
|
||||
/**
|
||||
* Optional semver range of `@soulcraftlabs/brainy` this plugin supports
|
||||
* Optional semver range of `@soulcraft/brainy` this plugin supports
|
||||
* (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is
|
||||
* OUTSIDE the range, `init()` THROWS rather than silently falling back to the
|
||||
* default JS engine. This is the version-coupling guard for the native
|
||||
|
|
@ -172,66 +171,6 @@ export interface ProviderInvariantReport {
|
|||
durationMs: number
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Where a {@link LedgerInvariantResult} verdict came from:
|
||||
* - `'ledger'` — decided from an exact, durable ledger (a real count, not a sample).
|
||||
* - `'deep'` — decided by a full/expensive scan (the `validateInvariants()` diagnostic path only).
|
||||
* - `'unledgered'` — this family has no ledger yet; the verdict is UNKNOWN, never healthy and never broken.
|
||||
*/
|
||||
export type InvariantSource = 'ledger' | 'deep' | 'unledgered'
|
||||
|
||||
/**
|
||||
* @description One invariant verdict inside a {@link HealthReport}. Extends
|
||||
* {@link InvariantResult} with the provenance of the verdict ({@link InvariantSource})
|
||||
* and, for a failing set-membership invariant, an exact count plus a capped sample
|
||||
* of the diverging ids — a VERDICT, never a dump. `sample` MUST be capped at 16 ids;
|
||||
* `count` is the exact number even when `sample` is truncated.
|
||||
*/
|
||||
export interface LedgerInvariantResult extends InvariantResult {
|
||||
/** Provenance of this verdict — see {@link InvariantSource}. */
|
||||
source: InvariantSource
|
||||
/** Exact count of diverging/missing items plus a capped (≤16 ids) sample. Present only on a failing set-membership invariant. */
|
||||
missing?: { count: number; sample: string[] }
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The NAMED, SYNCHRONOUS, O(1) health report a provider exposes via
|
||||
* {@link MetadataIndexProvider.healthReport} / {@link GraphIndexProvider.healthReport} /
|
||||
* {@link VectorIndexProvider.healthReport}. This is the read gate's ONLY source of
|
||||
* truth for "can I serve right now" — it replaces sampled self-probes and the
|
||||
* unnamed `isReady()` latch with an exact, ledger-derived verdict.
|
||||
*
|
||||
* Derivation laws (a provider MUST honor these; brainy's read gate assumes them):
|
||||
* - `healthy` = every VERIFIED invariant in {@link invariants} holds. An invariant
|
||||
* whose family is named in {@link unledgered} is NEVER counted toward `healthy`
|
||||
* either way — it is unknown, not passing.
|
||||
* - `serving` = no verified invariant in {@link invariants} FAILS with `heal: 'rebuild'`.
|
||||
* A failure with `heal: 'repair'` or `heal: 'none'` is degraded-but-serving —
|
||||
* `serving` stays `true`. Only a `'rebuild'`-grade failure makes `serving` `false`.
|
||||
* - `validateInvariants()` remains the async DEEP diagnostic (full scans allowed,
|
||||
* `source: 'deep'` results); `healthReport()` MUST be synchronous, O(1) from
|
||||
* exact ledgers/counters, and MUST NOT throw for a well-formed provider — a
|
||||
* provider that cannot produce a safe verdict reports it as a failing invariant,
|
||||
* it does not throw (a throw is read by the gate as a CONTRACT VIOLATION, not as
|
||||
* "unknown").
|
||||
*/
|
||||
export interface HealthReport extends ProviderInvariantReport {
|
||||
/**
|
||||
* Monotonic per provider: bumps on every ledger mutation and every rebuild
|
||||
* boundary. Consumers (the read gate's narration dedup, external callers) may
|
||||
* cache a verdict per generation.
|
||||
*/
|
||||
generation: number
|
||||
/** Each checked invariant, with provenance — see {@link LedgerInvariantResult}. */
|
||||
invariants: LedgerInvariantResult[]
|
||||
/**
|
||||
* Families with no ledger yet. NAMED here so an operator can see what is not
|
||||
* yet tracked — NEVER counted as healthy (they are not verified) and NEVER
|
||||
* counted as broken (there is nothing to fail).
|
||||
*/
|
||||
unledgered: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* @description A provider's self-report of its own outstanding background
|
||||
* maintenance work (compaction, deferred writes, a build-new→verify→swap in
|
||||
|
|
@ -327,20 +266,6 @@ export interface MetadataIndexProvider {
|
|||
*/
|
||||
validateInvariants?(): Promise<ProviderInvariantReport>
|
||||
|
||||
/**
|
||||
* @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this
|
||||
* provider derives from its own exact ledgers — see {@link HealthReport} for
|
||||
* the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a
|
||||
* well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never
|
||||
* as "unknown"). When present, brainy's read gate (`assessProviderHealth()`)
|
||||
* reads THIS instead of `isReady()` / size heuristics: `serving` decides
|
||||
* whether reads may proceed; a `false` refuses the read loudly rather than
|
||||
* triggering a rebuild. Absent → the gate falls back to `isReady?()` / the
|
||||
* size heuristic (this train's JS built-in providers stay on that interim
|
||||
* path).
|
||||
*/
|
||||
healthReport?(): HealthReport
|
||||
|
||||
/**
|
||||
* @description OPTIONAL. A native provider returns true from the moment its
|
||||
* `init()` detects a large epoch-drift until its background
|
||||
|
|
@ -537,20 +462,6 @@ export interface GraphIndexProvider {
|
|||
*/
|
||||
validateInvariants?(): Promise<ProviderInvariantReport>
|
||||
|
||||
/**
|
||||
* @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this
|
||||
* provider derives from its own exact ledgers — see {@link HealthReport} for
|
||||
* the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a
|
||||
* well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never
|
||||
* as "unknown"). When present, brainy's read gate (`assessProviderHealth()`)
|
||||
* reads THIS instead of `isReady()` / size heuristics: `serving` decides
|
||||
* whether reads may proceed; a `false` refuses the read loudly rather than
|
||||
* triggering a rebuild. Absent → the gate falls back to `isReady?()` / the
|
||||
* size heuristic (this train's JS built-in providers stay on that interim
|
||||
* path).
|
||||
*/
|
||||
healthReport?(): HealthReport
|
||||
|
||||
/**
|
||||
* @description OPTIONAL eager cold-load. Called once during brain init — AFTER
|
||||
* the metadata provider's `init()` (so the id-mapper is hydrated; a native int
|
||||
|
|
@ -1314,20 +1225,6 @@ export interface VectorIndexProvider {
|
|||
*/
|
||||
validateInvariants?(): Promise<ProviderInvariantReport>
|
||||
|
||||
/**
|
||||
* @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this
|
||||
* provider derives from its own exact ledgers — see {@link HealthReport} for
|
||||
* the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a
|
||||
* well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never
|
||||
* as "unknown"). When present, brainy's read gate (`assessProviderHealth()`)
|
||||
* reads THIS instead of `isReady()` / size heuristics: `serving` decides
|
||||
* whether reads may proceed; a `false` refuses the read loudly rather than
|
||||
* triggering a rebuild. Absent → the gate falls back to `isReady?()` / the
|
||||
* size heuristic (this train's JS built-in providers stay on that interim
|
||||
* path).
|
||||
*/
|
||||
healthReport?(): HealthReport
|
||||
|
||||
/**
|
||||
* @description OPTIONAL. A native provider returns true from the moment its
|
||||
* `init()` detects a large epoch-drift until its background
|
||||
|
|
@ -1575,13 +1472,9 @@ export class PluginRegistry {
|
|||
this.activated.add(name)
|
||||
activated.push(name)
|
||||
} else {
|
||||
// Documented graceful decline (activate() → false). Surface it on the
|
||||
// ALWAYS-ON channel: `silent: true` patches console, and a declined
|
||||
// accelerator warned into a patched console is a silent degrade to the
|
||||
// default engines — the exact invisible-fallback class this registry
|
||||
// exists to prevent (a production storm ran the WASM engine for 90s
|
||||
// behind one suppressed warn).
|
||||
prodLog.warn(
|
||||
// Documented graceful decline (activate() → false). Surface it loudly so
|
||||
// a silent degrade to the default engine never goes unnoticed.
|
||||
console.warn(
|
||||
`[brainy] Plugin "${name}" declined activation (activate() returned false); ` +
|
||||
`the default engine is in use for its providers.`
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ import {
|
|||
HNSWNounWithMetadata,
|
||||
HNSWVerbWithMetadata,
|
||||
NounMetadata,
|
||||
VerbMetadata,
|
||||
CanonicalCounts,
|
||||
VerbMetadata
|
||||
} from '../../coreTypes.js'
|
||||
import { StorageBatchConfig } from '../baseStorage.js'
|
||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
||||
|
|
@ -1029,55 +1028,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
// Universal count tracking - O(1) operations
|
||||
protected totalNounCount = 0
|
||||
protected totalVerbCount = 0
|
||||
/**
|
||||
* The ALL-visibility canonical scalars — every noun / verb the unfiltered
|
||||
* storage walk yields, system and internal tiers included. These are the
|
||||
* denominators a derived-index provider's coverage ledger subtracts from
|
||||
* (`posted === all` is the whole-store coverage verdict); the user-facing
|
||||
* `totalNounCount` / `totalVerbCount` skip hidden tiers by design and can
|
||||
* never serve as a ledger denominator. Maintained on the write path
|
||||
* (every new record +1, every proven delete −1), persisted beside the
|
||||
* counted scalars, recomputed by the sanctioned recount. Never clamped.
|
||||
*/
|
||||
protected totalNounCountAll = 0
|
||||
protected totalVerbCountAll = 0
|
||||
/**
|
||||
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
||||
* vector-side mirror of `totalNounCountAll` and the coverage denominator a
|
||||
* vector index's node-count ledger is measured against. A deferred-embed
|
||||
* noun (`add({ deferEmbedding: true })`) counts only once its vector
|
||||
* LANDS (the `system:embed-landing` commit) — its canonical record exists
|
||||
* (already counted in `totalNounCountAll`) with an empty vector until
|
||||
* then. Maintained on the write path (a fresh insert whose vector is
|
||||
* non-empty +1, a deferred embed's landing +1, a PROVEN delete of a
|
||||
* vectored noun −1), persisted beside the other ALL scalars, recomputed by
|
||||
* the sanctioned recount. Shares `allCountsSuspect` — no separate flag.
|
||||
*/
|
||||
protected totalVectoredNounCount = 0
|
||||
/**
|
||||
* `true` when a delete could not prove whether the record existed (no
|
||||
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
||||
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
||||
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
||||
* must treat a suspect scalar as unverified, never as exact. Also covers
|
||||
* `totalVectoredNounCount` — a delete whose vector-presence fact was
|
||||
* unknowable marks this SAME flag rather than minting a second one.
|
||||
*/
|
||||
protected allCountsSuspect = false
|
||||
/** One narration per session for the suspect transition (never per delete). */
|
||||
private allCountsSuspectNarrated = false
|
||||
/**
|
||||
* Which rule produced the ALL scalars currently in memory. `'identity-record'`
|
||||
* means one counted entity per metadata content leg — the honest rule: a
|
||||
* bare id-directory (a ghost or scar left by a partial-delete defect, no
|
||||
* content leg) counts zero. Set by the one-time derivation and by the
|
||||
* sanctioned recount, alongside `allCountsSuspect = false`; left `undefined`
|
||||
* when a loaded counts.json carries the ALL scalars but no stamp — the
|
||||
* legacy container-rule derivation, which forces `allCountsSuspect = true`
|
||||
* at load instead. A filesystem concern: `MemoryStorage` has no counts.json
|
||||
* and never sets this.
|
||||
*/
|
||||
protected allCountsDerivedBy?: 'identity-record'
|
||||
protected entityCounts: Map<string, number> = new Map() // type -> count
|
||||
protected verbCounts: Map<string, number> = new Map() // verb type -> count
|
||||
protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
|
||||
|
|
@ -1106,82 +1056,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
return this.totalVerbCount
|
||||
}
|
||||
|
||||
/**
|
||||
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
||||
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||
* against (the coverage-ledger denominator for derived-index providers);
|
||||
* `vectors.all` is the vectored-noun scalar — the coverage denominator for
|
||||
* a vector index's node-count ledger specifically.
|
||||
* `suspect` is `true` when an unprovable delete has made `all` (any
|
||||
* family, including `vectors`) unverified since the last sanctioned
|
||||
* recount (`rebuildTypeCounts`).
|
||||
* @returns All scalars per family plus the suspect flag.
|
||||
*/
|
||||
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
||||
return {
|
||||
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
||||
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
||||
vectors: { all: this.totalVectoredNounCount },
|
||||
suspect: this.allCountsSuspect
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark the ALL scalars unverified after a delete that could not prove the
|
||||
* record existed. Narrates ONCE per session (the flag is what persists);
|
||||
* the sanctioned recount clears it.
|
||||
* @param family - Which family's delete was unprovable.
|
||||
* @param id - The id whose existence could not be established.
|
||||
*/
|
||||
protected markAllCountsSuspect(family: 'noun' | 'verb' | 'noun-vector', id: string): void {
|
||||
this.allCountsSuspect = true
|
||||
if (!this.allCountsSuspectNarrated) {
|
||||
this.allCountsSuspectNarrated = true
|
||||
console.warn(
|
||||
`[Storage] ${family} delete of ${id} could not prove the record existed ` +
|
||||
`(no canonical read, no prior record) — the ALL-visibility count ledger is ` +
|
||||
`SUSPECT until brain.repairIndex() recounts. Further unprovable deletes ` +
|
||||
`this session are counted silently under the same flag.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorLanded}):
|
||||
* record a deferred-embed noun's FIRST real vector landing. The caller
|
||||
* (the deferred-embed worker) proves this is a genuine landing — not a
|
||||
* re-embed of an already-vectored row — by observing its own pre-embed
|
||||
* read's vector was empty, at no added storage cost.
|
||||
* @param id - The noun whose vector just landed (retained for a future
|
||||
* narration seam; the count itself needs no id-keyed state).
|
||||
*/
|
||||
async noteVectorLanded(id: string): Promise<void> {
|
||||
void id
|
||||
this.totalVectoredNounCount++
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}):
|
||||
* the mirror of {@link noteVectorLanded} — record a noun's vector was just
|
||||
* REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a
|
||||
* caller that (incorrectly) fires this for a noun already unvectored would
|
||||
* otherwise drive the ledger negative — clamped defensively, matching the
|
||||
* delete path's `if (this.totalVectoredNounCount > 0)` guard.
|
||||
* @param id - The noun whose vector was just removed (retained for a
|
||||
* future narration seam; the count itself needs no id-keyed state).
|
||||
*/
|
||||
async noteVectorUnlanded(id: string): Promise<void> {
|
||||
void id
|
||||
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment count for entity type - O(1) operation.
|
||||
* Concurrency is handled by the process-global mutex
|
||||
|
|
|
|||
|
|
@ -14,13 +14,10 @@ import {
|
|||
StorageBatchConfig,
|
||||
SYSTEM_DIR,
|
||||
STATISTICS_KEY,
|
||||
WriterLockInfo,
|
||||
WriterCloseRecord
|
||||
WriterLockInfo
|
||||
} from '../baseStorage.js'
|
||||
import { getBrainyVersion } from '../../utils/index.js'
|
||||
import { isAbsentError } from '../../utils/errorClassification.js'
|
||||
import { prodLog } from '../../utils/logger.js'
|
||||
import { isZeroNormVector } from '../../utils/distance.js'
|
||||
import {
|
||||
TornRecordError,
|
||||
isUnparseablePayloadError,
|
||||
|
|
@ -100,30 +97,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// timer rewrites the lock every 10s so stale-lock detection can tell a dead
|
||||
// writer from a slow one. The constant name matches the file path used.
|
||||
private static readonly WRITER_LOCK_FILE = '_writer.lock'
|
||||
/**
|
||||
* The clean-close record at `locks/_writer.close` (see
|
||||
* {@link WriterCloseRecord}). Written when the lock is released, consumed by
|
||||
* the next claim, so an open can distinguish "the previous writer left" from
|
||||
* "the previous writer died" without inferring either from a pid.
|
||||
*/
|
||||
private static readonly WRITER_CLOSE_FILE = '_writer.close'
|
||||
/**
|
||||
* How often the lock file's `lastHeartbeat` is rewritten.
|
||||
*
|
||||
* THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness
|
||||
* is decided by PID LIVENESS alone (see isWriterLockStale) and the fence
|
||||
* compares pid + hostname — no decision anywhere reads this timestamp. It
|
||||
* exists so an operator inspecting a lock file, or reading the
|
||||
* BRAINY_WRITER_LOCKED error, can judge liveness themselves.
|
||||
*
|
||||
* At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1
|
||||
* writes/s across a production process holding 21 idle brains, for a
|
||||
* human-readable timestamp nothing computes with. At 60s an operator still
|
||||
* sees a heartbeat inside the minute, at a sixth of the cost. With the
|
||||
* clean-close record now recording orderly releases explicitly, the
|
||||
* heartbeat carries even less weight than it did.
|
||||
*/
|
||||
private static readonly WRITER_HEARTBEAT_MS = 60_000
|
||||
private static readonly WRITER_HEARTBEAT_MS = 10_000
|
||||
private static readonly WRITER_STALE_THRESHOLD_MS = 60_000
|
||||
private writerLockHeartbeat?: NodeJS.Timeout
|
||||
private writerLockInfo?: WriterLockInfo
|
||||
|
|
@ -136,13 +110,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
*/
|
||||
private writerHeartbeatInFlight?: Promise<void>
|
||||
|
||||
/**
|
||||
* The in-flight background count-ledger derivation, if one was needed at
|
||||
* open. See {@link scheduleCountLedgerDerivation} — awaited only by
|
||||
* {@link whenCountLedgerSettled}, never by a read.
|
||||
*/
|
||||
private countLedgerDerivation?: Promise<void>
|
||||
|
||||
// Flush-request RPC state. The writer polls `locks/_flush_requests/` for
|
||||
// new `.req` files and emits `.ack` files in `locks/_flush_responses/` after
|
||||
// flushing. Inspectors call `requestFlushOverFilesystem` to drop a request
|
||||
|
|
@ -151,16 +118,9 @@ export class FileSystemStorage extends BaseStorage {
|
|||
private static readonly FLUSH_REQUEST_DIR = '_flush_requests'
|
||||
private static readonly FLUSH_RESPONSE_DIR = '_flush_responses'
|
||||
private static readonly FLUSH_WATCH_INTERVAL_MS = 500
|
||||
/**
|
||||
* The safety sweep behind the fs.watch: catches events an exotic filesystem
|
||||
* dropped, and runs the stale-request GC. See startFlushRequestWatcher.
|
||||
*/
|
||||
private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000
|
||||
private static readonly FLUSH_POLL_INTERVAL_MS = 100
|
||||
private static readonly FLUSH_REQUEST_TTL_MS = 60_000
|
||||
private flushWatcherInterval?: NodeJS.Timeout
|
||||
/** The inotify-backed watch on the request directory, when the FS supports one. */
|
||||
private flushWatcher?: import('node:fs').FSWatcher
|
||||
private flushWatcherInFlight = false
|
||||
private flushWatcherOnRequest?: () => Promise<void>
|
||||
|
||||
|
|
@ -279,54 +239,38 @@ export class FileSystemStorage extends BaseStorage {
|
|||
|
||||
// Finish any restore interrupted by a crash (resume the staged swap, or
|
||||
// discard an uncommitted staging area) BEFORE counts/derived state load,
|
||||
// so the rest of startup sees the completed store. ORDER-DEPENDENT:
|
||||
// `swapStagedRestoreIn()` reads `fs.readdir(rootDir)` and then
|
||||
// removes/renames rootDir's own TOP-LEVEL entries to place the staged
|
||||
// copy — racing that against the directory-creation batch below (which
|
||||
// also touches rootDir's children) could see a half-created directory
|
||||
// mid-swap or a mkdir racing a concurrent rm/rename on the same path.
|
||||
// Stays strictly sequential, never folded into the OPEN-PATH batch.
|
||||
// so the rest of startup sees the completed store.
|
||||
await this.completeInterruptedRestore()
|
||||
|
||||
// OPEN-PATH FIX: the remaining bootstrap directories are mutually
|
||||
// independent — each is its own subtree under rootDir, and
|
||||
// `fs.mkdir(dir, { recursive: true })` creates every intermediate
|
||||
// segment of ITS OWN path in one call, so it never depends on any
|
||||
// sibling here existing first. Nothing between here and
|
||||
// `initializeCounts()` reads any of them, so batching collapses what
|
||||
// was up to 8 sequential mkdir round-trips (each a real syscall+await)
|
||||
// into one wave — this is what serialized an N-writer restart storm on
|
||||
// filesystem I/O it never structurally needed. `initializeCounts()`
|
||||
// right after DOES depend on `systemDir` (which the batch creates), so
|
||||
// it stays outside, awaited only once every directory has landed.
|
||||
await Promise.all([
|
||||
// Create the nouns directory if it doesn't exist
|
||||
this.ensureDirectoryExists(this.nounsDir),
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
|
||||
// Create the verbs directory if it doesn't exist
|
||||
this.ensureDirectoryExists(this.verbsDir),
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
|
||||
// Create the metadata directory if it doesn't exist
|
||||
this.ensureDirectoryExists(this.metadataDir),
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
// Create the noun metadata directory if it doesn't exist
|
||||
this.ensureDirectoryExists(this.nounMetadataDir),
|
||||
await this.ensureDirectoryExists(this.nounMetadataDir)
|
||||
|
||||
// Create the verb metadata directory if it doesn't exist
|
||||
this.ensureDirectoryExists(this.verbMetadataDir),
|
||||
await this.ensureDirectoryExists(this.verbMetadataDir)
|
||||
|
||||
// Create both directories for backward compatibility
|
||||
this.ensureDirectoryExists(this.systemDir),
|
||||
// Only create legacy directory if it exists (don't create new legacy
|
||||
// dirs) — a read-then-maybe-write, but on its own subtree, so it's
|
||||
// still independent of every other entry in this batch.
|
||||
(async () => {
|
||||
await this.ensureDirectoryExists(this.systemDir)
|
||||
// Only create legacy directory if it exists (don't create new legacy dirs)
|
||||
if (await this.directoryExists(this.indexDir)) {
|
||||
await this.ensureDirectoryExists(this.indexDir)
|
||||
}
|
||||
})(),
|
||||
// Create the locks directory if it doesn't exist
|
||||
this.ensureDirectoryExists(this.lockDir),
|
||||
// Create the binary blobs directory if it doesn't exist
|
||||
this.ensureDirectoryExists(this.blobsDir)
|
||||
])
|
||||
|
||||
// Initialize count management — depends on systemDir, created above.
|
||||
// Create the locks directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.lockDir)
|
||||
|
||||
// Create the binary blobs directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.blobsDir)
|
||||
|
||||
// Initialize count management
|
||||
this.countsFilePath = path.join(this.systemDir, 'counts.json')
|
||||
await this.initializeCounts()
|
||||
|
||||
|
|
@ -642,20 +586,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
* automatically. Returns the pruned container ids so the caller can recompute
|
||||
* counts.
|
||||
*/
|
||||
/**
|
||||
* @description Whether an id directory's file legs include the metadata
|
||||
* CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single
|
||||
* test that decides whether an `entities/<kind>/<shard>/<id>/` container is
|
||||
* a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete
|
||||
* defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune
|
||||
* and {@link scanCanonicalEntities} so the two agree by construction — one
|
||||
* counted entity per identity record, never per bare container.
|
||||
* @param legs - File names in one `entities/<kind>/<shard>/<id>/` directory.
|
||||
*/
|
||||
private hasMetadataContentLeg(legs: string[]): boolean {
|
||||
return legs.some((f) => f.startsWith('metadata.json'))
|
||||
}
|
||||
|
||||
public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
|
||||
await this.ensureInitialized()
|
||||
const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
|
||||
|
|
@ -695,7 +625,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
// A live entity has its metadata content leg. No content leg → a
|
||||
// vector-only ghost or an empty scar → prune the whole container.
|
||||
if (this.hasMetadataContentLeg(legs)) continue
|
||||
if (legs.some((f) => f.startsWith('metadata.json'))) continue
|
||||
await fs.promises.rm(idAbs, { recursive: true, force: true })
|
||||
pruned[kind].push(entry.name)
|
||||
console.warn(
|
||||
|
|
@ -709,30 +639,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
return pruned
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The IMMEDIATE child directory names under a prefix — ONE
|
||||
* `readdir`, no recursion, no file paths. See the seam's JSDoc
|
||||
* (`src/db/types.ts`) for what this replaced: discovering the generations on
|
||||
* disk walked the entire generation log on every open, reading out every
|
||||
* file in every generation, to learn the set of integers the top-level
|
||||
* directory names already spell.
|
||||
* @param prefix - Storage-root-relative directory prefix.
|
||||
* @returns The child directory names (not paths); empty when the prefix does
|
||||
* not exist.
|
||||
*/
|
||||
public override async listRawPrefixes(prefix: string): Promise<string[]> {
|
||||
await this.ensureInitialized()
|
||||
const fullPath = path.join(this.rootDir, prefix)
|
||||
try {
|
||||
const entries = await fs.promises.readdir(fullPath, { withFileTypes: true })
|
||||
return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory())
|
||||
.map((e: { name: string }) => e.name)
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') return []
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Primitive operation: List objects under path prefix
|
||||
* All metadata operations use this internally via base class routing
|
||||
|
|
@ -1943,41 +1849,18 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see
|
||||
// WriterCloseRecord). A lock file whose release was RECORDED is
|
||||
// bookkeeping left by an orderly shutdown, not evidence of anything —
|
||||
// and that is true whether the previous holder was another process or
|
||||
// an earlier instance in THIS one. A production restart reported
|
||||
// "Re-acquiring writer lock ... this is a bug" immediately after a clean
|
||||
// close, sending an operator hunting for a leak that did not exist.
|
||||
const closeRecord = existing ? await this.readWriterCloseRecord() : null
|
||||
const releasedCleanly =
|
||||
existing !== null &&
|
||||
closeRecord !== null &&
|
||||
this.closeRecordVouchesFor(closeRecord, existing)
|
||||
|
||||
if (existing) {
|
||||
// Same-process re-open: a second Brainy instance in this Node process
|
||||
// (e.g. test "simulate server restart" patterns, or a consumer that
|
||||
// explicitly re-instantiates without closing first). This isn't the
|
||||
// dangerous cross-process case the lock exists to prevent — the two
|
||||
// instances share a memory space and can't silently diverge from each
|
||||
// other beyond what their callers already see. Warn and take over —
|
||||
// unless the record proves the previous instance already let go, in
|
||||
// which case there is nothing to warn about.
|
||||
// other beyond what their callers already see. Warn and take over.
|
||||
if (existing.pid === myPid && existing.hostname === hostname && !options?.force) {
|
||||
if (releasedCleanly) {
|
||||
console.warn(
|
||||
`[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` +
|
||||
`instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` +
|
||||
`${closeRecord!.closedAt} but could not remove the file. Nothing to recover.`
|
||||
)
|
||||
} else {
|
||||
console.warn(
|
||||
`[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
|
||||
`If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
|
||||
)
|
||||
}
|
||||
const info: WriterLockInfo = {
|
||||
pid: myPid,
|
||||
hostname,
|
||||
|
|
@ -1987,18 +1870,11 @@ export class FileSystemStorage extends BaseStorage {
|
|||
rootDir: this.rootDir
|
||||
}
|
||||
await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
|
||||
await this.clearWriterCloseRecord()
|
||||
this.installWriterLock(info)
|
||||
return info
|
||||
}
|
||||
|
||||
// A cleanly-released lock is stale by RECORD, not by inference. Only
|
||||
// when no record vouches for this lock do we fall back to pid
|
||||
// liveness, and then we say THAT honestly too: an unrecorded lock
|
||||
// means the writer did not complete its close, so the store was not
|
||||
// closed cleanly and this open pays recovery.
|
||||
const stale =
|
||||
releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing)))
|
||||
const stale = !options?.force && (await this.isWriterLockStale(existing))
|
||||
if (!options?.force && !stale) {
|
||||
// Consumer-facing error contract: callers detect this case via
|
||||
// err.code and read the holder's details from err.lockInfo.
|
||||
|
|
@ -2009,16 +1885,8 @@ export class FileSystemStorage extends BaseStorage {
|
|||
options?.force
|
||||
? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
|
||||
`(was held by PID ${existing.pid} on ${existing.hostname}).`
|
||||
: releasedCleanly
|
||||
? `[brainy] Clearing the leftover writer lock for ${this.rootDir} — ` +
|
||||
`PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` +
|
||||
`${closeRecord!.closedAt} but could not remove the file. ` +
|
||||
`Nothing to recover.`
|
||||
: `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
|
||||
`(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` +
|
||||
`clean-close record — that writer did not finish closing, so this ` +
|
||||
`store was not closed cleanly; open will run crash recovery and ` +
|
||||
`report its wall).`
|
||||
`(PID ${existing.pid} on ${existing.hostname} appears dead).`
|
||||
)
|
||||
// Takeover: verify the file still holds the lock we judged (a live
|
||||
// successor may have claimed meanwhile), then remove it and fall
|
||||
|
|
@ -2072,12 +1940,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
await fs.promises.unlink(claimTmp).catch(() => {})
|
||||
}
|
||||
|
||||
// CONSUME the previous writer's clean-close record. It described the
|
||||
// lock generation that just ended; leaving it in place would let it
|
||||
// vouch for OUR lock if this process later dies without closing —
|
||||
// turning a real crash into a "closed cleanly" verdict. One unlink.
|
||||
await this.clearWriterCloseRecord()
|
||||
|
||||
this.installWriterLock(info)
|
||||
return info
|
||||
}
|
||||
|
|
@ -2201,27 +2063,13 @@ export class FileSystemStorage extends BaseStorage {
|
|||
return
|
||||
}
|
||||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||||
const released = this.writerLockInfo
|
||||
try {
|
||||
// Only delete if we still own it — avoid clobbering a successor that
|
||||
// claimed the lock via force-override.
|
||||
const current = await this.readWriterLock()
|
||||
const ours =
|
||||
current === null ||
|
||||
(current.pid === released.pid && current.hostname === released.hostname)
|
||||
if (current && ours) {
|
||||
if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) {
|
||||
await fs.promises.unlink(lockFile)
|
||||
}
|
||||
// THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this
|
||||
// instance gives up a lock nobody else has taken — the unlink above
|
||||
// having succeeded OR the file already being gone. The next open reads
|
||||
// it instead of guessing from pid liveness: a recorded release is an
|
||||
// orderly shutdown, an absent record is a writer that never finished
|
||||
// closing. Not written when a successor holds the lock: our release is
|
||||
// then a no-op and a record would slander their live lock.
|
||||
if (ours) {
|
||||
await this.writeWriterCloseRecord(released)
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
console.warn('[brainy] Failed to release writer lock file:', err)
|
||||
|
|
@ -2231,97 +2079,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the clean-close record at `locks/_writer.close`, or
|
||||
* `null` when it is absent or unparseable. A torn record is treated as
|
||||
* absent — the conservative direction, since an unreadable record can
|
||||
* vouch for nothing.
|
||||
* @returns The record, or null.
|
||||
*/
|
||||
public async readWriterCloseRecord(): Promise<WriterCloseRecord | null> {
|
||||
await this.ensureInitialized()
|
||||
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||||
try {
|
||||
const raw = await fs.promises.readFile(recordFile, 'utf-8')
|
||||
const parsed = JSON.parse(raw) as WriterCloseRecord
|
||||
if (
|
||||
typeof parsed?.pid !== 'number' ||
|
||||
typeof parsed?.hostname !== 'string' ||
|
||||
typeof parsed?.startedAt !== 'string' ||
|
||||
typeof parsed?.closedAt !== 'string'
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return parsed
|
||||
} catch (err: any) {
|
||||
if (err.code === 'ENOENT') return null
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Whether a clean-close record describes the very lock
|
||||
* generation `lock` represents. The match is pid + hostname + `startedAt`:
|
||||
* `startedAt` is the lock generation's identity, so a record can never
|
||||
* vouch for a LATER lock taken by the same pid on the same host (the
|
||||
* same-process re-open path mints a fresh `startedAt`).
|
||||
* @param record - The clean-close record read from disk.
|
||||
* @param lock - The lock file's contents.
|
||||
*/
|
||||
private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean {
|
||||
return (
|
||||
record.pid === lock.pid &&
|
||||
record.hostname === lock.hostname &&
|
||||
record.startedAt === lock.startedAt
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Write the clean-close record for a lock this instance just
|
||||
* released. Atomic (temp + rename) so a concurrent opener never reads half
|
||||
* a record. A failure here costs the next open nothing but the honest
|
||||
* fallback (pid liveness), so it warns rather than failing the close.
|
||||
* @param released - The lock info this instance held.
|
||||
*/
|
||||
private async writeWriterCloseRecord(released: WriterLockInfo): Promise<void> {
|
||||
const record: WriterCloseRecord = {
|
||||
pid: released.pid,
|
||||
hostname: released.hostname,
|
||||
startedAt: released.startedAt,
|
||||
closedAt: new Date().toISOString(),
|
||||
version: released.version
|
||||
}
|
||||
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||||
try {
|
||||
await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2))
|
||||
} catch (err) {
|
||||
// ENOENT = the lock directory is gone, i.e. the whole store was removed
|
||||
// under us. There is no next open to inform.
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return
|
||||
console.warn(
|
||||
`[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` +
|
||||
`the next open will fall back to pid liveness and may report this orderly ` +
|
||||
`shutdown as a crash:`,
|
||||
err
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Remove the clean-close record. Called by every successful
|
||||
* lock claim so a record never outlives the lock generation it describes.
|
||||
*/
|
||||
private async clearWriterCloseRecord(): Promise<void> {
|
||||
const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE)
|
||||
try {
|
||||
await fs.promises.unlink(recordFile)
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
console.warn('[brainy] Failed to clear the writer clean-close record:', err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override async readWriterLock(): Promise<WriterLockInfo | null> {
|
||||
await this.ensureInitialized()
|
||||
const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
|
||||
|
|
@ -2408,115 +2165,36 @@ export class FileSystemStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Start watching for cross-process flush requests. Called by Brainy.init()
|
||||
* in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers
|
||||
* the supplied callback (`brain.flush()`), after which an `.ack` is written
|
||||
* to `locks/_flush_responses/` with the same request ID. Stale `.req` files
|
||||
* (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep.
|
||||
*
|
||||
* THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request
|
||||
* directory every 500 ms, per brain, for the entire life of every writer —
|
||||
* armed on every non-reader brain whether or not any inspector process
|
||||
* existed. MEASURED on a production process holding 21 brains: 42 directory
|
||||
* reads per second on a completely idle service, plus a stale-request GC
|
||||
* pass on every one of them. The engine does no periodic work without a
|
||||
* cause, and a request that has not been made is not a cause.
|
||||
*
|
||||
* `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is
|
||||
* seen SOONER than the old poll saw it. Two honest concessions ride with it:
|
||||
* - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because
|
||||
* `fs.watch` can miss events on network and fuse filesystems and because
|
||||
* the stale-request GC needs some tick of its own. At 30s that is 0.7
|
||||
* reads/s across 21 brains where the poll cost 42.
|
||||
* - a filesystem that cannot watch at all falls back to the ORIGINAL
|
||||
* 500 ms poll, narrated once, because correctness outranks idle cost:
|
||||
* an inspector whose request is never seen waits forever.
|
||||
* in writer mode. Polls `locks/_flush_requests/` every
|
||||
* FLUSH_WATCH_INTERVAL_MS — each new `.req` file triggers the supplied
|
||||
* callback (`brain.flush()`), after which an `.ack` is written to
|
||||
* `locks/_flush_responses/` with the same request ID. Stale `.req` files
|
||||
* (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick.
|
||||
*/
|
||||
public override startFlushRequestWatcher(onRequest: () => Promise<void>): void {
|
||||
// Already watching — or already ARMING. The arm is asynchronous (the
|
||||
// request directory is created before it can be watched), so neither the
|
||||
// watcher nor the interval exists yet during that window; the callback is
|
||||
// the flag that covers it. Without this a second call in the window would
|
||||
// leave two watchers and two sweeps running for the life of the store.
|
||||
if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return
|
||||
if (this.flushWatcherInterval) return // already watching
|
||||
this.flushWatcherOnRequest = onRequest
|
||||
|
||||
const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR)
|
||||
const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR)
|
||||
|
||||
const sweep = (): void => {
|
||||
if (this.flushWatcherInFlight) return // skip overlapping sweep
|
||||
// Ensure both dirs exist up front so the first .req drop doesn't race with mkdir.
|
||||
this.ensureDirectoryExists(reqDir).catch(() => {})
|
||||
this.ensureDirectoryExists(ackDir).catch(() => {})
|
||||
|
||||
this.flushWatcherInterval = setInterval(() => {
|
||||
if (this.flushWatcherInFlight) return // skip overlapping tick
|
||||
this.flushWatcherInFlight = true
|
||||
this.processFlushRequests(reqDir, ackDir).finally(() => {
|
||||
this.flushWatcherInFlight = false
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure both dirs exist up front so the first .req drop doesn't race with
|
||||
// mkdir — and so there is a directory to watch.
|
||||
void this.ensureDirectoryExists(reqDir)
|
||||
.then(() => this.ensureDirectoryExists(ackDir))
|
||||
.then(() => {
|
||||
if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile
|
||||
try {
|
||||
const watcher = fs.watch(reqDir, () => sweep())
|
||||
this.flushWatcher = watcher
|
||||
watcher.on('error', (err: Error) => {
|
||||
// A watch that dies mid-life must not leave the door deaf.
|
||||
console.warn(
|
||||
`[brainy] Flush-request watch failed (${err.message}) — falling back to polling.`
|
||||
)
|
||||
this.flushWatcher?.close()
|
||||
this.flushWatcher = undefined
|
||||
// The SAFETY sweep must go first. It is already armed at 30s, and
|
||||
// startFlushRequestPolling() declines to arm over an existing
|
||||
// interval — so leaving it would quietly leave this store answering
|
||||
// flush requests on a 30s cadence instead of the 500ms one the door
|
||||
// promises. A degrade nobody asked for is still a degrade.
|
||||
if (this.flushWatcherInterval) {
|
||||
clearInterval(this.flushWatcherInterval)
|
||||
this.flushWatcherInterval = undefined
|
||||
}
|
||||
this.startFlushRequestPolling(sweep)
|
||||
})
|
||||
if (typeof watcher.unref === 'function') watcher.unref()
|
||||
// The safety sweep: missed events on exotic filesystems, and the
|
||||
// stale-request GC.
|
||||
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS)
|
||||
if (typeof this.flushWatcherInterval.unref === 'function') {
|
||||
this.flushWatcherInterval.unref()
|
||||
}
|
||||
// One sweep now: a request may have been dropped before the watch armed.
|
||||
sweep()
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`[brainy] Flush-request directory cannot be watched on this filesystem ` +
|
||||
`(${(err as Error).message}) — polling every ` +
|
||||
`${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.`
|
||||
)
|
||||
this.startFlushRequestPolling(sweep)
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
// The request directory could not be created; nothing to watch. A
|
||||
// cross-process flush request cannot be made either, so there is
|
||||
// nothing to miss.
|
||||
})
|
||||
}
|
||||
|
||||
/** The original 500 ms poll — the fallback when a directory cannot be watched. */
|
||||
private startFlushRequestPolling(sweep: () => void): void {
|
||||
if (this.flushWatcherInterval) return
|
||||
this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
|
||||
}, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS)
|
||||
if (typeof this.flushWatcherInterval.unref === 'function') {
|
||||
this.flushWatcherInterval.unref()
|
||||
}
|
||||
}
|
||||
|
||||
public override stopFlushRequestWatcher(): void {
|
||||
if (this.flushWatcher) {
|
||||
this.flushWatcher.close()
|
||||
this.flushWatcher = undefined
|
||||
}
|
||||
if (this.flushWatcherInterval) {
|
||||
clearInterval(this.flushWatcherInterval)
|
||||
this.flushWatcherInterval = undefined
|
||||
|
|
@ -2883,82 +2561,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.totalNounCount = counts.totalNounCount || 0
|
||||
this.totalVerbCount = counts.totalVerbCount || 0
|
||||
|
||||
// The ALL-visibility scalars (ledger denominators). A counts.json
|
||||
// written before they existed carries neither key: derive both ONCE
|
||||
// from the canonical id tree (an id-directory listing — O(ids), no
|
||||
// record reads), persist, and never scan again. Absent keys are a
|
||||
// legacy file, not a zero — a zero here would make every provider's
|
||||
// coverage ledger read "over-posted" on a populated store.
|
||||
let needsPersist = false
|
||||
if (
|
||||
typeof counts.totalNounCountAll === 'number' &&
|
||||
typeof counts.totalVerbCountAll === 'number'
|
||||
) {
|
||||
this.totalNounCountAll = counts.totalNounCountAll
|
||||
this.totalVerbCountAll = counts.totalVerbCountAll
|
||||
if (counts.allCountsDerivedBy === 'identity-record') {
|
||||
// Derived (or recounted) under the honest rule — one counted
|
||||
// entity per metadata content leg. Trust the persisted suspect
|
||||
// flag as-is; an unprovable delete since may still have set it.
|
||||
this.allCountsDerivedBy = 'identity-record'
|
||||
this.allCountsSuspect = counts.allCountsSuspect === true
|
||||
} else {
|
||||
// The ALL scalars exist but predate the identity-record stamp —
|
||||
// they were derived under the legacy rule that counted one
|
||||
// entity per id DIRECTORY, so orphaned ghost/scar containers (a
|
||||
// pre-8.3.1 partial-delete defect — see pruneOrphanedEntities())
|
||||
// were counted as entities too. O(1) field read, NEVER a walk
|
||||
// here: force suspect and name it loudly. A sanctioned recount
|
||||
// (repairIndex) restores exact denominators and clears this.
|
||||
this.allCountsDerivedBy = undefined
|
||||
this.allCountsSuspect = true
|
||||
needsPersist = true
|
||||
prodLog.narrate(
|
||||
'[FileSystemStorage] canonical count ledger was derived under the legacy ' +
|
||||
'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' +
|
||||
'container inflates it. Marked suspect, and an honest recount is scheduled to ' +
|
||||
'run in the background after this open; until it lands, do not subtract ' +
|
||||
'against these ALL scalars.'
|
||||
)
|
||||
// A suspect ledger used to stay wrong for the life of the store,
|
||||
// waiting for an operator to run repairIndex. A downstream index
|
||||
// heal took its "remaining" figure from these inflated
|
||||
// denominators and reported work that did not exist. The ledger
|
||||
// now HEALS ITSELF — in the background, because a denominator is
|
||||
// a derived scalar and no read is ever served from it.
|
||||
this.scheduleCountLedgerDerivation('legacy container-rule ledger')
|
||||
}
|
||||
} else {
|
||||
// No ALL scalars at all. There is nothing to serve in the meantime —
|
||||
// a zero would read as an empty store — so the scalars stay unknown
|
||||
// and SUSPECT until the background derivation lands. The open does
|
||||
// not wait for it: an id-tree walk is O(ids) and this file has been
|
||||
// the whole reason a 24k-id store opened in silence.
|
||||
this.allCountsSuspect = true
|
||||
this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger')
|
||||
}
|
||||
|
||||
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
||||
// counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but
|
||||
// still predate THIS key). Unlike the ALL scalars, presence cannot be
|
||||
// decided from the id-directory listing alone: a deferred-embed
|
||||
// noun's `vectors.json` EXISTS with an empty `vector: []` until its
|
||||
// embed lands, so this derivation reads every noun's `vectors.json`
|
||||
// ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost.
|
||||
if (typeof counts.totalVectoredNounCount === 'number') {
|
||||
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
||||
} else {
|
||||
// O(nouns) CONTENT reads — the most expensive derivation of the
|
||||
// three, and the one most likely to have been the silent minutes at
|
||||
// the front of a large store's open. Background, suspect until it
|
||||
// lands, same as the ALL scalars.
|
||||
this.allCountsSuspect = true
|
||||
this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger')
|
||||
}
|
||||
if (needsPersist) {
|
||||
await this.persistCounts()
|
||||
}
|
||||
|
||||
// Also populate the cache for backward compatibility
|
||||
this.countCache.set('nouns_count', {
|
||||
count: this.totalNounCount,
|
||||
|
|
@ -2982,22 +2584,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
* Initialize counts by scanning disk (only done once)
|
||||
*/
|
||||
private async initializeCountsFromDisk(): Promise<void> {
|
||||
const startedAt = Date.now()
|
||||
// THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating:
|
||||
// it derives `totalNounCount` / `totalVerbCount`, the scalars
|
||||
// `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would
|
||||
// make a populated store answer "0 entities" until the walk landed — a
|
||||
// wrong answer, not a slow one, and the serving law grades a failure by
|
||||
// whether an answer could be wrong. The ALL-visibility denominators, which
|
||||
// no read is served from, DO run in the background (see
|
||||
// scheduleCountLedgerDerivation). What this walk owes the operator instead
|
||||
// is narration: it announces itself, and reports its wall.
|
||||
prodLog.narrate(
|
||||
`[FileSystemStorage] no usable counts.json — deriving the entity counters from ` +
|
||||
`the canonical id tree now. This is O(ids) listings plus one vectors.json read ` +
|
||||
`per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` +
|
||||
`served from it. It runs once; the result is persisted.`
|
||||
)
|
||||
try {
|
||||
// Count the CANONICAL 8.0 layout (`entities/<kind>/<shard>/<id>/…`) —
|
||||
// the tree saveNoun/getNouns actually read and write. The previous scan
|
||||
|
|
@ -3010,17 +2596,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.totalNounCount = nouns.count
|
||||
const verbs = await this.scanCanonicalEntities('verbs')
|
||||
this.totalVerbCount = verbs.count
|
||||
// The id-tree scan counts every tier — it IS the ALL-visibility ledger.
|
||||
this.totalNounCountAll = nouns.count
|
||||
this.totalVerbCountAll = verbs.count
|
||||
this.allCountsSuspect = false
|
||||
this.allCountsDerivedBy = 'identity-record'
|
||||
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
|
||||
// (a deferred-embed noun's file exists but holds an empty vector until
|
||||
// its embed lands), so this is a full O(nouns) content scan — see
|
||||
// scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here,
|
||||
// alongside the rest of this from-disk recovery.
|
||||
this.totalVectoredNounCount = await this.scanVectoredNounCount()
|
||||
|
||||
// Sample some entities for the type distribution (don't read all).
|
||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||
|
|
@ -3045,11 +2620,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
|
||||
await this.persistCounts()
|
||||
prodLog.narrate(
|
||||
`[FileSystemStorage] counter derivation from the canonical id tree finished in ` +
|
||||
`${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` +
|
||||
`${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.`
|
||||
)
|
||||
} catch (error) {
|
||||
console.error('Error initializing counts from disk:', error)
|
||||
}
|
||||
|
|
@ -3057,132 +2627,11 @@ export class FileSystemStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting
|
||||
* one entity per id directory that holds the metadata CONTENT leg
|
||||
* (`metadata.json` or its `.json.gz` variant — see
|
||||
* {@link hasMetadataContentLeg}). A bare container — a ghost (a stale
|
||||
* `vectors.json` left with no metadata leg) or a scar (an empty directory),
|
||||
* both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the
|
||||
* identity record IS the population (ADR-008 G1), never the directory.
|
||||
* This is the ONE-TIME legacy derivation walk (see callers); a prior
|
||||
* version of this scan counted every id directory regardless of content,
|
||||
* over-counting any store carrying orphaned containers — see
|
||||
* `allCountsDerivedBy` for how a counts.json derived under that old rule is
|
||||
* marked suspect on load. Returns up to 100 sampled *counted* entity
|
||||
* directories (absolute paths) — nouns feed the type-distribution estimate
|
||||
* above. An absent tree (fresh store) counts zero.
|
||||
* one entity per id directory (the layout `getNounVectorPath`/`getNouns`
|
||||
* use). Returns up to 100 sampled entity directories (absolute paths) —
|
||||
* nouns feed the type-distribution estimate above. An absent tree (fresh
|
||||
* store) counts zero.
|
||||
*/
|
||||
/**
|
||||
* @description Derive the ALL-visibility count ledger honestly — one entity
|
||||
* per IDENTITY RECORD, never per id directory — IN THE BACKGROUND, once,
|
||||
* and persist the result stamped `identity-record`.
|
||||
*
|
||||
* Why background: these scalars are DENOMINATORS. No read is served from
|
||||
* them, so deriving them cannot be allowed to hold an open hostage — a
|
||||
* store with 24,898 ids spent minutes of a production restart inside walks
|
||||
* exactly like these, in silence, before serving anything. Why at all: a
|
||||
* ledger derived under the old container rule stayed wrong for the life of
|
||||
* the store, and a downstream index heal subtracted against it and reported
|
||||
* remaining work that did not exist (measured on a real store: 14,231
|
||||
* derived against 14,056 identity records — precisely the store's 25 noun
|
||||
* scar directories; verbs 72,729 against 72,679, its 50 verb scars).
|
||||
*
|
||||
* Idempotent: a second call while one is in flight joins the first.
|
||||
* @param reason - What made the ledger untrustworthy, quoted in narration.
|
||||
* @returns Nothing; observe completion with {@link whenCountLedgerSettled}.
|
||||
*/
|
||||
private scheduleCountLedgerDerivation(reason: string): void {
|
||||
if (this.countLedgerDerivation) return
|
||||
this.countLedgerDerivation = (async () => {
|
||||
const startedAt = Date.now()
|
||||
prodLog.narrate(
|
||||
`[FileSystemStorage] count-ledger derivation started in the background ` +
|
||||
`(${reason}) — counting identity records, not id directories; the open does ` +
|
||||
`not wait for it and no read is served from these scalars.`
|
||||
)
|
||||
try {
|
||||
const beforeNouns = this.totalNounCountAll
|
||||
const beforeVerbs = this.totalVerbCountAll
|
||||
const beforeVectored = this.totalVectoredNounCount
|
||||
// A walk that RACED A WRITE cannot prove its number: a row that landed
|
||||
// mid-walk may or may not have been in the shard the walk had already
|
||||
// passed. Rather than persist a figure that might be off by one and
|
||||
// stamp it "exact", the walk is repeated once on a quiet store, and if
|
||||
// the store is never quiet the ledger stays SUSPECT and says so. One
|
||||
// retry, never a spin.
|
||||
let attempt = 0
|
||||
let derived: { nouns: number; verbs: number; vectored: number } | null = null
|
||||
while (attempt < 2 && derived === null) {
|
||||
attempt++
|
||||
const activityBefore = this.ledgerActivityStamp()
|
||||
const nouns = await this.scanCanonicalEntities('nouns')
|
||||
const verbs = await this.scanCanonicalEntities('verbs')
|
||||
const vectored = await this.scanVectoredNounCount()
|
||||
if (this.ledgerActivityStamp() === activityBefore) {
|
||||
derived = { nouns: nouns.count, verbs: verbs.count, vectored }
|
||||
}
|
||||
}
|
||||
if (derived === null) {
|
||||
this.allCountsSuspect = true
|
||||
prodLog.narrate(
|
||||
`[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` +
|
||||
`after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` +
|
||||
`every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` +
|
||||
`against; brain.repairIndex() derives them under a recount barrier.`
|
||||
)
|
||||
return
|
||||
}
|
||||
this.totalNounCountAll = derived.nouns
|
||||
this.totalVerbCountAll = derived.verbs
|
||||
this.totalVectoredNounCount = derived.vectored
|
||||
this.allCountsDerivedBy = 'identity-record'
|
||||
this.allCountsSuspect = false
|
||||
await this.persistCounts()
|
||||
prodLog.narrate(
|
||||
`[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` +
|
||||
`${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` +
|
||||
(beforeNouns !== derived.nouns ||
|
||||
beforeVerbs !== derived.verbs ||
|
||||
beforeVectored !== derived.vectored
|
||||
? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` +
|
||||
`difference is ghost and scar containers the old rule counted as entities)`
|
||||
: ' (unchanged)') +
|
||||
` — persisted, stamped identity-record, no longer suspect.`
|
||||
)
|
||||
} catch (error) {
|
||||
// The ledger stays suspect and the next open retries. Loud: a
|
||||
// denominator nobody can derive is a fact an operator must have.
|
||||
this.allCountsSuspect = true
|
||||
prodLog.error(
|
||||
`[FileSystemStorage] count-ledger derivation FAILED after ` +
|
||||
`${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` +
|
||||
`and must not be subtracted against; the next open retries:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
})()
|
||||
}
|
||||
|
||||
/**
|
||||
* @description A cheap witness that the ledger changed while a walk was
|
||||
* running. Every landed write moves one of these live counters, so an
|
||||
* unchanged stamp across a walk means no write landed during it.
|
||||
* @returns A value that differs whenever the live ALL scalars have moved.
|
||||
*/
|
||||
private ledgerActivityStamp(): string {
|
||||
return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}`
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Resolve once any background count-ledger derivation has
|
||||
* settled (succeeded or failed). Resolves immediately when none was needed.
|
||||
* Exists so tests and operators can observe the ledger's honest value rather
|
||||
* than race it; nothing in the read path waits on this.
|
||||
* @returns A promise that settles with the derivation.
|
||||
*/
|
||||
public async whenCountLedgerSettled(): Promise<void> {
|
||||
await this.countLedgerDerivation
|
||||
}
|
||||
|
||||
private async scanCanonicalEntities(
|
||||
kind: 'nouns' | 'verbs'
|
||||
): Promise<{ count: number; sampleDirs: string[] }> {
|
||||
|
|
@ -3198,21 +2647,9 @@ export class FileSystemStorage extends BaseStorage {
|
|||
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
||||
for (const entry of ids) {
|
||||
if (!entry.isDirectory()) continue
|
||||
const idAbs = path.join(shardPath, entry.name)
|
||||
let legs: string[]
|
||||
try {
|
||||
legs = await fs.promises.readdir(idAbs)
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') continue
|
||||
throw error
|
||||
}
|
||||
// No metadata content leg → a ghost or scar container → not an
|
||||
// entity. Same test pruneOrphanedEntities() uses, so the two agree
|
||||
// by construction.
|
||||
if (!this.hasMetadataContentLeg(legs)) continue
|
||||
count++
|
||||
if (sampleDirs.length < SAMPLE_MAX) {
|
||||
sampleDirs.push(idAbs)
|
||||
sampleDirs.push(path.join(shardPath, entry.name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3244,72 +2681,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs
|
||||
* — the vector-side mirror of {@link readEntityMetadataRaw}, same
|
||||
* reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`).
|
||||
* @param entityDir - Absolute `entities/nouns/<shard>/<id>` directory.
|
||||
* @returns The parsed vector record, or null when absent/unreadable.
|
||||
*/
|
||||
private async readEntityVectorRaw(entityDir: string): Promise<any | null> {
|
||||
const base = path.join(entityDir, 'vectors.json')
|
||||
try {
|
||||
return JSON.parse(await fs.promises.readFile(base, 'utf-8'))
|
||||
} catch {
|
||||
// fall through to the compressed variant
|
||||
}
|
||||
try {
|
||||
const gz = await fs.promises.readFile(`${base}.gz`)
|
||||
return JSON.parse(zlib.gunzipSync(gz).toString('utf-8'))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector —
|
||||
* the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities},
|
||||
* presence cannot be decided from the id-directory listing alone: a
|
||||
* deferred-embed noun's `vectors.json` EXISTS (written at `add()` time
|
||||
* with `vector: []`) until its embed LANDS, so this walk reads every
|
||||
* noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing.
|
||||
* ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts
|
||||
* here either (Brainy's write paths normalize an explicit zero-norm
|
||||
* vector to `[]` at write time, but a store created before that fix may
|
||||
* still carry legacy all-zero rows on disk; this derivation must agree
|
||||
* with the live ledger's definition of "vectored" regardless of when the
|
||||
* row was written). Used ONLY for a one-time legacy-counts.json derivation
|
||||
* or a lost/corrupted counts.json recovery; the result is persisted so
|
||||
* this scan never repeats.
|
||||
*/
|
||||
private async scanVectoredNounCount(): Promise<number> {
|
||||
const base = path.join(this.rootDir, 'entities', 'nouns')
|
||||
let vectored = 0
|
||||
try {
|
||||
const shards = await fs.promises.readdir(base, { withFileTypes: true })
|
||||
for (const shard of shards) {
|
||||
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue
|
||||
const shardPath = path.join(base, shard.name)
|
||||
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
||||
for (const entry of ids) {
|
||||
if (!entry.isDirectory()) continue
|
||||
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
|
||||
if (
|
||||
record &&
|
||||
Array.isArray(record.vector) &&
|
||||
record.vector.length > 0 &&
|
||||
!isZeroNormVector(record.vector)
|
||||
) {
|
||||
vectored++
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
return vectored
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist counts to filesystem storage
|
||||
*/
|
||||
|
|
@ -3322,34 +2693,13 @@ export class FileSystemStorage extends BaseStorage {
|
|||
verbCounts: Object.fromEntries(this.verbCounts),
|
||||
totalNounCount: this.totalNounCount,
|
||||
totalVerbCount: this.totalVerbCount,
|
||||
// ALL-visibility ledger scalars (+ the suspect flag) — absent in files
|
||||
// written before the ledger existed; initializeCounts() derives them once.
|
||||
totalNounCountAll: this.totalNounCountAll,
|
||||
totalVerbCountAll: this.totalVerbCountAll,
|
||||
// Vectored-noun ledger scalar — absent in files written before it
|
||||
// existed; initializeCounts() derives it once (a content scan, see
|
||||
// scanVectoredNounCount()'s JSDoc).
|
||||
totalVectoredNounCount: this.totalVectoredNounCount,
|
||||
allCountsSuspect: this.allCountsSuspect,
|
||||
// Derivation-rule stamp for the ALL scalars above — 'identity-record'
|
||||
// when they were counted one-per-metadata-content-leg (the honest
|
||||
// rule); omitted (JSON.stringify drops `undefined`) when the current
|
||||
// in-memory scalars came from a legacy container-rule counts.json
|
||||
// that hasn't been through a sanctioned recount yet, so a future load
|
||||
// keeps naming them suspect rather than trusting an unproven value.
|
||||
allCountsDerivedBy: this.allCountsDerivedBy,
|
||||
lastUpdated: new Date().toISOString()
|
||||
}
|
||||
|
||||
// ATOMIC (temp + rename), never a plain writeFile. A direct write
|
||||
// truncates the file first, so every persist opened a window — measured
|
||||
// at roughly 750ms after a flush or close on a real store — in which a
|
||||
// concurrent reader saw counts.json EMPTY. An empty file is unparseable,
|
||||
// and an unparseable ledger sends the next open down the full-rescan
|
||||
// path: the cheapest file in the store was costing the most expensive
|
||||
// recovery. The rename is atomic, so a reader sees the old ledger or the
|
||||
// new one, never neither.
|
||||
await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2))
|
||||
await fs.promises.writeFile(
|
||||
this.countsFilePath,
|
||||
JSON.stringify(counts, null, 2)
|
||||
)
|
||||
} catch (error) {
|
||||
console.error('Error persisting counts:', error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -520,12 +520,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
|
||||
let totalNouns = 0
|
||||
let totalVerbs = 0
|
||||
// Vectored-noun scalar: unlike the bare presence check above, this needs
|
||||
// the vectors.json RECORD'S content — a deferred-embed noun's record
|
||||
// exists with an empty `vector: []` until its embed lands. In-memory this
|
||||
// is a free field access (no I/O), unlike the filesystem adapter's
|
||||
// per-noun disk read.
|
||||
let totalVectoredNouns = 0
|
||||
|
||||
// Scan all paths in objectStore
|
||||
for (const path of this.objectStore.keys()) {
|
||||
|
|
@ -534,10 +528,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
if (nounMatch) {
|
||||
// Type is in metadata, not path - just count total
|
||||
totalNouns++
|
||||
const record = this.objectStore.get(path) as { vector?: unknown } | undefined
|
||||
if (Array.isArray(record?.vector) && record.vector.length > 0) {
|
||||
totalVectoredNouns++
|
||||
}
|
||||
}
|
||||
|
||||
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
||||
|
|
@ -550,11 +540,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
|
||||
this.totalNounCount = totalNouns
|
||||
this.totalVerbCount = totalVerbs
|
||||
// A scan of every canonical record IS the ALL-visibility count.
|
||||
this.totalNounCountAll = totalNouns
|
||||
this.totalVerbCountAll = totalVerbs
|
||||
this.totalVectoredNounCount = totalVectoredNouns
|
||||
this.allCountsSuspect = false
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue