diff --git a/.claude/skills/architecture.md b/.claude/skills/architecture.md index de046b17..4de3c287 100644 --- a/.claude/skills/architecture.md +++ b/.claude/skills/architecture.md @@ -2,7 +2,7 @@ ## What Is Brainy -@soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. +@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. ## Core Architecture diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 00000000..da5887f6 --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,66 @@ +name: CI + +# Branch pushes only — a release TAG deliberately does not re-run CI: the +# tagged commit's CI already ran on its branch push, and the runner is +# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the +# tag's publish-source run and starve every release (observed on 8.10.3 and +# 9.0.0: the publish sat behind the tag's own redundant CI). +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ['**'] + pull_request: + +jobs: + node: + name: Node ${{ matrix.node-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ['22', '24'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + - run: npm ci + - run: npm run test:unit + + # The correctness plant's full gate: integration + conformance run here on + # dedicated iron, on every push, so a release never depends on any other + # machine being up. Verdicts live in this run's log (never inferred). + integration: + name: Integration + conformance (Node 22) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: npm ci + - run: npm run test:ci-integration + - run: npx vitest run tests/conformance + + bun: + name: Bun (latest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + - run: npm ci + # test:bun imports the built dist/, so build first. + - run: npm run build + # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). + - run: npm run test:bun diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml new file mode 100644 index 00000000..6bd42b2a --- /dev/null +++ b/.forgejo/workflows/publish-source.yml @@ -0,0 +1,84 @@ +name: Publish (The Source) + +# Datacenter-side publish to The Source (source.soulcraft.com — our +# self-hosted Forgejo; never call it "the forge", Forge is a different +# product), moved off the laptop: an 87MB tarball PUT over the laptop's WAN +# times out; The Source's own runner does it in seconds. +# scripts/release.sh tags + pushes, then polls this workflow's result (npm +# view against The Source's registry) before it ever touches the npmjs leg — +# see the "delegation contract" in scripts/release.sh's home-publish step. + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + ref_reason: + description: 'why this manual run (e.g. tag event dropped)' + required: false + +jobs: + publish: + name: Publish to The Source registry + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: npm ci + - run: npm run build + - name: Publish + readback-verify on The Source registry + env: + # The stored repo-settings secret keeps its historical name. + FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }} + run: | + set -eo pipefail + + SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" + VERSION="$(node -p "require('./package.json').version")" + # The dist-tag follows the version: a prerelease (any hyphen — + # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — + # every consumer resolving 'latest' from this registry would otherwise + # be handed a release candidate. Same rule scripts/release.sh applies + # to the storefront leg. + NPM_TAG="latest" + case "$VERSION" in + *-*) NPM_TAG="rc" ;; + esac + echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." + + TMPRC="$(mktemp)" + chmod 600 "$TMPRC" + { + echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}" + } > "$TMPRC" + + # The release script bumps package.json's version before it tags, so + # this tag's checkout already carries the version being published — + # nothing here re-derives it from the tag name. + PUBLISH_OK=true + if ! npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then + PUBLISH_OK=false + fi + + # Readback verify is the source of truth, run regardless of the publish + # exit code: a benign duplicate publish (a prior run, or a mirror, already + # landed this exact version) reports failure even though the registry + # already holds the right content. + LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" + rm -f "$TMPRC" + + if [ "$LANDED_VERSION" != "$VERSION" ]; then + echo "::error::Readback verify FAILED — The Source registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." + exit 1 + fi + + if [ "$PUBLISH_OK" = true ]; then + echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry." + else + echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index cdb2ab14..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: CI - -on: - push: - pull_request: - -jobs: - node: - name: Node ${{ matrix.node-version }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - node-version: ['22', '24'] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: npm - - run: npm ci - - run: npm run test:unit - - bun: - name: Bun (latest) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - run: npm ci - # test:bun imports the built dist/, so build first. - - run: npm run build - # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). - - run: npm run test:bun diff --git a/CHANGELOG.md b/CHANGELOG.md index fcfeadb9..a54d609e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,314 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) + +- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) +- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25) +- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40) +- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2) +- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927) +- docs: measurements in public history carry numbers, not provenance (02c61636) +- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010) +- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950) +- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9) +- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5) +- perf(open): answer "are there any entities?" with one directory read (417ddb51) +- perf(generations): discover generations by directory name, not by walking the log (9dd39921) +- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc) +- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca) +- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f) +- chore: keep the generated neural stamps at main's values (c1f09723) +- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3) +- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02) +- feat(open): open never waits for a provider that is rebuilding itself (131daa08) +- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f) +- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6) +- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b) +- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f) +- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c) +- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b) + + +### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) + +- Merge branch 'next/open-brainy-rename' (a58372f0) +- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) +- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) + + +### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) + +- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) + + +### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) + +- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) +- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) +- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1) +- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4) +- Merge branch 'next/enumeration-identity-rekey' (204d74c1) +- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16) +- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a) +- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab) +- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9) +- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245) +- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee) +- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb) +- chore(release): 10.4.1-rc.1 (7870dc40) + + +### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26) + +- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) +- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) + + +### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) + +- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) + + +### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) + +- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) + + +### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) + +- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) +- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) +- fix(add): empty string is real data, not a missing field (258e9042) +- feat(vfs): implement readdir's recursive option — typed since 7.30, never read (fc516da6) +- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) + + +### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) + +- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) +- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) +- fix(storage): an unknown nested storage config can never silently land on the shared default root (ddd5e719) +- docs(release): the 10.4.0 entry, the index-health concept doc, and the API surfaces — written from the tree, not the plan (8cced871) +- fix(plugins): the silent-degrade doors close — a broken accelerator install can never read as absent (b9ba50fb) +- feat(recovery): the catchup verdict is consumed; verb rows go live; the metadata rebuild goes online (18f172e0) +- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) + + +### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) + +- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) +- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) +- test(fold-checkpoint): the ARM-AT-FLIP pin arms its crash instead of racing the pending-flush timer (4176439b) +- fix(health): one contract for a throwing probe — heal is none, serving is not withheld; repair report gains missing/rebuilt/reason (116550eb) +- feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete (7c8c8be3) +- fix(delete): the null-metadata skip closes — index legs run id-keyed or narrate, never silently strand postings (607e9f54) +- feat(repair): repairIndex returns the per-family receipt and narrates its summary (8d45f964) +- fix(reads): the readiness gate guards every index read surface — serving empty from a not-ready provider is unrepresentable (40e7119b) +- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) + + +### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) + +- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) +- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) + + +### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) + +- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) +- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) +- test(budgets): iron-honest wall-clock budgets — 3x the worst honest-iron measurement (314e0e6c) +- fix(locks): live writers are never auto-evicted; evicted writers are fenced at every commit barrier (292e7c04) +- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) + + +### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) + +- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) +- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) +- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) + + +### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) + +- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) +- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) +- feat(recovery): the fold-checkpoint bound — crash folds (checkpoint, head], never the whole log twice (ff43de1a) +- fix(log): pad-frame construction is total; the at-ack sync-failure compensation splits by phase — a production adoption's two write-path defects, cured at their roots (cbe34d11) +- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) + + +### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) + +- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) +- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) +- fix(recovery): walks are healers — the typed/tolerant boundary redrawn where block-layer fault injection proved it belonged (0e3facf4) +- feat(log): log authority is the fleet default — adopt-at-open, oracle-gated; plus the power-cut throw-site cures and the loud torn-record contract (214c98b4) +- fix(durability): three block-layer power-loss findings from the first fault-injection box run — all cured, matrix 15/15 (67c606be) +- docs: RELEASES.md frames the release as 10.0.0 — honest major (log format v2 forward-only); comment wording cleanup (d1698fa5) +- fix(persistence): the idle flush trigger debounces under load — deferred to the floor, never dropped, never a flush-per-gap amplifier (a50726e6) +- feat(reprojection): the one doors-open machinery — budget-capped, yielding, foreground-preempted, atomic-swap; poison records quarantine typed (d1651f98) +- feat(embedding): deferred-embed markers become log records — the sidecar recovery path is deleted (b47787bb) +- feat(conformance): the golden-log fold oracle — encoder bytes and fold semantics pinned by content hash (c95bea88) +- feat(engine): the wiring wave — stamps ride every flush, provider generations, waitForIndexed, adopt-backfill, match-all serves (b53e6e89) +- feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data (b35d87a7) +- feat(log): v2 is the LIVE write format — envelope records with minted ints, genesis, sector seals; v1 readable forever (26c60251) +- docs: RELEASES.md — the unreleased write-path and lifecycle entry (consumer-facing draft; version set at cut) (73eb88d4) +- feat(temporal): as-of semantic recall joins the release contract — past vectors byte-exact, pinned (f7ca0d26) +- fix(log): acked writes survive power loss; rejected writes never silently commit — the kill-matrix goes 11/11 with zero .fails debt (13022c51) +- feat(plugin): every provider write surface carries the real committed generation (2d532684) +- feat(log): fact-log format v2 codec — record envelope, type registry, genesis, sector seals; fault-injection shim (34841074) +- feat(log): the guarded log-authority core — group-commit durable-at-ack, the per-brain switch, the verification oracle (65953097) +- docs: Path Registry rows DP6/DP8/MT5 flip to contracted+pinned — the deferred-embedding and atomic-update train landed with cited tests (9fda6d95) +- feat(embedding): MT5 — deferred embedding with durable markers; write acks never wait on a neural net (287384cf) +- fix(index): the flicker window dies — atomic in-place vector update; lazy open honors every provider's not-ready report; the Path Registry twin table (ebe06cdf) +- feat(persistence): the engine owns its flush cadence — callers never call flush() in hot paths again (3236a01b) +- fix(aggregation): the lifecycle cluster — flush stamps, behind-stamp catches up incrementally, the native rebuild finally gets invoked, deletes are never silently skipped (1dc861d2) +- perf(sort): ordered reads never do per-row storage round-trips — the 199-317s production scan class dies structurally (607b6b56) +- chore: the home registry is The Source, never 'the forge' — sweep the misnomer out of the release rail, workflows, and release notes (Forge is a different product; the stored CI secret keeps its historical name) (09352c2b) +- ci: tags stop triggering the CI matrix (redundant re-run of already-tested commits starved every release's publish run on the sequential runner) + release.sh forge poll window 20→50 min (c6c6ea6b) +- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) + + +### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) + +- docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) +- fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) +- docs: v9.0.0 release notes — the field-addressing law migration ledger; retitle the shipped 8.11.0 canonical-enumeration entry (header went stale at its cut) (55a7512c) +- feat(namespace): merge the field-addressing law train — no special names, system.* scalars, nested-bag storage, epoch-3 index keys (19b477ae) +- feat(namespace): NO SPECIAL NAMES + storage fidelity — the ruled completion of the field-addressing law (24bf6cdb) +- feat(namespace): write-door forgery refusal (user metadata keys may never start 'system.') + refusal messages name both spellings in every branch (the non-colliding case marks system. honestly as NOT valid) — cross-engine message pin alignment (48a6130a) +- feat(namespace): conformance green 19/19 — data-aware did-you-mean on unindexed bare addresses, ordering contract on the column top-K path (never drop, nulls last, ties by id), shape-complete addressed reads (entity views AND raw storage shapes, shadow-proof both scopes), per-key source matching for dotted addresses; refusal classes unified under UnresolvableFieldError (8e962dab) +- feat(namespace): aggregation reads under the law + epoch 3 (the key-split rebuild) + THE ARMING COMMIT — the capability constant, the law module, and the typed refusals export from the package root; both engines' conformance suites light on this signal (7492b6cb) +- feat(namespace): egress guard + validation speak the law — whereMatcher's resolver reads system.* from the record and bare names from the metadata bag only (the bare-system switch is dead); validateFindParams refuses cursor/includeRelations/writeOnly typed (accepted-and-ignored dies as a class), validates order, and parses every orderBy address (c2fb28a2) +- fix(namespace): noun-record updates preserve legacy inline HNSW adjacency — the placeholder-adjacency write stamped out pre-codec records' stored connections (crash-window unreachability); codec-era records were never at risk (empty field is the blob marker); pin covers the legacy shape (4679c894) +- feat(namespace): find's own filter builders speak the frozen keys — params.type/subtype/service become system.* index keys at every construction site (three pipelines + the canonical buildMetadataFilter); the where.type→noun alias is dead (bare 'type' belongs to the user now) (7a28a946) +- feat(namespace): the index speaks the frozen keys — record-frame scalars index under literal 'system.' (legacy 'noun' spelling folds into system.type; plumbing never indexed from a record frame), user fields stay bare in every shape; filter + sorted paths route every address through parseFieldAddress; storage fallbacks read the addressed side of the record (11c724bc) +- docs(namespace): the d.ts JSDoc wave — the sealed field-addressing law on the full find + aggregation surface, present-tense, with the refusal semantics and migration note inline (comment-only; verified zero code lines changed) (fcb24ab6) +- test(namespace): unit pins for the pure law — the ruled maps verbatim (incl. the relation mirror, unpinnable via public API), plumbing refusals both kinds, did-you-mean text (5502abcd) +- fix(namespace): the JS sorted fallback honors the ruled ordering contract — nulls last in BOTH directions (was nulls-first on desc) + deterministic id-ascending tie-break (56deb2e8) +- test(namespace)+docs: the cross-engine conformance suite (self-arming — skips until the resolver exports land) + the public field-addressing docs page; sidebar order deconflicted to 7 (d8d0b55f) +- feat(namespace): the one field-addressing law as a single source of truth — parseFieldAddress + the ruled ten-scalar system maps + plumbing invisibility + refusal builders (module only; query surfaces wire in next) (8f9a9989) +- docs: port the 8.10.3 backport-release changelog entry to main (f6b14d21) +- docs: port the 8.10.2 backport-release changelog entry to main — release branches carry the version bump, main carries the durable record (0b059ac5) +- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (1a09be06) +- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (cb717be2) +- fix(release): double the forge-publish poll budget — the runner executes jobs sequentially and the publish run queues behind the ci matrix (64049631) +- Merge branch 'release/8.11.0' (1865f60a) +- Merge branch 'release/8.10.1' (fc9f0d72) +- chore: the forge is the address — retire the archived mirror from every live surface (415e824a) +- Merge remote-tracking branch 'origin/main' (069a8894) +- Merge branch 'release/8.10.0' (d918c060) +- ci: run the pipeline on the forge (9a5a9ccc) +- feat: two-tier history reads + the repacker + generationDigest — D1+D3 wired end-to-end (1201e255) +- feat: generation-segment store — the D1+D3 packed-tier file format (d8acb377) +- feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) + + +### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) + +- docs: the last two archived-host links point home (91ef1c8b) +- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) +- feat(release): the forge publish leg moves to CI on the tag push; the laptop verifies by readback and keeps the abort-before-storefront guard (3e4a17dc) +- feat: canonical enumeration mode for export — storage-walked, canon-complete, with an index-drift report (4d196af4) +- ci: run the pipeline on the forge (999d0ebb) + + +### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) + +- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) +- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) + + +### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) + +- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) +- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) + + +### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) + +- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) +- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) +- fix: transaction timeouts are a typed no-hot-retry contract; engine-side non-retry pinned; dead transaction path removed (003e2a74) +- chore: the forge is the address — retire the archived mirror from every live surface (22702b81) + + +### [8.10.0](https://github.com/soulcraftlabs/brainy/compare/v8.9.0...v8.10.0) (2026-07-23) + +- docs: adoption storefront — contributing guide, security policy, README support + cor section (9a99a7b) +- fix(release): push the public mirror explicitly and verify the tag lands at the right commit before publishing (6ba94c8) +- docs: project guide version line points at npm instead of a hardcoded stale number (3a1efc9) +- feat: vector provider identity is a required name field (hnsw-js), rendered [vector-index:] (3be4ba9) +- feat: warm contract (warm/warmOnOpen/provider warm hook), configurable transact budget floor, backend-neutral vector index op names (55b867c) + + +### [8.9.0](https://github.com/soulcraftlabs/brainy/compare/v8.8.2...v8.9.0) (2026-07-19) + +- docs: measured performance envelopes v1 (per-op p50/p95 at 1k and 10k, pure-JS floor) (5cabd78) +- fix: release drains in-flight writer-lock heartbeat — no phantom lock after unlink (70e4bc8) +- feat: flush() never compacts — history maintenance moves to close() with bounded passes (300d9f2) + + +### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19) + +- fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d) +- chore: push public docs to the soulcraft.com ingest door on release (42037d0) + + +### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18) + +- fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48) +- fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass (4fcef7b) + + +### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17) + +- feat: OS-limit detection for pool-scale deployments (16a73b8) + + +### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17) + +- fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46) + + +### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17) + +- feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb) + + +### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17) + +- feat: brain.auditGraph() — read-only graph-truth audit (2a03fae) + + +### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17) + +- fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064) + + +### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17) + +- fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7) +- docs: external-backups/sparse-storage guide + generation fact log concept (593bb8b) + + +### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15) + +- test: tolerant timing assertion in the execution-time measure test (4dc0a92) +- feat: committedGeneration capability + pinned durability/stability contracts (d1ecee1) +- docs: RELEASES.md entry for 8.5.0 (provider fact-log access + shared verifier) (e4f37cd) +- feat: provider access to the fact log + shared stamp verifier via internals (352e356) + + +### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15) + +- docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43) +- feat: entity-tree family stamp — sourceGeneration + rollup coherence at open (2888ae6) +- feat: generation fact log — after-image commit records, dual-written at every commit point (38b0041) + + ### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15) - docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd) diff --git a/CLAUDE.md b/CLAUDE.md index 568c10db..56df0b72 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md` **Brainy's current open actions:** None. MIT open-source — no platform-specific actions. -**Current version:** `@soulcraft/brainy@7.31.5` (latest published; 8.0.0 release candidate on `feat/8.0-u64-ids`) +**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` --- ## Project Overview -Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license. +Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license. ## Getting Started @@ -91,7 +91,7 @@ test: add/update tests (patch version bump) ## Docs Pipeline — soulcraft.com/docs -Docs in `docs/**/*.md` are published with the npm package (included in `files`) and synced to soulcraft.com/docs on every portal deploy. Frontmatter controls what appears publicly. +Docs in `docs/**/*.md` are published with the npm package (included in `files`) and go live on soulcraft.com/docs via the docs ingest API: the release script's `scripts/push-docs.js` step POSTs every public doc to `https://soulcraft.com/api/docs/ingest` (auth: `DOCS_INGEST_SECRET` in the environment). No separate deploy step is involved (the old deploy-to-publish flow was retired in a platform change, 2026-08). Frontmatter controls what appears publicly. ### Docs check triggers @@ -161,9 +161,9 @@ npm run release:major # Breaking changes (rare, manual decision) The script: verifies clean git state, builds, tests, bumps version, updates CHANGELOG.md, commits, tags, pushes, publishes to npm, and creates a GitHub release. After a successful release, remind the user: -> "Published. Deploy portal to pick up the new docs → go to the portal project and deploy." +> "Published. Docs are live on soulcraft.com/docs (pushed via the ingest API during the release) — spot-check a changed page with curl." -Do NOT deploy portal from here. Portal is always deployed separately from within the portal project. +There is no separate deploy step anymore. If the docs push failed (the script warns loudly), re-run `node scripts/push-docs.js` with `DOCS_INGEST_SECRET` set. ## Closed-Source Product Names — HARD RULE diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab8a0246..54d4f784 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,298 +1,77 @@ # Contributing to Brainy -Thank you for your interest in contributing to Brainy! This document provides guidelines and instructions for contributing to the project. +Brainy is MIT-licensed and genuinely open to outside contributions. This page +is the honest, current path — please don't rely on older instructions you +may find elsewhere in the repo's history. -## Code of Conduct +## Where the project lives -By participating in this project, you agree to abide by our Code of Conduct: -- Be respectful and inclusive -- Welcome newcomers and help them get started -- Focus on constructive criticism -- Respect differing viewpoints and experiences +The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. +It's anonymously readable and cloneable — no account needed to browse, clone, +or build. -## How to Contribute +## How to contribute -### Reporting Issues +**Found a bug, or have an idea?** Email **brainy@soulcraft.com**. No account, +no ceremony — you'll get a receipt, and it goes to a human. -Before creating an issue, please check existing issues to avoid duplicates. +**Want to send a patch?** Two ways, both first-class: -When creating an issue, include: -- Clear, descriptive title -- Detailed description of the problem -- Steps to reproduce -- Expected vs actual behavior -- System information (OS, Node version, Brainy version) -- Code examples if applicable +- **Email a patch.** Run `git format-patch` against your change and email the + output to **brainy@soulcraft.com**. This is a genuinely supported path, not + a fallback — plenty of good contributions arrive this way. +- **Open a pull request on the forge.** Request an account at + **source.soulcraft.com** (registration is request-with-approval, so allow + a little lag), clone, push a branch, and open a PR there. Maintainers + review and land it. -### Suggesting Features +Either way, for anything beyond a small fix, opening an issue first (email is +fine) to talk through the approach saves everyone rework. -Feature requests are welcome! Please provide: -- Clear use case -- Proposed API/interface -- Examples of how it would work -- Any potential challenges or considerations +## Development setup -### Pull Requests - -#### Before Starting - -1. Check existing issues and PRs -2. Open an issue to discuss significant changes -3. Fork the repository -4. Create a feature branch from `main` - -#### Development Setup - -**Quick Setup (Recommended):** ```bash -# Clone your fork -git clone https://github.com/your-username/brainy.git +git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git cd brainy - -# Run setup script (installs all dependencies including Rust) -./scripts/setup-dev.sh -``` - -**Manual Setup:** -```bash -# Clone your fork -git clone https://github.com/your-username/brainy.git -cd brainy - -# Install system dependencies (Ubuntu/Debian) -sudo apt-get install -y build-essential pkg-config libssl-dev - -# Install Rust (for WASM embedding engine) -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source ~/.cargo/env -rustup target add wasm32-unknown-unknown -cargo install wasm-pack - -# Install Node.js dependencies npm install - -# Build Candle WASM embedding engine -npm run build:candle - -# Build TypeScript npm run build - -# Run tests npm test ``` -#### Making Changes +Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite; +see `package.json` for `test:integration`, `test:coverage`, and friends. -1. **Follow the code style** - - TypeScript for all source code - - Clear variable and function names - - Comments for complex logic - - JSDoc for public APIs +## Standards -2. **Write tests** - - Add tests for new features - - Update tests for changes - - Ensure all tests pass +- **Strict TypeScript.** No `any` escape hatches to dodge the type checker. +- **Tests exercise real behavior.** No mocking away the thing you're supposed + to be testing. +- **No stubs, no TODO-code.** If something can't be finished, say so and + leave it out — don't merge a placeholder. +- **JSDoc on every exported function, class, and type.** +- **[Conventional Commits](https://www.conventionalcommits.org/).** `feat:`, + `fix:`, `docs:`, `perf:`, `refactor:`, `test:`, `chore:`. Never + `BREAKING CHANGE` in a commit message — major version bumps are a separate, + deliberate decision. +- **Performance claims are measured or labeled projected.** If a PR or its + description states a number, cite the benchmark that produced it (see + [docs/performance-envelopes.md](docs/performance-envelopes.md) for the + pattern). Don't state an estimate as if it were measured. +- **Measurements carry numbers, not provenance.** Public commit messages and + docs give the SHAPE a number was taken at and never where it was taken: no + hostnames, no store or deployment identities, no operational anecdotes about + someone's running system. "A 14,056-noun / 72,679-verb production-shaped + store, measured solo under an exclusive lock" tells a reader everything the + number depends on; the machine it ran on and whose data it was tell them + nothing except where somebody's infrastructure lives. +- **Documents that answer or reference a confidential specification never enter + this repository, even summarized.** The public docs describe THIS engine and + the published contract, and nothing else — a summary of a private document is + still that document's contents. -3. **Update documentation** - - Update README if needed - - Add/update API documentation - - Include examples +## License -#### Commit Guidelines +Brainy is [MIT licensed](LICENSE). Contributions are accepted under the same +license — there's no CLA to sign. -Follow conventional commits format: - -``` -type(scope): description - -[optional body] - -[optional footer] -``` - -Types: -- `feat`: New feature -- `fix`: Bug fix -- `docs`: Documentation changes -- `style`: Code style changes -- `refactor`: Code refactoring -- `perf`: Performance improvements -- `test`: Test changes -- `chore`: Build/tooling changes - -Examples: -```bash -feat(triple): add graph traversal depth limit -fix(storage): handle concurrent write conflicts -docs(api): update search method documentation -``` - -#### Submitting PR - -1. Push to your fork -2. Create PR against `main` branch -3. Fill out PR template -4. Ensure CI checks pass -5. Wait for review - -### Testing - -#### Running Tests - -```bash -# Run all tests -npm test - -# Run specific test file -npm test tests/core.test.ts - -# Run with coverage -npm run test:coverage - -# Watch mode -npm run test:watch -``` - -#### Writing Tests - -```typescript -import { describe, it, expect } from 'vitest' -import { Brainy } from '../src' - -describe('Feature Name', () => { - it('should do something specific', async () => { - const brain = new Brainy() - await brain.init() - - // Test implementation - const result = await brain.search("test") - - expect(result).toBeDefined() - expect(result.length).toBeGreaterThan(0) - }) -}) -``` - -## Architecture Guidelines - -### Adding New Features - -1. **Check existing functionality** - - Review `ARCHITECTURE.md` - - Check if similar features exist - - Consider if it should be an augmentation - -2. **Design considerations** - - Maintain backward compatibility - - Consider performance impact - - Think about all storage adapters - - Plan for extensibility - -3. **Implementation checklist** - - [ ] Core functionality - - [ ] Tests (unit and integration) - - [ ] Documentation - - [ ] TypeScript types - - [ ] Examples - - [ ] Performance benchmarks (if applicable) - -### Creating Augmentations - -Augmentations extend Brainy's functionality: - -```typescript -import { BrainyAugmentation } from '../types' - -export class MyAugmentation extends BrainyAugmentation { - name = 'MyAugmentation' - - async onInit(brain: Brainy): Promise { - // Initialize augmentation - } - - async onAdd(item: any, brain: Brainy): Promise { - // Process before adding - return item - } - - async onSearch(query: any, results: any[], brain: Brainy): Promise { - // Process search results - return results - } -} -``` - -### Performance Considerations - -- Use batch operations where possible -- Implement caching strategically -- Consider memory usage -- Profile performance impacts -- Add benchmarks for critical paths - -## Documentation - -### API Documentation - -Use JSDoc for all public APIs: - -```typescript -/** - * Searches for similar items using vector similarity - * @param query - Search query (text or vector) - * @param options - Search options - * @returns Array of search results with scores - * @example - * ```typescript - * const results = await brain.search("machine learning", { limit: 10 }) - * ``` - */ -async search(query: string | Vector, options?: SearchOptions): Promise { - // Implementation -} -``` - -### Examples - -Add examples for new features: - -```typescript -// examples/feature-name.ts -import { Brainy } from 'brainy' - -async function exampleUsage() { - const brain = new Brainy() - await brain.init() - - // Show feature usage - // Include comments explaining what's happening - // Handle errors appropriately -} - -exampleUsage().catch(console.error) -``` - -## Release Process - -1. **Version bump**: Follow semantic versioning -2. **Update CHANGELOG**: Document all changes -3. **Run tests**: Ensure all tests pass -4. **Build**: Generate distribution files -5. **Tag**: Create git tag for version -6. **Publish**: Release to npm - -## Getting Help - -- **Discord**: Join our community -- **Issues**: Ask questions on GitHub -- **Discussions**: Share ideas and get feedback - -## Recognition - -Contributors will be recognized in: -- CHANGELOG.md for their contributions -- README.md contributors section -- GitHub contributors page - -Thank you for contributing to Brainy! 🧠 \ No newline at end of file +Thank you for considering a contribution. diff --git a/README.md b/README.md index d1342f52..762c9ec3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- Brainy + Brainy

Brainy

@@ -11,9 +11,9 @@

- npm version - npm downloads - CI + Package on The Source + Repository + CI Documentation MIT License TypeScript @@ -23,12 +23,15 @@ Quick start · One query · Features · - Scale with Cor · - Docs + Scale with Cor · + Docs · + Support

--- +**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license. + Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: | You write | Brainy indexes it as | You query it with | @@ -44,12 +47,14 @@ It runs **inside your process** — no server, no Docker, nothing to operate — ## Quick start ```bash -bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended -npm install @soulcraft/brainy # Node.js ≥ 22 +bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended +npm install @soulcraftlabs/brainy # Node.js ≥ 22 ``` +> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read). + ```javascript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() // in-memory; one line swaps to disk await brain.init() @@ -172,9 +177,11 @@ await brain.vfs.search('React components with hooks') // semantic file **[Multi-process model](docs/concepts/multi-process.md)** · **[Inspection guide](docs/guides/inspection.md)** -## From laptop to hundreds of millions +## When you outgrow Brainy -Brainy's TypeScript engines take you a long way. When you outgrow them, add the native engine — **the API doesn't change**: +Brainy's pure-TypeScript engines carry real workloads a long way on their own — see the measured, per-operation numbers (not marketing figures) in **[docs/performance-envelopes.md](docs/performance-envelopes.md)** for what to expect, unaccelerated, on plain filesystem storage. + +When a deployment needs native-scale vector/graph performance — memory-mapped indexes that don't need your dataset in RAM, billion-scale ambitions — add the native engine. **The API doesn't change:** ```bash npm install @soulcraft/cor @@ -187,13 +194,14 @@ await brain.init() // @soulcraft/cor detected — same code, native engines un Installing the package is the opt-in: if `@soulcraft/cor` is present, it loads and announces itself in the init log; if it's present but broken, `init()` **throws** — an installed accelerator never silently vanishes behind the JS engines. Opt out with `plugins: []`, or pin exactly what loads with `plugins: ['@soulcraft/cor']`. [`@soulcraft/cor`](https://www.npmjs.com/package/@soulcraft/cor) (Brainy 8.x ↔ Cor 3.x, version-matched) registers Rust implementations behind every provider seam: SIMD distance kernels, memory-mapped storage, a disk-native vector index that doesn't need your dataset in RAM, durable LSM field/graph indexes that serve cold opens instantly, and native aggregation. Recall@10 measured **0.99 / 0.96 / 0.96 at 1M / 10M / 100M vectors** in Cor's release gate. -Open core, commercial accelerator: Brainy is MIT and complete on its own; Cor is licensed and funds both. +Open core, commercial accelerator: Brainy is MIT and complete on its own — Cor is more headroom for when you need it, not capability held back to sell you later. Licensing and support: **cor@soulcraft.com**. ## Performance +- Per-operation p50/p95 at 1k and 10k entities, pure-JS floor, measured and re-run every release that touches a measured path: **[docs/performance-envelopes.md](docs/performance-envelopes.md)**. - JS distance kernels: **~6× faster cosine, ~1.4× euclidean** than 7.x (measured: [`tests/benchmarks/distance-microbench.mjs`](tests/benchmarks/distance-microbench.mjs), 384-dim, median of 41). - Whole-graph reads are single **O(N + E)** cursor walks — a consumer-measured 19k-edge export dropped from ~27 s of per-node calls to one scan. -- Full numbers and capacity planning: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)** +- Capacity planning and architecture: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)** ## Use cases @@ -212,6 +220,10 @@ Open core, commercial accelerator: Brainy is MIT and complete on its own; Cor is **Bun ≥ 1.1** (recommended) or **Node.js ≥ 22**. Brainy 8.x is server-only; the 7.x line remains on npm for browser use. -## Contributing & license +## Support & community -Contributions welcome — see **[CONTRIBUTING.md](CONTRIBUTING.md)**. MIT © Brainy Contributors. +- **Bugs and ideas** → **brainy@soulcraft.com** — no account needed, you'll get a receipt. +- **Security reports** → **security@soulcraft.com** — see **[SECURITY.md](SECURITY.md)**. +- **Contributing** → see **[CONTRIBUTING.md](CONTRIBUTING.md)**. + +MIT © Brainy Contributors. diff --git a/RELEASES.md b/RELEASES.md index 69159560..c875cb26 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,15 +1,1186 @@ # @soulcraft/brainy — Release Notes for Consumers +Machine-readable release notes are published at +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json +(this engine) and +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json +(the product engine) — read by HQ's `/hq/releases` door, and the source of +truth ahead of this file. + This file is the **quick reference for downstream sessions** tracking Brainy changes. -Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/soulcraftlabs/brainy/releases +Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases **How to use:** Brainy is the underlying data engine for downstream applications. Read this when: - Upgrading `@soulcraft/brainy` in your application - Debugging data, query, or storage behaviour - A new Brainy feature is available that you want to adopt +## Removed APIs — 7.x → 8.x (the complete ledger) + +Every public API removed at the 8.0 major, with its sanctioned replacement. If your code +still calls a left-column name on 8.x it throws (or the config key is rejected) — the +replacement is always a one-line change. (Standing contract from 8.9.0 forward: removals +happen only at majors, after ≥1 minor of loud runtime deprecation naming the replacement.) + +| Removed (7.x) | Replacement (8.x) | +|---|---| +| `brain.search(query, k)` | `find({ query })` — semantic; `find({ query, searchMode })` for hybrid | +| `brain.getRelations({...})` | `related(id, opts)` for adjacency; `find({ connected: {...} })` for scoped traversal | +| `brain.neural()` clustering | `find({ vector })` + aggregation `GROUP BY` | +| `Db.search()` | `db.find({ vector })` | +| Pre-8.0 storage path aliases (`directory`, `basePath`, …) | one `storage.path` key (old aliases throw) | +| Reserved keys inside `metadata` bags (silently remapped in 7.x) | top-level params (`subtype`, `visibility`, `confidence`, `weight`, …) — reserved-in-bag throws | +| 7.x COW branches layout (`branches/main/`) | generational MVCC (`asOf()`, `now()`, `db.persist(path)`) — on-disk migration is automatic at first 8.x open | + +The fork/snapshot family (`brain.snapshot()`, `createSnapshot()`, `restoreSnapshot()`) +is sometimes cited as a 7.x removal — those methods never existed on 7.x; the 8.0 Db API +(`asOf`/`persist`/`restore({confirm})`) is their first real implementation. + --- +## v10.4.4 — 2026-08-28 + +**A correctness and observability release.** The headline is not speed: it is that a +restart now tells you the truth about itself, a store stops lying about how much it +holds, and the engine stops doing work nobody asked for. There is a performance +improvement and it is modest; it is stated exactly below rather than rounded up. + +### The dark restart — fixed at the root + +A service could stop cleanly, exit 0, having awaited `close()` on every store it held, +and its next boot would announce `Overwriting stale writer lock … appears dead` for +every one of them. Nothing had crashed. Two deployments hit this; the same defect also +made those boots pay a crash-recovery fold they did not owe. + +The cause was not the lock. `close()` released it correctly — when it got there. A +failure part-way through close skipped both the release AND the clean-shutdown marker, +and "the recorded pid is gone" reads identically for an orderly restart and a crash. + +- `close()` is now two parts and the second is unconditional: the flush-request watcher, + the **writer lock**, the VFS timers and the terminal `closed` flag are released whether + the durable steps succeeded or not. The original failure is narrated with what it costs + the next open, then rethrown. +- Releasing the lock writes a **clean-close record** naming the lock generation it gave + up. The next open reads that record instead of guessing: recorded → nothing to recover; + absent → it says so, and names the recovery it is about to run. This also ends two + long-standing false alarms — a recycled pid locking a store out of its own reopen, and + `Re-acquiring writer lock … this is a bug` after a perfectly clean close. +- The signal path stopped failing in a batch. One store's failing flush used to strand + every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the + generation store's close (the marker) is part of shutdown, the lock goes in a `finally`, + and the handler no longer calls `process.exit()` when the host application has its own + signal handler, a race that truncated the host's own shutdown mid-flight. + +### The count ledger stops lying, and `counts.json` is written atomically + +The all-tier scalars are the denominator a coverage check subtracts against. A ledger +derived under the old rule — one entity per id DIRECTORY — counted ghost and scar +containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for +the life of the store. Two copies of one archive could disagree, and a downstream index +heal reported remaining work that did not exist. + +- Such a ledger now derives itself honestly **in the background** after the open, counting + identity records, and persists the correction stamped. Nothing waits for it, because no + read is served from a denominator. +- A derivation that raced a write refuses to stamp its number: one retry on a quiet store, + then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under + a barrier. +- `counts.json` is written temp+rename. A truncating write left a window in which a + concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open + down the full-rescan path, so the cheapest file in the store was buying the most + expensive recovery. + +### An open and a repair narrate themselves — on a channel a log level cannot silence + +A store could open for three minutes and print nothing at all. The phase timings existed; +they were written to a channel that every production-looking environment clamps away. + +- Narration moved to an always-visible channel. An open now heartbeats the phase it is in, + names each phase as it ends with what it was paying for, and names the expensive STEP + inside a phase. `repairIndex()` does the same and its receipt carries a per-family + `durationMs` — a repair that ran for half an hour with no output could only be watched + through `top`. +- A brain nobody has written to now does nothing: a flush over a clean store is a no-op + and says nothing, the graph index's auto-flush asks before it acts, and the + cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a + directory every 500 ms per store forever, with a slow safety sweep behind it and a + narrated fall back to polling where a filesystem cannot be watched. +- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()` + does not wait for it, every other family serves, and that family's doors refuse **by + name, carrying the provider's own progress**, saying plainly that they open by + themselves and no action is needed. Health narration dedupes by content, so an unchanged + verdict is silent however a provider's generation counter moves. + +### For operators — one behaviour change + +**Four `where` operators that previously returned an empty page now raise +`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range +posting index cannot evaluate a substring, a pattern or an array length without reading +every row, and it now refuses by name instead of answering with an empty result that +looks like an answer. + +**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and +`excludes`. All 25 accepted operator tokens now agree between this engine and its +accelerated counterpart. + +### Performance — stated exactly + +Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under +an exclusive lock: + +- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is + one fix — generation discovery reads directory NAMES instead of recursively walking the + entire generation log (−9.2 s, and it scales with history rather than row count). The + VFS phase is **unchanged**. +- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving + off the critical path accounts for storage-init dropping 5,941 ms → 25 ms. +- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own + init is under 2 s of that phase. It is the log-authority adoption and/or the + pending-embed log recovery, both now instrumented so the next measurement names the + culprit outright. + +Continuing work, named so nobody has to rediscover it: that ~38 s term; making the +generation store's committed-range set lazy; the hydration path that substitutes +`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix +filter built with a `$startsWith` spelling no operator set accepts, so +`searchFiles({ path })` throws today. + +--- + +## v10.4.3 — 2026-08-27 (Open Brainy's first release) + +**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for +byte — only the name, the registry, and the pointers changed.** Install: + +```bash +npm install @soulcraftlabs/brainy +``` + +with the registry line in your `.npmrc` (anonymous read): + +``` +@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/ +``` + +- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the + npmjs republish step is retired from the release rail. Existing npmjs versions of + `@soulcraft/brainy` stay as they are and receive no new versions. +- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects. +- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one + install-line change (`@soulcraft/brainy` → `@soulcraftlabs/brainy`), which downstream + applications make together with their native-engine bump. + +## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector) + +**This is the last release of the MIT engine under the `@soulcraft/brainy` name.** +The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API, +client library, types and protocol, an openly specified canonical format, and the TypeScript +reference engine, scoped honestly as a single-node engine for stores up to roughly one +million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a +major version bump; that engine implements the same API over the same open format at +production scale, requires a license, and refuses loudly without one. Nothing changes +for existing installs until that major ships; the move is announced with it. + +Six fixes, one law: a vector with no magnitude carries no information, so it must +never reach a vector index — in any engine — and the canonical store must say so. + +- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item + shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted + with an empty vector leg, never embedded, never indexed, counted as unvectored in + the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing — + no longer need a placeholder vector and never enter the vector leg. `vector: []` + together with `deferEmbedding: true` is refused with a typed error (a supplied + vector has nothing to defer). Previously `vector: []` threw a dimension error. +- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is + the sanctioned way to strip a vector from an existing row: canonical vector → `[]`, + removal from the vector index, the vectored ledger decremented exactly once — and + idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and + it clears a pending deferred-embed marker durably so the background worker cannot + re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the + index from canonical rows); shedding historical vectors needs this door. +- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on + any write path is persisted as unvectored (`[]`) with one warning naming the row; + the vector-index operations keep their own refusal as a second line. The engine's + own VFS root, which used to persist a deliberate all-zero placeholder (harmless + under cosine distance, a false attractor under a downstream engine's + squared-euclidean serving — a production incident this week), is now created + unvectored, and an existing store's legacy root is migrated on open by a single + fixed-path read before the health gate runs — never a walk. +- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the + cursor walks behind them enumerate by the metadata record, the same key the + canonical ledger counts by — previously the walk keyed on the vector file, so a + row holding metadata but no vector was counted yet never yielded (a permanent + "missing" phantom in coverage math), while an orphaned vector-only directory + could be yielded as a phantom id. The recovery fold also never deletes an existing + vector when it replays a metadata-only after-image (preserve-if-absent). One + documented gap remains: a verb's endpoints live only in its vector leg, so a + metadata-only verb is counted and loudly skipped, never fabricated — the fix is a + canonical-format change and lands with the open format. +- **The ledger's one-time derivation counts identity records.** Stores upgraded from + pre-ledger versions derived their ALL-visibility scalars once by counting id + directories, which included ghost and scar containers left by an old partial-delete + defect — an inflated denominator whose coverage row could never reach exact. The + derivation now counts only directories holding a metadata record, `counts.json` + carries a derivation-rule stamp, and a ledger derived under the old rule is marked + `suspect` at open (one O(1) field read, one warning) so the online `repairIndex()` + path clears it with a real recount. +- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and + zero-norm rows (one summary line), re-pins the vector dimension from the first real + vector after a restart (previously a restart left the pin unset, so a wrong-length + insert became the new pin instead of being rejected), and `addItem` / `updateItem` + throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing + a vector-less node. +- **Smaller:** a failing plugin activation now rethrows with the original error as + `cause` (the originating file and line survive to the caller's log); build + generators stamp from the repository history of their inputs instead of wall clock, + so two builds of the same tree are byte-identical. + +Adoption: one restart, paired with its native-engine release. The first open of an +existing store runs the legacy-root migration (one narrated line) and, on stores that +upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned +recount — no rebuild in either case. + +## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds) + +Two production defects from the same week, fixed together as a patch to 10.4.0. + +- **The read gate is per family.** A read now refuses only when the index family it + actually consults is unhealthy: a metadata filter is served while the vector leg is + rebuilding; a semantic query is refused only by the vector family; a graph + traversal only by the graph family. Previously any unhealthy family refused every + read on the brain — under a long vector rebuild, a production deployment's + metadata-only reads were refused for the duration, and the retries became a write + pump of their own. +- **Unchanged data never re-embeds.** `update()` compares the incoming `data` + structurally with the stored record; an update carrying identical data (a common + shape for periodic upserts) no longer embeds again and no longer churns the vector + leg. Previously every such update re-embedded and re-inserted, which under load + saturated the vector index with near-identical vectors. + +Adoption: one restart, paired with its native-engine release. + +## v10.4.0 — 2026-08-25 (the health report has a name) + +Three related cures, one root cause: an index deciding whether it could be trusted +by sampling itself instead of by exact accounting. This release replaces every +sampled self-probe with ledger-derived truth, and a read against an unhealthy index +now refuses loudly instead of guessing. + +- **The canonical count ledger.** Storage now tracks two scalars per family + (nouns/verbs) on the write path: the user-facing `counted` total — unchanged, + still what `getNounCount()` / `getVerbCount()` return — and a new ALL-visibility + `all` total covering every tier, the real denominator a derived index's own + coverage math needs. The unfiltered storage-level `totalCount` returned by + `getNouns()` / `getVerbs()` is now this unclamped ALL scalar; previously it could + only ever move up (`Math.max(scalar, scanned)`), so an inflated counter could + never self-correct. A delete that cannot prove the record it removed actually + existed (no canonical read, no prior image available) no longer decrements on + faith — it marks the ledger `suspect` (narrated once per session) instead of + silently drifting, and the next `repairIndex()` clears the flag with a real + recount. +- **One contract for a throwing health probe.** A provider's `validateInvariants()` + is documented to never throw — but if one does anyway (a bug, a transient fault), + it is now read the same way everywhere: `heal: 'none'`, the error named in the + report, never synthesized into a rebuild trigger and never swallowed into "looks + fine." A flaky check can no longer buy itself a rebuild. `repairIndex()`'s + per-family receipt also gains `missing` (an exact count plus a capped id sample), + `rebuilt` (a full rebuild ran, vs. an incremental heal), and `reason`. +- **The named health report; reads refuse instead of rebuilding.** Any index + provider may now expose a synchronous, O(1) `healthReport()` — composed from the + provider's own exact ledgers, never a sample — and this is the one signal + Brainy's read gate trusts. The first-query lazy-build path is gone: `brain.init()` + now runs every needed rebuild to completion before it returns, always, regardless + of dataset size. A read that lands on a provider whose health report says it + isn't serving throws a typed error instead of triggering a rebuild mid-query — + `GraphIndexNotReadyError`, `MetadataIndexNotReadyError`, or + `VectorIndexNotReadyError` (all exported from `@soulcraft/brainy`), naming the + reasons. `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] | 'all' })` is + the new explicit operator door: it rebuilds the named family unconditionally, no + health check consulted — reach for it when you have independent reason to + distrust a family regardless of what it self-reports. Bare `repairIndex()` is + unchanged in spirit: report-driven, heals only what its own checks say needs it. +- New concept doc: [Index Health](docs/concepts/index-health.md) walks the whole + story from a consumer's side — degraded-but-serving vs. not-ready, what + `repairIndex()` checks and heals per family, what `suspect` counts mean. + +**Nothing to change to adopt this.** No API removed, no signature narrowed — +`repairIndex()` gains an optional options bag and its return value gains fields, +both additive. The honest notes: if your code ever relied on a `find()` against a +cold/not-yet-built index quietly triggering a rebuild and returning results a beat +later, that behavior is gone — it now throws one of the three typed +`*NotReadyError` classes instead (catch them if you need to distinguish "not ready +yet" from "no results"). And `disableAutoRebuild: true` no longer defers index +construction to the first query — a needed rebuild always runs at `open()` now; +the flag has no effect on timing. Full manual control still lives in +`repairIndex({ rebuild: [...] })`. + +- **Crash-reopen catchup.** After an unclean shutdown, the metadata index now + folds the exact fact window it missed — `find()` serves every acked write on + reopen, closing the gap where canonical reads and counts recovered a + crash-window write but the index kept serving its pre-crash state until the + next full rebuild. Related root-cause fixed alongside: `close()` never + stamped the index watermarks (only `flush()` did), so a close without a + prior flush caused a needless full rescan verdict on the next open. +- **Relation rows are live in the metadata index.** Previously verb rows + entered the metadata index only during a rebuild — so a rebuilt store's + relation postings went stale from the first `relate()` after it. Relations + are now posted and retracted on the live write path (relate / unrelate / + updateRelation / remove's cascade, and their `transact()` forms), in the + same commit as the graph leg. +- **The metadata rebuild is online.** `rebuild()` for the metadata family no + longer clears and rebuilds in place (reads went empty for the duration): it + builds a complete replacement beside the serving index, mirrors concurrent + writes to both, swaps atomically, and persists once after the swap. Reads + never observe a partial index. `repairIndex({ rebuild: ['metadata'] })` uses + it automatically. +- **Incremental heal is routed.** A provider invariant that asks for the + incremental heal (`heal: 'repair'`) now routes to the provider's own + `repair()` when it exposes one — re-posting exactly what its ledger names, + never a store-sized rebuild — and the post-heal re-read of the report decides + success; a repair that doesn't converge is recorded with the escalation named. +- **The vector family joins the count ledger.** `getCanonicalCounts()` gains + `vectors: { all }` — the count of canonical entities holding a real vector + (deferred-embed entities count when their vector lands). And the open gate + closes the vector leg: a store whose canonical rows hold vectors but whose + derived vector index is empty now builds at `open()` (or refuses with the + typed error) instead of silently serving empty vector-search results. +- **An unknown storage config shape fails loudly.** A nested `config` object + carrying a path-shaped key (a shape that was never supported) used to fall + through silently to the default shared directory — every instance writing one + store while callers believed each had its own. It now throws, naming the + canonical `path` key. +- **Relation index rows are JSON-safe.** Internal endpoint identifiers can no + longer ride the metadata-index crossing (a native provider serializes it); + they stay on the graph operations where they belong. +- **A broken accelerator install can never read as "not installed."** The + auto-detection free pass now requires the resolution error to name the + accelerator package itself, exactly — a missing platform-binary sibling + package, an inner file path, or a dependency failure is a broken install and + `init()` throws loudly. And a plugin that declines activation is narrated on + the always-on log channel, so `silent: true` can no longer hide a fallback + to the default engines. + +--- + +## v10.3.1 — 2026-08-18 (the fold that behaves) + +Three recovery cures from one production first-boot incident (a brain's first +process restart after a live storage-authority flip looked hung and was +restarted three times mid-recovery). **Adopt this version before flipping +brains with existing history** — it is the intended adoption target for +fleets moving to the crash-safe authority. + +- **Recovery streams.** The boot-time log fold now consumes the generation + log one segment-batch at a time — memory stays bounded at one segment for + any log size. Previously it materialized every fact into one array, which + on a ~7k-fact log produced multi-GB allocation pressure and a process that + looked wedged while it worked. +- **Recovery narrates.** The fold announces itself before the work begins + ("recovery fold beginning — do not restart, the fold is finite") and prints + progress every thousand facts. A visible fold gets to finish; a silent one + gets killed by a well-meaning operator, and each kill makes the next boot + pay the whole fold again. +- **Bounded recovery from the flip itself.** Adopting the log authority now + founds the recovery checkpoint at the moment of the flip (one paged + canonical sync, bounded memory, then the stamp) — so even the FIRST unclean + shutdown after a flip replays only the log's tail. Previously the bound + could only establish itself at a completed crash recovery, which is exactly + the recovery the incident kept interrupting. + +--- + +## v10.3.0 — 2026-08-18 (the trust-and-provenance release) + +Four consumer-driven cures. Pairs with the same native accelerator line +(>=4.1.0); adopt alongside the accelerator's 4.2.0 for its paired fixes. + +- **Writer-lock fencing.** A live writer is never auto-evicted (staleness now + requires the holding process to be dead — a >60s stall is a slow writer, not + a dead one); the lock claim is atomic (no empty-file window a racer can + misread as torn); and every flush commit and transact barrier verifies lock + ownership first, so a forced-out or lock-deleted writer fails typed + (`BRAINY_WRITER_FENCED`) instead of writing on unaware — the split-brain + class a shared dev store hit is dead at all three roots. The documented + same-process re-open ("warn and take over") stays benign: ownership is + per-process. Consumers that raised stop-timeouts as mitigation can retire + them. +- **Transaction-log provenance.** `TxLogEntry` gains an optional `origin` + field — absent means a user write (existing consumers unchanged); + engine-originated commits stamp themselves (`system:embed-landing`, + `system:adoption-backfill`, `system:reconcile`), and the same stamp rides + the commit fact's meta. Activity feeds filter on fact instead of guessing; + a reported "double tick" (the deferred vector landing indistinguishable from + a user save) is cured without collapsing genuine rapid saves. +- **The attested reconcile door.** `reconcileLogDivergence(id, {attest})` + resolves the one adoption-refusing divergence class + (`log-live-canonical-absent`) with a human's word: `'deleted'` mints the + tombstone the log always lacked; `'restore'` folds the log's only copy back + into canonical; wrong-class calls refuse typed with nothing written. Loud, + narrated, single-row. +- **Iron-honest test budgets.** The wall-clock micro-budgets are recalibrated + as order-of-magnitude guards (3x the worst measurement across three machine + classes) so honest hardware differences can never again read as failures; + real performance enforcement lives in the dedicated perf lanes. + +--- + +## v10.2.0 — 2026-08-17 (adoption completes in one call) + +One fix, headline-sized for large stores. Pairs with the same native accelerator +version as 10.1.0 — no accelerator bump needed. + +- **The adoption backfill runs to completion.** Adopting the crash-safe storage + authority first re-commits every row the log never saw (a one-time baseline + backfill). That backfill had a fixed ceiling of 800 rows per + `adoptLogAuthority()` call — sized for small drift, not for a large pre-existing + store — so a store with a 12,700-row baseline advanced 800 rows per call and + stayed on the prior authority across restarts (a production deployment's + report). Now one call adopts a baseline of any size: the backfill sees the + entire curable set at once, cures all of it, and loops only until green — the + no-progress guard is the sole stop. Pace rides the write path (~100 rows/s + measured end to end, versus ~1.7 rows/s under the old page-per-scan shape), + and progress is narrated so an operator watching a live service sees motion. + Stores that already adopted are unaffected; stores still on the prior authority + flip in a single call on their next open or on an explicit + `adoptLogAuthority()`. +- Verification report unchanged on the wire (still lists at most 200 mismatches; + counts remain complete) — only the adoption path reads the full set. + +--- + +## v10.1.0 — 2026-08-13 (the bounded-recovery and write-path-cure release) + +The theme: **crash recovery is bounded, restores are durably founded, and two +production-reported write-path defects are cured at their roots.** Ships together +with the matching native accelerator version; adopt as a pair. + +- **Bounded crash recovery (the fold-checkpoint bound).** Recovery after an unclean + shutdown now replays only the log segment above a durably-stamped checkpoint + instead of the whole log. The checkpoint advances only after a canonical-sync + barrier makes every touched record durable (deletes included), so the bound can + lag but can never overstate durability. Existing stores converge automatically at + their first recovery — zero operator steps; recovery cost stops scaling with + store age. +- **Restores are unclean events, by construction.** `restore()` now runs its swap + fully quiesced (no background flush can race the directory replacement — a + consumer-reported `ENOTEMPTY` crash class is dead), and a snapshot's durability + stamps never survive the restore: the reopen folds the restored log, re-syncs + what it re-applied, and stamps fresh. Restored state is durably founded at + restore time instead of inheriting assertions about bytes the disk never synced. +- **Write-path cures from a production report.** (1) Log pad-frame construction is + total — a size-class boundary hole could previously kill a sync with "pad frame + not constructible". (2) The at-ack sync-failure compensation now splits by phase: + the generation counter can never re-mint a number the log may already carry, so + the non-monotonic append refusal loop reported by a downstream deployment cannot + recur. Both pinned with the reporter's exact shapes. +- **Operator-truthful sparse queries.** `where` on a field no store row has ever + carried now serves the honest answer (`eq`/`in`/range → empty; `ne`/`exists:false` + → all rows; `exists:true` → empty) with a throttled did-you-mean warning, instead + of refusing. `orderBy` on unknown fields and ambiguous spellings keep their typed + refusals. +- **Cross-package error identity.** `UnresolvableFieldError` thrown across package + boundaries is re-normalized so `instanceof` checks in consuming applications + match regardless of duplicated dependency trees. +- Release tooling: publishes now push the tag before the branch (the publish + workflow can no longer queue behind a redundant CI run) and verify registry + byte-identity with a propagation-tolerant raw-registry probe. + +--- + +## v10.0.0 — 2026-08-10 (the write-path and lifecycle release) + +The theme: **writes ack fast and honestly, startup adopts instead of rebuilding, and +every query path serves, announces, or refuses — never silently degrades.** Ships as +one release together with the matching native accelerator version. + +**The storage-authority posture (the release's headline):** a NEW brain's default +is **durable-at-ack log authority** — the generation log is the source of truth, +every write acknowledgment is covered by a group-committed fsync, and crash +recovery is a replay of the log (an acked write survives power loss, proven by +fault-injection tests). An EXISTING brain adopts at its first open under 10.0.0, +gated by a verification oracle: the log is replayed and diffed against stored +truth record-by-record; curable gaps are backfilled; the brain flips only on a +green verdict and a brain that cannot verify stays on the previous posture and +says so loudly. The explicit opt-out is `logAuthority: 'defer'` in the config +(no automatic adoption; flip later with `adoptLogAuthority()`). + +**Why a major:** the generation log gains write format v2 — new segments carry typed, +versioned records with integrity seals. A 9.x build refuses a v2 segment with a clear +version-naming error (never a misread), which means **a brain written by 10.x cannot +be opened by 9.x**. Existing v1 history stays readable forever; upgrading requires no +migration and no data touch — the format moves forward only as you write. + +### New capabilities + +- **`deferEmbedding: true`** on `add()`/`update()`: the write acks at durability; the + embedding runs on a crash-safe background worker and the vector swaps in atomically. + The row is id/metadata-findable immediately; semantic recall converges when the embed + lands. Barriers and gauges: `awaitPendingEmbeds()`, `waitForIndexed('semantic')`, + `getIndexStatus().pendingEmbeds`. VFS file writes adopt this end to end — file-write + ack no longer waits on a neural net (measured ~50× faster serial writes on a + production-shaped corpus). +- **`waitForIndexed(path?, { generation?, timeoutMs? })`** — the one honest read + barrier for write-then-recall flows. Typed timeout error naming what was still + pending; never a silent partial wait. +- **Engine-owned persistence cadence** (`persistence.policy: 'auto'`, now the default): + the engine flushes on write-count/interval/idle triggers in the background, + single-flight. **Delete `flush()` calls from hot paths** — `flush()` remains as an + awaitable durability barrier. A hung flush can never block a write ack. +- **Time-travel recall contract**: `asOf(G).find()` serves vectors exactly as they + stood at G — a later update never leaks into an earlier pin; deleted rows mask; + beyond-head pins refuse typed. +- **Log-authority storage (opt-in, per brain)**: `verifyLogAuthority()` audits the + generation log against stored truth record-by-record and names every divergence; + `adoptLogAuthority()` flips a brain to log-authoritative storage only on a green + audit (self-healing curable divergences first), enabling durable-at-ack writes: + concurrent writers share one fsync and an acked write survives power loss, by + construction (crash-recovery replay is pinned by fault-injection tests). + +### Behaviour changes + +- **`find({ where: {} })` now serves match-all** (previously returned an empty result + silently — warm and cold). Same fix applies to count, streaming, and graph-scoped + seeding paths. +- **`removeMany({ where: {} })` now refuses with a typed error** — a match-all bulk + delete must be explicit, never inherited from an empty filter object. +- **Aggregations always answer**: state persists at every `flush()` (not only close), + an unclean exit reconciles incrementally instead of rescanning the store, and + deletes without a before-image flag a loud rescan instead of silently skipping. +- **Vector updates are atomic in place** — a row is never transiently absent from + search during an update (the "flicker" class is gone); type-only re-index of an + unchanged vector is a no-op. + +### Format note + +- The generation log gains **format v2** (typed, versioned records with integrity + seals). v1 segments remain readable forever; new segments write v2. Older brainy + builds refuse v2 segments with a clear version-naming error rather than misreading + them. Records reserve encryption fields for a future release — zero behaviour today. + +## v8.11.0 — 2026-07-27 (canonical enumeration mode for export — storage-walked, canon-complete) + +From a fleet data-migration program's requirement for whole-brain exports that are +provably canon-complete: `export()`'s default enumeration for a whole-brain/predicate +selector is a generation-correct paginated `find()` walk — a projection query riding +the metadata index as an acceleration structure. Production has documented both of the +index's failure classes: a lost/stale posting can silently OMIT a canonical record from +an export, and a stale posting can silently INCLUDE a phantom row. Neither is visible +to the caller today. + +- **New: `export(selector, { enumeration: 'canonical' })`** (default remains `'index'` — + unchanged behavior on this release). Canonical mode walks every live noun/verb + directly off the storage adapter's canonical shard layout (`storage.getNouns()` / + `getVerbs()` — the same primitive `repairIndex()`'s recount and every index-heal + walk use) instead of the metadata/graph indexes, then applies the selector as a + plain predicate over the walked records. This guarantees canon-completeness — index + corruption cannot hide a live record from the export — at the cost of an O(N) walk + regardless of selector selectivity. Relations are also walked canonically in this + mode, for every selector, not just the whole-brain case. Requires the LIVE current + generation: called on a historical `asOf()` view or a speculative `with()` overlay it + throws `CanonicalEnumerationUnavailableError` rather than silently mixing generations + or missing an overlay's own entities — `enumeration: 'index'` (the default) is + unaffected and still composes with `asOf()`/`with()` as before. +- **New: `export(selector, { enumeration: 'canonical', reportIndexDrift: true })`** — + also runs the index-based enumeration and diffs it against canonical ground truth, + attaching `PortableGraph.drift: { canonicalOnly: string[], indexOnly: string[] }` + (canon-present ids the index missed; index-visible ids canon-absent — phantoms). + Migration-audit evidence, not a repair: nonzero drift is reported loudly + (`console.warn` with the counts) and nothing is auto-healed — run `brain.repairIndex()` + to reconcile the metadata index once drift is confirmed. +- **New: `export(selector, { includeHidden: true })`** (default: false — unchanged + behavior). Without it, a whole-brain/predicate export could never carry a + `visibility:'internal'` or `'system'` row, in EITHER `enumeration` mode — a real gap + for a bulk-migration fold auditing per-visibility-tier, where a hidden tier is real + user data, not noise to drop. `includeHidden` admits both tiers into candidacy in + both modes (and implies `includeSystem`; `includeSystem` alone keeps its narrower, + pre-existing meaning). **Migration-grade exports set `includeHidden: true`** — a + complete-canon export must carry every visibility tier; consumer-facing exports + leave it off. +- **Ops note (consumer-invisible): the release pipeline's home-registry publish (The + Source, source.soulcraft.com) now runs on CI**, triggered by the release tag, instead of PUTting the tarball from the laptop + over WAN — no change to what gets published or how a consumer installs it. + +## v9.0.0 — 2026-08-04 (the field-addressing law: your names and system.*, nothing in between) + +**Major.** One law now governs every field name, on every surface: + +> **Data is either in main space — where you can use ANY name — or it is in +> `system.*`.** + +Read `docs/concepts/field-addressing.md` (published on the docs site) for the +full contract; this entry is the migration ledger. + +### Breaking — query surfaces (`where` / `orderBy` / `groupBy` / aggregation) + +- **A bare field name ALWAYS addresses your metadata.** `orderBy: 'createdAt'` + no longer silently means the engine timestamp — it now refuses with a typed + `UnresolvableFieldError` naming both candidates unless you actually have a + user field of that name. Engine scalars are addressed explicitly: + `system.id`, `system.type`, `system.subtype`, `system.createdAt`, + `system.updatedAt`, `system.confidence`, `system.weight`, + `system.visibility`, `system.service`, `system.createdBy` (relations mirror + with `system.verb`/`system.sourceId`/`system.targetId`). + **Sweep list:** `where: { subtype: … }` → `where: { 'system.subtype': … }` · + `orderBy: 'createdAt'` → `'system.createdAt'` · `groupBy: ['noun']` → + `['system.type']` · any bare `visibility`/`service`/`confidence` filter that + meant the engine value → its `system.*` spelling. Every missed site fails + LOUDLY with the correction in the error message — nothing silently changes + meaning without telling you. +- **Unimplemented `find()` options refuse** (`cursor`, `includeRelations`, + `writeOnly` → `UnsupportedFindOptionError`); `order` is validated; + accepted-and-ignored is dead as a class. +- **The ordering contract is pinned cross-engine:** missing/null `orderBy` + values sort LAST in both directions, ties break by id ascending, and rows + are never dropped from an ordered read. + +### Breaking — write surfaces + +- **There are no reserved metadata names anymore.** `metadata: { confidence, + type, id, level, data, content, … }` are ordinary user fields — stored + verbatim, indexed, filterable, sortable, aggregatable, faithful across + restarts, index rebuilds, and `asOf()` time travel. The 8.x + reserved-key-in-bag throw is GONE; code that relied on it (or on the + `'warn'`/`'remap'` lift) must set engine scalars via their dedicated params + (`confidence`, `weight`, `subtype`, `visibility`, …) — the bag never touches + them now. +- **`reservedFieldPolicy` is removed.** Passing it throws at construction with + the migration note. `RESERVED_ENTITY_FIELDS`/`RESERVED_RELATION_FIELDS` + remain exported but now describe the stored record's engine half, not a ban + list; the `NoReservedEntityKeys`/`NoReservedRelationKeys` types are no-op + (deprecated). +- **The one refused spelling:** a metadata key literally starting `system.` + (namespace forgery) — typed error on `add`/`update`/`relate`/`updateRelation`. +- **Name-based index exclusions are gone.** Fields named `content`, `data`, + `id`, `vector`, … in your bag now INDEX like everything else (they were + silently un-indexed before — `where` on them returned `[]` with no error). + Value-shape rules stay, uniform across all names: arrays >10 never become + posting scalars; long values index hashed. +- **Migration transforms receive one normalized view** (engine fields + top-level, your bag nested under `metadata`) regardless of how old the + stored record is, and must return the same shape — a stray non-engine + top-level key refuses with the fix in the message. + +### Storage format (automatic, no action) + +- New/updated records persist as **nested-bag records** (engine fields + top-level, your bag verbatim under `metadata`, sealed by a format stamp) — + the shape that makes collider names lossless. Old flat records stay + readable forever; nothing rewrites your data in place. +- **Index epoch 3:** derived-index keys split the namespaces (bare user keys · + literal `system.` keys; the legacy `noun` column is gone). Every + brain rebuilds its derived indexes from canonical once, at first open — + observable via `getIndexStatus()`, no manual step. Pair this release with + the same-day native-accelerator release (its peer floor rises to `>=9`). +- Raw-record consumers (fact-log scanners, export tooling): read bags through + the exported shape-aware splitters (`splitNounMetadataRecord` / + `splitVerbMetadataRecord`) — they handle both record eras. + +### Fixed in the same train + +- Default visibility exclusion was a silent no-op under the new addressing on + pre-release builds (internal/system-tier rows could leak into default + reads) — now pinned by conformance tests at every lifecycle boundary. +- Per-type count surfaces (`getStats()`, count-by-type) read the new type + column, with a legacy fallback for pre-rebuild reads. +- Aggregation `source.where` evaluated dotted keys as nested paths — dotted + addresses now match per-key, and the internal per-type counts aggregate + rebuilds itself onto the new keys automatically. + +### Conformance + +Both engines ship a shared self-arming conformance suite (the law cases, the +ordering contract, and the reopen-collider fidelity case: every collider name +written as user data, verified verbatim through live reads, reopen, a forced +epoch rebuild, and time travel). Capability signal: +`FIELD_ADDRESSING_CAPABILITY = 'field-addressing/v1'` plus the typed error +classes, exported from the package root. + +## v8.10.3 — 2026-08-03, 8.10-line backport (natural field names stop colliding with engine internals) + +From a production report: sorting by a user metadata field named `level` silently +returned insertion order — the engine's internal HNSW node layer (also called +`level`) shadowed the user's field in every by-name read, and the indexing path +stamped a hardcoded `0` into the same index column (multi-valued poison). `level` +is a perfectly natural field name (game characters, priorities, floors); the +engine was wrong, not the caller. + +- **`level` is user data now, everywhere.** Engine plumbing no longer resolves by + name, never shadows metadata, and never enters the indexed views. `orderBy: + 'level'`, `where: { level: 9 }`, `groupBy: ['level']` all read YOUR field. + Regression pins: `tests/integration/level-field-shadow.test.ts` (the reporting + consumer's exact repro rows). +- **Index epoch 2.** The derived posting set changed, so every existing brain + rebuilds its metadata index from canonical at first open — poisoned columns + heal automatically; no manual step. First open after upgrade pays one rebuild + (observable via `getIndexStatus()`); pair this release with the same-day + native-accelerator release, which makes `level` indexable on the native path. +- **`transact()` metadata-only updates stop rewriting the vector record** — the + v8.10.2 write-granularity law now covers the batch/plan path too (it was + fixed for `update()` but the transact plan builder still staged the + unconditional save). If you batch stat touches through `transact()`, this is + your write-amplification fix. +- (The "coming next" note this entry carried shipped as v9.0.0 — the + field-addressing law above.) + +--- + +## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) + +From a production incident on a large deployment: a read-heavy sweep that bumped +per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written +in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun +record, unchanged vector included, fsynced. + +- **`update()` write granularity fixed at the core.** A metadata-only update (no new + `data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY — + the vector-bearing noun record is never rewritten. Vector-side writes and HNSW + reindexing still happen exactly when the vector side actually changed. Regression + pins: `tests/integration/update-write-granularity.test.ts`. +- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway + (one `transact()` instead of N `update()` calls) — granularity fixes the cost per + touch; batching fixes the count. +- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log + only on new traffic, at debug level). +- Native graph providers' `graph-lsm-*` storage keys are recognized as system + resources — the per-boot `Unknown key format` warning for them is gone. + +Pairs with the native accelerator's same-day patch release; adopt as one bump. + +--- + +## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) + +From a production incident: a native-provider op ground 38-40s inside a transaction, +blew the ~32s apply-phase budget, was rolled back (zero loss, by design), and a +downstream pipeline hot-retried the identical operation into a 6-minute, 100%-CPU +storm. Investigation confirmed Brainy itself never auto-retries a timed-out +transaction — the storm was entirely the consumer's own retry loop, driven by a +"retryable" doc-prose claim with no machine-readable contract to branch on. This +release closes that contract gap and, separately, fixes a real `warm()` reporting gap +surfaced by the same investigation. + +- **`TransactionTimeoutError` is now a machine-readable no-hot-retry contract.** Two + new typed, always-`true` fields replace prose-only guidance: + - `retryable: true` — the operation MAY succeed on a later attempt, once the + underlying slowness resolves or the budget is deliberately raised + (`transactionBudgetFloorMs`, or a batch's own `timeoutMs` override). + - `hotRetryUnsafe: true` — an immediate, identical retry re-pays the FULL cost of + the work that just timed out (it does not resume partway) and can cascade into + exactly the CPU storm above. **Never loop on this error.** The documented pattern + is a latch, not a retry loop: + ``` + on TransactionTimeoutError: + record { at: Date.now(), error } + rethrow loudly to your own caller + hold a cooldown window before any re-attempt + clear the latch only on a subsequent success + ``` + - `context` (unchanged, now fully documented) carries the backoff inputs: + `timeoutMs`, `operationIndex`, `elapsedMs`, `totalOperations`, `operationName`. + - Every "retryable" doc-prose site referencing this error (`transact()`'s + `timeoutMs` option, `transactionBudgetFloorMs`, `Transaction.execute()`) now + points at these fields instead of bare prose. + - Regression-pinned: the engine never internally re-drives a timed-out operation + (verified via an execution counter through both the single-op write path and + `add()`'s upsert-race retry loop), so this has always been true — it is now + provable and typed. +- **Dead code removed**: `TransactionManager.executeTransactionWithResult()` had zero + callers in this codebase and is deleted. +- **`brain.warm()`'s metadata surface now routes through the ACTIVE provider.** A + production deployment's warm report showed `metadata: 'unavailable'` under a native + metadata provider — the previous logic only duck-typed the built-in JS manager's + `hydrateAll()` method, which a native provider has no reason to implement. The + metadata provider contract (`MetadataIndexProvider`, `src/plugin.ts`) gains an + optional `warm?(): Promise` hook, mirroring the existing vector and graph + provider hooks. `brain.warm()` now checks the active provider's own `warm()` FIRST, + falls back to the JS manager's `hydrateAll()` when absent, and only reports + `'unavailable'` when neither exists — never `init()` as a stand-in, since a native + provider's `init()` may be a cheap verify rather than a real warm. A native + provider lights this surface up the same way `@soulcraft/cor` already lights the + vector and graph surfaces: implement `warm()` on its metadata provider. +- **New: `brain.maintenanceDebt()`** — the observability seam so an operator sees a + provider's outstanding background maintenance work (pending bytes/items, last pass + outcome, whether it's converging) BEFORE it grinds into the kind of budget-busting + op this release's timeout contract exists for, instead of discovering it as a CPU + storm. It is a pure passthrough: brainy applies no thresholds, no polling, and no + estimation — it calls each active provider's own optional `maintenanceDebt?()` hook + (vector, metadata, graph — the same three contracts `warm?()` lives on) and reports + the payload verbatim, or `'unavailable'` when a surface's provider doesn't track + debt. Useful as a pre-warm/post-warm check or a boot gate. `@soulcraft/cor` does not + yet implement the hook as of this release — expect it on cor's next release; until + then all three surfaces honestly report `'unavailable'`. + +## Unreleased (the warm contract: cold-restart writes stop paying demand-load latency) + +From a production deployment's cold-restart incident: the FIRST writes after every +restart on a large brain measured 33–35s each (page cache cold) against the transact +apply budget — Brainy's own op-count-scaled budget, `max(30s, opCount × 2s)`, e.g. +32,000ms for a 16-op batch. There is no external deadline in this story, and no +post-completion veto either: every write is itself a multi-operation transaction (a +single `add()` applies several operations — canonical writes, the vector-index insert, +the metadata-index update), so ONE cold operation that legitimately runs ~33s consumes +the whole budget, the gate before the NEXT operation trips, and the write rolls back +atomically (zero loss, by design) — refused, retried, refused again, until the page +cache warms passively (~30 minutes). The cure is not weaker atomicity; it is the two +new knobs below — a budget floor sized for cold stores, and a warm contract that pays +demand-load cost OFF the transaction path. + +- **The budget's start-gating contract is now explicit, documented, and pinned by + regression tests.** The budget gates STARTING the next operation — completed work is + never rolled back for elapsed time — and a transaction's first operation now + unconditionally starts by code, not merely because elapsed time happens to be ~0 when + it is checked. This has been the shipped schedule since 8.7.0 (no behavior change for + existing integrations); it is now stated in `Transaction.execute()`'s contract JSDoc + and enforced by tests so it cannot silently regress. Mid-batch atomicity is unchanged: + a trip before operation `i+1` still rolls back `0..i` and throws a retryable + `TransactionTimeoutError`. +- **The budget's 30s floor is now configurable**: `new Brainy({ transactionBudgetFloorMs })` + raises (or lowers) the floor of `max(transactionBudgetFloorMs, opCount × 2000)` for every + internal transact batch. Useful for a store whose cold writes legitimately run past 30s + per operation, so a bulk batch gets a proportionally larger runway instead of tripping + mid-batch on cold-cache latency. +- **New: `brain.warm()`** — eagerly loads/faults-in the vector index, metadata index, and + graph adjacency so the first real operation after a cold restart runs at steady-state + cost instead of paying demand-load latency on the critical path. Returns a `WarmReport` + with one honest outcome per surface — never conflate the first two: + - `'warmed'` — the surface's own provider `warm()` hook ran, or a full-hydration seam + loaded every shard/field/segment from storage. Steady-state cost is paid. + - `'probed'` — no `warm()` hook was available, so a best-effort read (one `search()` call + for the vector index) faulted in *some* backing storage as a side effect — real work, + but never reported as `'warmed'`. + - `'unavailable'` — nothing ran (no hook, no hydration seam, or nothing to probe). +- **New config: `warmOnOpen: true`** makes `init()` await `brain.warm()` before it resolves + — a deliberate blocking trade-off: startup takes longer, the first request doesn't. + Default `false` (unchanged lazy behavior). +- **New optional provider hook: `warm?(): Promise`** on the vector and graph + acceleration provider contracts (`src/plugin.ts`) — a native provider can implement it to + eagerly pretouch its own backing storage (e.g. mmap pretouch); absence means brainy falls + back to the probe/hydration behavior above. +- **Transaction op-name strings changed in journals/timings**: the vector-index + transaction operations were renamed from `AddToHNSW`/`RemoveFromHNSW` to backend-neutral + `AddToVectorIndex(...)`/`RemoveFromVectorIndex(...)` — the old names hard-coded an + algorithm that may not be the one actually running (a non-HNSW native vector provider + emitting `"RemoveFromHNSW"` has sent an operator hunting an index that doesn't exist). + The parenthesized suffix names the ACTIVE backend: `hnsw-js` for the built-in engine, or + the native provider's own identity when it self-identifies. **If you parse these op-name + strings** (log processors, journal tooling), update your matcher from + `AddToHNSW`/`RemoveFromHNSW` to `AddToVectorIndex(`/`RemoveFromVectorIndex(`. +- **Provider identity is now a REQUIRED `name` field** on the vector provider contract + (`VectorIndexProvider.name`, `src/plugin.ts`) — every implementation self-reports its own + identity truthfully (its algorithm/engine), never inheriting a default. It renders as the + op-name suffix above and, wherever the vector index identifies itself in prose log lines, + as the tag `[vector-index:]`. **Native provider adoption is a one-line change**: + declare `readonly name = ''`. A provider instance that still lacks + `name` at runtime (an older native build compiled against the previous, optional field) is + never crashed on and never silently mislabeled: it stamps `unknown-provider` and emits one + loud warning naming the missing field, so the gap is discoverable instead of a permanent + fossil label in every journal line. + +## v8.9.0 — 2026-07-19 (flush is durability-only: history maintenance moves to close()) + +The write path stops paying maintenance costs — the last structural piece of the +flush-storm class (a production deployment measured single writes blocked 25–191s behind +history reclaim running inline on flush under memory pressure): + +- **`flush()` never compacts history.** It persists the current window's deltas and + nothing else — its cost no longer depends on history backlog or retention mode, in any + configuration. **`close()` is the auto-compaction site** (time-bounded per pass, ~5s; + an early stop is a consistent prefix and the next pass resumes). +- **`compactHistory()` gains `timeBudgetMs`** — bound your own maintenance windows; the + same resumable-prefix guarantee applies. +- **The documented trade**: a long-lived writer that never closes accumulates history + until its next explicit `compactHistory()`. Predictable writes, explicit maintenance. + If you run bounded retention on an always-on service, schedule a periodic + `compactHistory({ ...caps, timeBudgetMs })` in your maintenance window. +- **New public doc: `docs/performance-envelopes.md`** — measured per-op envelopes + (p50/p95 at stated scales, hardware, and backend, with the measuring script cited). + Refresh rule going forward: any release touching a measured path re-runs that op's + benchmark and updates the envelope in the same release. +- **New in this file: the Removed APIs 7.x→8.x table** (top of this document) — every + removal with its sanctioned replacement, one place, per the engine-currency contract. + Standing from here: removals only at majors, after ≥1 minor of loud runtime deprecation. + +## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting) + +Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution +regimes where there must be one: + +- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on + delete.** The delete/update hooks fed the aggregation engine a partial entity view (type, + service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group + on the way DOWN — counts drifted upward forever after any delete, and updates that moved an + entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity + entity view (every reserved field top-level, the same shape the add path uses). If your + deployment derives stats from reserved-field aggregates, re-define those aggregates once + after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted + persisted counts do not self-heal retroactively. +- **Aggregation `source.where` on reserved fields now filters** instead of silently matching + nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level + standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says. +- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally + (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or + `ids: []` used to resolve successfully having deleted nothing. All three now throw. +- **`find()` accepts both where-key spellings.** Metadata is flattened at index time + (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now + falls back to its flattened spelling when the prefixed one isn't indexed — the + "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A + literal nested custom key named `metadata` still wins when indexed as spelled.) + +## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest) + +### The flush-storm fix (production incident, reported by a long-running deployment) + +Under the default adaptive retention, **every `flush()` re-walked the entire committed +generation history** to compute total history bytes for the budget check — O(all +generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with +70,000+ accumulated generations that turned every write into a full-tail scan (60-100s +writes), even though the budget (free-RAM-based) never tripped and nothing was ever +reclaimed. Fixed: + +- `historyBytes()` now maintains a **running total**: seeded by one walk on first use, + then updated incrementally at every commit and reclaim — the adaptive retention check + on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through + both commit paths and compaction). +- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count, + total on-disk bytes, generation/timestamp range, compaction horizon, retention mode, + and the effective adaptive budget — the one-call fleet-audit for sizing retention + exposure per brain. +- Interim guidance for keep-everything deployments already affected: `retention: 'all'` + skips the adaptive accounting entirely (and is the correct policy if you never want + history reclaimed). The accumulated files are harmless at rest; this release removes + the per-write cost of their existence. + +### The import dedup off-switch (lifecycle honesty) + +The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes +after an import, merging entities judged duplicates by id / name / vector similarity) had +three lifecycle defects, all fixed: + +- **`enableDeduplication: false` now actually disables it.** The background pass was + scheduled unconditionally — an import that explicitly opted out could still have + entities auto-removed 5 minutes later. The flag now gates BOTH the inline merge and + the background pass (regression-pinned). +- **One deduplicator per brain, owned by the brain.** Each `import()` call constructed its + own coordinator + deduplicator, so the "debounced" timer never actually debounced across + imports (N imports = N delete timers). The brain now owns a single instance — the + debounce genuinely spans imports — and `close()` cancels pending work, so a delete pass + can never fire against a closed brain. +- **The 5-minute timer is unref'd** — a pending pass no longer holds the process open + (the exit-hang class; this timer had escaped the earlier sweep). + +Retention note for keep-everything deployments: with `enableDeduplication: false` on +import calls and `retention: 'all'` in config, no engine path removes records +automatically. + +## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments) + +Small minor: brains now detect the two OS limits that bite at pool scale and warn **before** +the incident instead of during it. + +- At open (once per process, Linux-only, measurement-only), Brainy reads `RLIMIT_NOFILE` + (soft/hard, from `/proc/self/limits`) and `vm.max_map_count`, and warns loudly when either + sits below the pool-scale floors (soft NOFILE < 65 536; max_map_count < 262 144) — with the + exact raise commands (`ulimit -n` / `LimitNOFILE=` / `sysctl vm.max_map_count`). On stock + defaults the failure otherwise arrives as `EMFILE` or a failed mmap deep inside an index + open, long after the real cause stopped being visible. An unreadable limit produces **no** + warning — no measurement, no claim (non-Linux platforms stay silent). +- Exported for ops doors: `checkOsLimits()` returns the full `OsLimitsReport` + (values + warnings) programmatically, with the floors exported as constants. + +## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init) + +Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a +second writer on the same brain directory fail loudly): + +- **Lock acquisition claims atomically.** The acquire path used read-then-write, leaving a + window where two processes racing an *absent* lock could both "succeed" — and the loser + kept running unlocked, silently. The claim is now an atomic create-exclusive write + (`O_EXCL`): exactly one racer wins; the loser re-evaluates and either fails loudly with + the winner's details or performs a verified stale-takeover. Bounded retries; contention + beyond them fails loudly rather than degrading into a lockless open. +- **`BRAINY_WRITER_LOCKED` survives `init()`.** The conflict error documents a + machine-readable contract (`err.code`, `err.lockInfo` with the holder's pid/host/ + heartbeat), but init's error wrapping silently stripped both, leaving consumers a message + to regex against. The error now passes through unwrapped. + +Measured while verifying (for operators sizing audits): `brain.auditGraph()` at a +production-consumer scale of ~2,600 relationships / 800 entities costs ~0.1 s warm and +~0.5 s cold, with exact scar counting across reopen. + +## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry) + +The bulk-import ergonomics release, from a consumer's measured production incident (a serial +import on network-attached storage at ~2 s/op met a flat 30 s transaction budget): + +- **The transact apply budget now scales with the batch** — `max(30 s, opCount × 2 s)` — or + is exactly what you pass as the new `TransactOptions.timeoutMs`. A flat 30 s cap silently + limited honest bulk work to ~15 operations on slow disks while looking generous for small + batches. Internal batch paths (e.g. `removeMany` chunks) get the same scaling. +- **`TransactionTimeoutError` is a diagnosis, not just a failure**: it now reports the + operation it stopped at as `i/N` with the operation's name, elapsed vs budgeted time, and + states the batch rolled back atomically and is retryable. Its `context` carries the same + fields programmatically. +- **The transact envelope is documented** — batch sizing, budget math, chunking with + `ifAbsent` idempotency, and the precompute pattern (`embedBatch` + per-op `vector`) that + keeps model inference out of the commit path. Guide: `docs/guides/optimistic-concurrency.md`. +- Note: `brain.embed()` / `brain.embedBatch()` (the precompute APIs) already ship — public, + with native-provider passthrough, verified end-to-end (batch and single paths produce + bit-identical vectors; a vector-supplied `add` is fully searchable). Honest measurement: + on the default WASM engine, batch throughput ≈ sequential (~160 ms/text) — the precompute + win is keeping inference out of the budgeted commit path, not raw embedding speed. + +## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument) + +A minor release adding one new public API, from the fleet's graph-trust program: a read-only +audit that **proves whether relationship reads return stored truth** on a given brain. + +- **`brain.auditGraph(options?)`** walks every canonical relationship record, queries the same + read path applications use (`related()` / VFS `readdir`) with all visibility tiers included, + and classifies every discrepancy into its failure family: `missingFromReads` (records the + read path omits — a stale adjacency index), `danglingEndpoints` (relationships whose endpoint + entity no longer exists — the historical partial-delete scar class), and `readOnlyVerbIds` + (read-path edges with no stored record — ghosts). Design-hidden internal/system edges are + counted separately so intentional hiding is never misclassified as loss. Counts are exact; + example lists cap at `maxExamples` with an explicit `truncatedExamples` flag; the result is + narrated loudly on incoherence. Mutates nothing — safe on a live brain. +- The operational pairing: audit → if incoherent, `repairIndex()` → audit again. A `coherent` + report after the repair is the verified all-clear. Run it after any engine upgrade, restore, + or migration. Guide: `docs/guides/inspection.md`. +- Also: `getNounIds` pagination now refuses an undecodable resume cursor loudly (the same + contract `getNouns`/`getVerbs` gained in 8.5.2 — the third and final walk brought under it). +- Types exported: `GraphAuditReport`, `GraphAuditDiscrepancy`. + +## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud) + +Hardening patch from a migration incident (a byte-copied store on new hardware; the service +entered a silent full-CPU loop at boot). Four changes, all in the aggregation engine's +backfill/adoption path: + +- **Backfill walks are exception-safe and non-destructive.** A rescan now builds into a + staging map and swaps in atomically on completion; a mid-walk failure drops the staging map, + keeps the previous live state serving, and surfaces the storage error to the failing query. + Previously the walk wiped live state *before* a scan that could throw, never cleared the + pending flag on failure, and re-ran a full walk on every subsequent query — a silent + wipe/walk/throw loop at the caller's retry rate. +- **Failed walks are latched.** After a walk fails, retries within a 30-second cooldown rethrow + the recorded error instantly instead of re-walking — a tight caller-side retry loop now costs + one loud error per query, never a full store walk per query. +- **Adoption is generation-verified.** Persisted aggregation state is stamped with the store's + committed generation at flush; reopen adoption requires the stamp to equal the current + watermark. Stale state (unclean shutdown) or over-counting state (a fact-log truncation on a + copied store pulled the watermark back) triggers exactly one loud rescan — never a silent + adopt. Pre-8.5.2 state on generation-aware stores rescans once after upgrade, then is stamped. +- **The path narrates.** Adoption decisions, no-adoptable-state outcomes, walk start/finish + (entity count + duration), and walk failures all log by default; a non-advancing storage + pagination cursor aborts the walk loudly instead of looping forever. + +Plus three guards from a full audit of every loop in the open/init path: + +- **Invalid pagination cursors fail loudly.** A supplied-but-undecodable resume token to + `getNouns`/`getVerbs` used to silently restart the walk at offset 0 — to a `while(hasMore)` + caller that re-serves page 1 forever (an unbounded silent CPU loop). It now throws with a + clear message instead. +- **The graph cold-load verb walk has a stall guard.** `hasMore=true` with a missing or + non-advancing cursor aborts loudly instead of re-reading the same page forever. +- **A derived index AHEAD of the store is named at open.** Brainy already surfaced a provider + generation *behind* the committed watermark; the *ahead* direction (the signature of a + byte-copy of a live service, or a log truncation during crash recovery) now logs a loud + warning explaining what happened and that `brain.repairIndex()` forces a heal — instead of + passing unnamed into whatever the derived index does next. + +## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed) + +Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box +while vector search stayed fast, and every `find({ limit: 5000 })` suddenly failing against an +"auto-configured query limit of 1000"). Three fixes, one cosmetic: + +- **Aggregation state is actually adopted on reopen.** The boot pattern `defineAggregate()` → + query raced the engine's async state load: the synchronous define always won, flagged a + backfill, and the first query then wiped the just-loaded persisted state and re-walked the + entire store — every restart, forever. Reopening with an unchanged definition now adopts the + persisted state directly (zero scans); a backfill runs only on a real definition change, a + missing/failed state load, or a write that landed before adoption (exactness wins). Apps that + rely on persisted definitions without re-defining at boot also no longer race a spurious + "Aggregate not defined". +- **Backfills are single-flight and batched.** Concurrent queries on a cold aggregate used to + each wipe the others' partial state and start their own full store walk — under steady query + arrival the store never converged (the 40–90 s loop). Now all concurrent queries share one + walk, and one walk fills every aggregate pending backfill (M aggregates ≠ M scans). +- **The query-cap "learning" ratchet is removed.** `maxLimit` is a memory-protection bound, but + a hidden tuner shrank it 20 % per recorded query while the lifetime-average query time + exceeded 1 s — down to a floor of 1000, below the documented 10 000 auto floor, with no + practical recovery, and the resulting error blamed "available free memory" (stale basis + label). The cap now comes from its construction-time basis (or your explicit + `maxQueryLimit` / `reservedQueryMemory`) alone and never changes at runtime; query timing is + recorded for diagnostics only. +- **Cosmetic:** the engine's own persistence keys (`__aggregation_*`, `brainy:entityIdMapper`) + no longer log `[Storage] Unknown key format` at boot — they were always routed correctly; + they're now recognized before the warning fires. + +Operationally: if a host was bitten, upgrading and restarting is the whole fix — no repair +ritual needed. Setting `maxQueryLimit` explicitly remains the valve that bypasses auto-detection +entirely. + +## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier) + +Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass: + +- **Index providers can now reach the fact log through the storage adapter** — new optional + capability `storage.scanFacts()` / `storage.factLogHeadGeneration()` / `storage.factSegmentPaths()`, + wired by the host brain at init as a closure over its live log. Providers hold only `storage` and + must never construct their own fact-log reader (the log's open path is writer-side); `null` means + "no fact log here — use the enumeration walk." +- **The family-stamp verifier is shared** via `@soulcraft/brainy/internals` + (`readFamilyStamp` / `writeFamilyStamp` / `verifyFamilyStamp` + types) so first-party native + providers run literally the same verification function, never a synchronized copy. +- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a + per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass. +- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider + compares its stamp's `sourceGeneration` against the store's truth without parsing the private + manifest format. +- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for + `transact()` today; the single-op path is pinned as the documented future target — group commit + becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan + snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate + beneath it). + +No behavior change for applications; all additions. + +## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream) + +A minor release, fully backward-compatible (all additions; no behavior change for existing APIs). +This is infrastructure: it changes nothing about how you query today, and lays the substrate that +makes index heals and incremental catch-up sequential-read problems instead of directory walks. + +- **Every committed write now also appends a "fact" — an after-image commit record.** Alongside the + existing before-image history, each committed generation (single-op and `transact()` alike) appends + what each touched entity/relationship *became* — or a body-less tombstone for a removal — to an + append-only, checksummed segment log under `_generations/facts/`. Crash-safe by construction: a + torn tail is detected and ignored; on open the log is reconciled to committed truth, so an absent + generation always means "never committed." `transact()` facts are durable when `transact()` + returns; single-op facts ride the same group-commit flush as their history. + +- **New: `brain.scanFacts()`** — stream committed facts in commit order, in batches, with heal-grade + telemetry (total scope up front; per-batch generation range, byte size, and segment; a summary + cross-check at the end; loud abort on any gap — never a silent skip). **`brain.factSegmentPaths()`** + hands zero-copy consumers the immutable sealed segment files directly. New exported types: + `CommitFact`, `FactOp`, `FactScanBatch`, `FactScanHandle`. Facts accumulate from the first write + after upgrading — pre-existing history is not retroactively converted (enumeration remains the + fallback for old data). + +- **New: the entity-tree family stamp.** At every flush/close, brainy stamps which committed + generation the canonical entity files reflect plus the rollup invariants (entity/relationship + counts) that verify the tree whole. At open, coherence is a comparison — a genuine divergence is + loud and names the failing invariant; `repairIndex()` recounts from canonical and re-stamps. New + exports: `readFamilyStamp`, `verifyFamilyStamp`, `ENTITY_TREE_STAMP_PATH`, `FamilyStamp` types. + +- **Storage adapters** gain optional binary raw-byte primitives (`appendRawBytes`, `readRawBytes`, + `writeRawBytes`, `rawByteSize`) — feature-detected; the filesystem and memory adapters implement + them; an adapter without them simply hosts no fact log. The fact-log namespace is registered as a + protected family: no sweeper or GC can delete under it. + +No API breaks. 24 new tests; the full commit-path regression suite is green. + ## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory) One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..91d40d49 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,36 @@ +# Security Policy + +## Reporting a vulnerability + +Email **security@soulcraft.com**. That's the one door for security reports +across the company, and it works the same way for Brainy: every report is +read by a human, you'll get a private receipt, and we'll work with you on +coordinated disclosure — please don't open a public issue for anything +that isn't already public. + +Include what you'd want if you were on the other end: affected version, +how to reproduce, and what you think the impact is. If you have a patch or +a suggested fix, send it along — it's welcome but not required. + +There is no bounty program today. We're saying that plainly so you know +what to expect going in. + +## Response time + +We respond as fast as truth allows. That means: no fixed SLA, no promise of +a reply within a specific number of hours — but a real report from a real +person gets read promptly and taken seriously. If you haven't heard anything +in a reasonable stretch, a follow-up email is completely fine. + +## Supported versions + +The latest `8.x` minor release line receives security fixes. If you're +running an older major version, please upgrade before reporting — we can't +commit to backporting fixes to unsupported lines. + +## Scope + +This policy covers the `@soulcraftlabs/brainy` package itself — the code in +this repository. If you're evaluating a deployment that also uses +`@soulcraft/cor`, report issues in that package the same way, to the same +address; we'll route internally. diff --git a/bin/brainy-ts.js b/bin/brainy-ts.js index 4e9aedb8..90a35e98 100644 --- a/bin/brainy-ts.js +++ b/bin/brainy-ts.js @@ -3,7 +3,7 @@ /** * Modern TypeScript CLI Runner * - * This is the entry point after npm install @soulcraft/brainy + * This is the entry point after npm install @soulcraftlabs/brainy * It runs the compiled TypeScript CLI code */ diff --git a/bun.lock b/bun.lock index c31b3865..1e3e66e2 100644 --- a/bun.lock +++ b/bun.lock @@ -3,7 +3,7 @@ "configVersion": 0, "workspaces": { "": { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", "@azure/identity": "^4.0.0", diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md index 4134ae63..b2d22fb1 100644 --- a/docs/DEVELOPER_LEARNING_PATH.md +++ b/docs/DEVELOPER_LEARNING_PATH.md @@ -25,13 +25,13 @@ ### Prerequisites ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Your First Neural Database ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // Step 1: Create and initialize Brainy const brain = new Brainy({ @@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou ### Building a Knowledge Graph ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**. ### Triple Intelligence in Action ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in ### Files as Intelligent Entities ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**. ### Production-Ready Deployment ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // 1. PRODUCTION STORAGE - Filesystem with off-site snapshots console.log('Initializing production storage...\n') diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 1cc38ce9..77fbbd79 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -1217,7 +1217,7 @@ where: { await brain.find({ type: 'Document' }) // ✅ Correct: Use NounType enum -import { NounType } from '@soulcraft/brainy' +import { NounType } from '@soulcraftlabs/brainy' await brain.find({ type: NounType.Document }) // ❌ Error: Operator not recognized diff --git a/docs/MIGRATION-V3-TO-V4.md b/docs/MIGRATION-V3-TO-V4.md index 29c409ac..680b6928 100644 --- a/docs/MIGRATION-V3-TO-V4.md +++ b/docs/MIGRATION-V3-TO-V4.md @@ -153,13 +153,13 @@ brainy-data/ ### Step 1: Update Brainy Package ```bash -npm install @soulcraft/brainy@latest +npm install @soulcraftlabs/brainy@latest ``` **Check your version:** ```bash -npm list @soulcraft/brainy -# Should show: @soulcraft/brainy@4.0.0 +npm list @soulcraftlabs/brainy +# Should show: @soulcraftlabs/brainy@4.0.0 ``` ### Step 2: No Code Changes Required! ✅ @@ -374,7 +374,7 @@ If you encounter issues, you can rollback: ```bash # Reinstall v3 -npm install @soulcraft/brainy@^3.50.0 +npm install @soulcraftlabs/brainy@^3.50.0 # Restart application ``` @@ -389,7 +389,7 @@ rm -rf ./data cp -r ./data-backup ./data # Reinstall v3 -npm install @soulcraft/brainy@^3.50.0 +npm install @soulcraftlabs/brainy@^3.50.0 ``` ## Common Migration Scenarios @@ -539,7 +539,7 @@ console.log('Storage type:', status.type) **Migration Checklist:** - ✅ Backup data -- ✅ Update npm package (`npm install @soulcraft/brainy@latest`) +- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`) - ✅ Restart application (automatic migration) - ✅ Verify data integrity - ✅ Enable lifecycle policies diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 248a2c70..b543e84a 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -323,58 +323,24 @@ Only the graph adjacency index carries a committed scale assertion: - ✅ **Single-Node by Design**: One process owns one `path`; scale out at the service layer - ✅ **Zero Stubs**: Every line of code is production-ready -## Lazy Loading Performance +## Index Build at Open (10.4+) -Brainy supports two initialization modes for optimal performance across different use cases: +As of 10.4, `brain.init()` runs every needed index rebuild to completion before +it returns — always, regardless of dataset size. There is no lazy, +first-query rebuild path: a brain either finishes opening healthy, or `init()` +fails loudly. `disableAutoRebuild` no longer defers index construction to a +first query; it has no effect on *when* a rebuild runs. Manual control over +rebuilds is `repairIndex({ rebuild: [...] })`. See +[Index Health](concepts/index-health.md) for the full read-gate contract +(providers self-report readiness via `healthReport()`; a read against a +not-serving provider throws a typed `*NotReadyError` rather than rebuilding +mid-query). -### Mode 1: Auto-Rebuild (Default) - -```javascript -const brain = new Brainy() -await brain.init() // Rebuilds indexes during init (~500ms-3s for 10K entities) -``` - -**Performance:** -- Init time: 500ms-3s (depends on dataset size) -- First query: Instant (indexes already loaded) -- Use case: Traditional applications, long-running servers - -### Mode 2: Lazy Loading - -```javascript -const brain = new Brainy({ disableAutoRebuild: true }) -await brain.init() // Returns instantly (0-10ms) - -const results = await brain.find({ limit: 10 }) // First query triggers rebuild (~50-200ms) -const more = await brain.find({ limit: 100 }) // Subsequent queries instant (0ms check) -``` - -**Performance:** -- Init time: 0-10ms (instant) -- First query: 50-200ms (includes index rebuild for 1K-10K entities) -- Subsequent queries: 0ms check (instant) -- Concurrent queries: Wait for same rebuild (mutex prevents duplicates) - -**Concurrency Safety:** -```javascript -// 100 concurrent queries immediately after init -await brain.init() - -const promises = Array.from({ length: 100 }, () => - brain.find({ limit: 10 }) -) - -const results = await Promise.all(promises) -// ✅ Only 1 rebuild triggered (mutex) -// ✅ All 100 queries return correct results -// ✅ Total time: ~60ms (not 6000ms!) -``` - -**Use Cases for Lazy Loading:** -- **Serverless/Edge**: Minimize cold start time (0-10ms init) -- **Development**: Faster restarts during development -- **Large datasets**: Defer index loading until needed -- **Read-heavy workloads**: Writes don't wait for index rebuild + ## Zero Configuration Required @@ -384,10 +350,6 @@ Brainy is designed to be **smart enough to tune itself dynamically**. No configu // That's it. Brainy handles everything. const brain = new Brainy() await brain.init() - -// Or with lazy loading for serverless -const brain = new Brainy({ disableAutoRebuild: true }) -await brain.init() // Instant (0-10ms) ``` ### Automatic Self-Tuning @@ -395,7 +357,6 @@ await brain.init() // Instant (0-10ms) - **Metadata Index**: Auto-builds sorted indices for range queries on first use - **Graph Index**: Auto-flushes every 30 seconds - **Default Tuning**: Research-based vector index defaults -- **Lazy Loading**: Indices built only when needed - **Cache Management**: LRU caches with TTL ### Intelligent Defaults diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index d9a4d3e7..238d2252 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -10,7 +10,7 @@ next: - guides/storage-adapters --- -# Plugin Development Guide +# Plugin System Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cor` provides optional native acceleration, and it's the same system available to any developer. @@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement ### 1. Implement the `BrainyPlugin` interface ```typescript -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' const myPlugin: BrainyPlugin = { name: 'my-brainy-plugin', // Must be unique (typically your npm package name) @@ -90,7 +90,7 @@ await brain.init() **Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`: ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import myPlugin from './my-plugin.js' const brain = new Brainy() @@ -200,15 +200,30 @@ members so a warm reopen never pays a redundant rebuild-from-canonical: - **`init?(): Promise`** — eager cold-load. Brainy awaits it once during `brain.init()`, after the metadata provider's `init()` (the id-mapper hydrates first) and **before the rebuild gate**. -- **`isReady?(): boolean`** — honest durability signal. `true` ⇔ the persisted index is - loaded (or cheaply demand-loadable) and consistent with what was last persisted. When - exposed, the rebuild gate defers to this signal **instead of** the `size() === 0` / - `totalEntries === 0` heuristics — a disk-native index may report 0 resident entries - while fully durable. Never return `true` if the durable state failed to load: the - signal is honest in both directions, and a not-ready provider gets its rebuild even - when `size() > 0`. +- **`healthReport?(): HealthReport`** — the PREFERRED signal (10.4+). A named, + synchronous, O(1) verdict derived from the provider's own exact ledgers — never a + sample, never I/O, must never throw for a well-formed provider. Brainy's read gate + (`assessProviderHealth()`) reads this INSTEAD of `isReady()` / size heuristics when + present: `serving: false` refuses the read with a typed `*NotReadyError` rather than + triggering a rebuild — a read never starts a store walk. `healthy` marks every + *verified* invariant holding; a family named in `unledgered` counts as neither + healthy nor broken. See `HealthReport` / `LedgerInvariantResult` / + `InvariantSource` in `src/plugin.ts`, and + [Index Health](concepts/index-health.md) for the consumer-facing story. +- **`isReady?(): boolean`** — honest durability signal, the fallback when + `healthReport()` is absent. `true` ⇔ the persisted index is loaded (or cheaply + demand-loadable) and consistent with what was last persisted. When exposed, the + gate defers to this signal **instead of** the `size() === 0` / `totalEntries === 0` + heuristics — a disk-native index may report 0 resident entries while fully durable. + Never return `true` if the durable state failed to load: the signal is honest in + both directions, and a not-ready provider gets its rebuild even when `size() > 0`. - **`isMigrating?(): boolean`** — while `true`, the provider owns its index (background migration); brainy skips its rebuild entirely. +- **`validateInvariants?(): Promise`** — the async DEEP + diagnostic (full scans allowed), distinct from the bounded, sync `healthReport()`. + Must never throw — a failure is `healthy: false` data, not an exception; a provider + that throws anyway is read as a loud, unverified failure (never as "healthy") by + every caller, never silently retried into a rebuild. Providers that implement none of these keep the size/count heuristics — correct for engines whose `rebuild()` *is* their load path (like brainy's built-in JS vector index). @@ -257,10 +272,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor #### `cache` **Type:** `UnifiedCache` -Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`). +Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`). ```typescript -import type { UnifiedCache } from '@soulcraft/brainy/internals' +import type { UnifiedCache } from '@soulcraftlabs/brainy/internals' context.registerProvider('cache', myNativeCache) ``` @@ -310,8 +325,8 @@ Plugins can register custom storage backends that users reference by name. ### Implementing a Storage Adapter ```typescript -import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin' -import type { StorageAdapter } from '@soulcraft/brainy' +import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin' +import type { StorageAdapter } from '@soulcraftlabs/brainy' class MyStorageAdapter implements StorageAdapter { async init(): Promise { /* ... */ } @@ -345,9 +360,9 @@ Brainy provides three entry points for plugin developers: | Import Path | Contents | Stability | |-------------|----------|-----------| -| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) | -| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | -| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | +| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) | +| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | +| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | ## Diagnostics @@ -425,7 +440,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations ```typescript // simd-distance-plugin/src/plugin.ts -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' // Hypothetical native module import { simdCosineDistance } from './native.js' @@ -455,7 +470,7 @@ export default simdDistancePlugin "main": "./dist/plugin.js", "types": "./dist/plugin.d.ts", "peerDependencies": { - "@soulcraft/brainy": ">=7.0.0" + "@soulcraftlabs/brainy": ">=7.0.0" } } ``` @@ -463,7 +478,7 @@ export default simdDistancePlugin Usage: ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) await brain.init() diff --git a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md index 4568cd31..ad4a4a40 100644 --- a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md +++ b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md @@ -54,7 +54,7 @@ After 40 API calls: ```typescript // server.ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // SINGLETON INSTANCE let brainInstance: Brainy | null = null @@ -174,7 +174,7 @@ process.on('SIGTERM', async () => { ```typescript // server.ts - Clean Bun implementation -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain: Brainy | null = null diff --git a/docs/README.md b/docs/README.md index 3290001f..ddb37d20 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,7 +5,7 @@ ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md index 94c6a6cb..4b120bd8 100644 --- a/docs/RELEASE-GUIDE.md +++ b/docs/RELEASE-GUIDE.md @@ -99,7 +99,7 @@ Examples: ```bash # 1. Deprecate wrong version on npm -npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y" +npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y" # 2. Fix version in package.json # 3. Republish correct version diff --git a/docs/SCALING.md b/docs/SCALING.md index e9ae1136..054d2096 100644 --- a/docs/SCALING.md +++ b/docs/SCALING.md @@ -13,7 +13,7 @@ ### In-Memory ```typescript -import Brainy from '@soulcraft/brainy' +import Brainy from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) ``` @@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the -open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native +open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`. `find()` query latency, p50 / p95 (200 queries each): diff --git a/docs/api-contract.json b/docs/api-contract.json new file mode 100644 index 00000000..aafd838a --- /dev/null +++ b/docs/api-contract.json @@ -0,0 +1,1544 @@ +{ + "contractVersion": 1, + "engine": "@soulcraftlabs/brainy", + "compatibility": { + "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", + "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" + }, + "doors": [ + { + "name": "adaptiveHistoryBudgetBytes", + "kind": "method", + "arity": 1 + }, + { + "name": "add", + "kind": "method", + "arity": 1 + }, + { + "name": "addMany", + "kind": "method", + "arity": 1 + }, + { + "name": "adoptLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "adoptLogAuthorityInner", + "kind": "method", + "arity": 0 + }, + { + "name": "aggViewFromEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "anyProviderMigrating", + "kind": "method", + "arity": 0 + }, + { + "name": "applyFusionScoring", + "kind": "method", + "arity": 2 + }, + { + "name": "applyGraphConstraints", + "kind": "method", + "arity": 2 + }, + { + "name": "armIdleFlushTimer", + "kind": "method", + "arity": 2 + }, + { + "name": "asOf", + "kind": "method", + "arity": 2 + }, + { + "name": "assertGenerationStoreReady", + "kind": "method", + "arity": 1 + }, + { + "name": "assertWritable", + "kind": "method", + "arity": 1 + }, + { + "name": "audit", + "kind": "method", + "arity": 0 + }, + { + "name": "auditGraph", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAdoptLegacyVfsBlobsIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAlpha", + "kind": "method", + "arity": 1 + }, + { + "name": "autoCompactHistory", + "kind": "method", + "arity": 0 + }, + { + "name": "awaitMigrationLock", + "kind": "method", + "arity": 1 + }, + { + "name": "awaitPendingEmbeds", + "kind": "method", + "arity": 0 + }, + { + "name": "backfillAggregateIfNeeded", + "kind": "method", + "arity": 1 + }, + { + "name": "batchGet", + "kind": "method", + "arity": 2 + }, + { + "name": "brainWideStrictRequiresSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "bridgeLegacyPendingEmbedSidecars", + "kind": "method", + "arity": 0 + }, + { + "name": "buildAtGenerationVectors", + "kind": "method", + "arity": 2 + }, + { + "name": "buildGraphView", + "kind": "method", + "arity": 4 + }, + { + "name": "buildMetadataFilter", + "kind": "method", + "arity": 1 + }, + { + "name": "buildMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "buildRelationMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "cacheVerbInt", + "kind": "method", + "arity": 2 + }, + { + "name": "canServeVectorAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "checkHealth", + "kind": "method", + "arity": 0 + }, + { + "name": "checkMigrations", + "kind": "method", + "arity": 0 + }, + { + "name": "clear", + "kind": "method", + "arity": 0 + }, + { + "name": "clearPendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "close", + "kind": "method", + "arity": 0 + }, + { + "name": "closeDurableSteps", + "kind": "method", + "arity": 0 + }, + { + "name": "cluster", + "kind": "method", + "arity": 1 + }, + { + "name": "collectProviderInvariants", + "kind": "method", + "arity": 0 + }, + { + "name": "compactHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "consumeMetadataWatermarkVerdict", + "kind": "method", + "arity": 1 + }, + { + "name": "convertMetadataToEntity", + "kind": "method", + "arity": 2 + }, + { + "name": "convertNounToEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "counts", + "kind": "accessor" + }, + { + "name": "createIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "createMigrationBackupIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "createPinnedDb", + "kind": "method", + "arity": 1 + }, + { + "name": "createResult", + "kind": "method", + "arity": 4 + }, + { + "name": "dbFinalizationRegistry", + "kind": "accessor" + }, + { + "name": "dbHost", + "kind": "accessor" + }, + { + "name": "defineAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "detectIdKind", + "kind": "method", + "arity": 3 + }, + { + "name": "diagnostics", + "kind": "method", + "arity": 0 + }, + { + "name": "diff", + "kind": "method", + "arity": 2 + }, + { + "name": "embed", + "kind": "method", + "arity": 1 + }, + { + "name": "embedBatch", + "kind": "method", + "arity": 2 + }, + { + "name": "emitCommitted", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnAdd", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnRelate", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceTrackedFieldValues", + "kind": "method", + "arity": 2 + }, + { + "name": "enhanceNLPResult", + "kind": "method", + "arity": 2 + }, + { + "name": "enqueuePendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "ensureAggregationIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureIndexesLoaded", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureInitialized", + "kind": "method", + "arity": 1 + }, + { + "name": "entityForAggFromRawRecord", + "kind": "method", + "arity": 1 + }, + { + "name": "entityFromGenerationRecord", + "kind": "method", + "arity": 3 + }, + { + "name": "entityIntsToUuids", + "kind": "method", + "arity": 1 + }, + { + "name": "entityViewFromRawRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "excludedVisibilityTiers", + "kind": "method", + "arity": 1 + }, + { + "name": "executeGraphSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeProximitySearch", + "kind": "method", + "arity": 1 + }, + { + "name": "executeTextSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeVectorSearch", + "kind": "method", + "arity": 3 + }, + { + "name": "explain", + "kind": "method", + "arity": 1 + }, + { + "name": "export", + "kind": "method", + "arity": 0 + }, + { + "name": "extract", + "kind": "method", + "arity": 2 + }, + { + "name": "extractConcepts", + "kind": "method", + "arity": 2 + }, + { + "name": "extractEntities", + "kind": "method", + "arity": 2 + }, + { + "name": "factSegmentPaths", + "kind": "method", + "arity": 1 + }, + { + "name": "fieldCountsAggregateName", + "kind": "method", + "arity": 1 + }, + { + "name": "fillSubtypes", + "kind": "method", + "arity": 1 + }, + { + "name": "filterIdsBelted", + "kind": "method", + "arity": 2 + }, + { + "name": "find", + "kind": "method", + "arity": 1 + }, + { + "name": "findAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "findDuplicates", + "kind": "method", + "arity": 1 + }, + { + "name": "findMatchingWords", + "kind": "method", + "arity": 3 + }, + { + "name": "flush", + "kind": "method", + "arity": 0 + }, + { + "name": "formatInfo", + "kind": "method", + "arity": 0 + }, + { + "name": "formatSubtypeError", + "kind": "method", + "arity": 1 + }, + { + "name": "generation", + "kind": "method", + "arity": 0 + }, + { + "name": "generationDigest", + "kind": "method", + "arity": 1 + }, + { + "name": "get", + "kind": "method", + "arity": 2 + }, + { + "name": "getActivePlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "getAvailableFields", + "kind": "method", + "arity": 0 + }, + { + "name": "getBackgroundDeduplicator", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsForType", + "kind": "method", + "arity": 1 + }, + { + "name": "getFieldStatistics", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsWithCardinality", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldValues", + "kind": "method", + "arity": 1 + }, + { + "name": "getIndexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getIndexStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "getMemoryStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getNeighborUuids", + "kind": "method", + "arity": 2 + }, + { + "name": "getNounCount", + "kind": "method", + "arity": 0 + }, + { + "name": "getOptimalQueryPlan", + "kind": "method", + "arity": 1 + }, + { + "name": "getStats", + "kind": "method", + "arity": 1 + }, + { + "name": "getStorageType", + "kind": "method", + "arity": 0 + }, + { + "name": "getSubtypeRule", + "kind": "method", + "arity": 1 + }, + { + "name": "getTripleIntelligence", + "kind": "method", + "arity": 0 + }, + { + "name": "getTypedNeighbors", + "kind": "method", + "arity": 4 + }, + { + "name": "getVerbCount", + "kind": "method", + "arity": 0 + }, + { + "name": "graph", + "kind": "accessor" + }, + { + "name": "graphAccelerationProvider", + "kind": "method", + "arity": 0 + }, + { + "name": "graphCommunities", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphEntityInt", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExport", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphPath", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathFallback", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathNative", + "kind": "method", + "arity": 4 + }, + { + "name": "graphRank", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraphFallback", + "kind": "method", + "arity": 4 + }, + { + "name": "graphSubgraphFromQuery", + "kind": "method", + "arity": 5 + }, + { + "name": "graphSubgraphNative", + "kind": "method", + "arity": 5 + }, + { + "name": "groupByLabel", + "kind": "method", + "arity": 2 + }, + { + "name": "hasStorageMethod", + "kind": "method", + "arity": 1 + }, + { + "name": "hasVectorOrTextCriteria", + "kind": "method", + "arity": 1 + }, + { + "name": "health", + "kind": "method", + "arity": 0 + }, + { + "name": "highlight", + "kind": "method", + "arity": 1 + }, + { + "name": "highlightSemanticPhase", + "kind": "method", + "arity": 5 + }, + { + "name": "history", + "kind": "method", + "arity": 2 + }, + { + "name": "historyStats", + "kind": "method", + "arity": 0 + }, + { + "name": "hub", + "kind": "accessor" + }, + { + "name": "hydrateIdMapperForGraphRebuild", + "kind": "method", + "arity": 0 + }, + { + "name": "hydrateNativeSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "import", + "kind": "method", + "arity": 2 + }, + { + "name": "importPluginPackage", + "kind": "method", + "arity": 1 + }, + { + "name": "incidentEdges", + "kind": "method", + "arity": 3 + }, + { + "name": "indexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "init", + "kind": "method", + "arity": 1 + }, + { + "name": "insights", + "kind": "method", + "arity": 0 + }, + { + "name": "isEmbeddingReady", + "kind": "method", + "arity": 0 + }, + { + "name": "isInfrastructureWrite", + "kind": "method", + "arity": 1 + }, + { + "name": "isInitialized", + "kind": "accessor" + }, + { + "name": "isReadOnly", + "kind": "accessor" + }, + { + "name": "kickBackgroundFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "kickEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "legacyLayoutMigrationPhase", + "kind": "method", + "arity": 0 + }, + { + "name": "loadAnalyticsGraph", + "kind": "method", + "arity": 1 + }, + { + "name": "loadPlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "logAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "maintenanceDebt", + "kind": "method", + "arity": 0 + }, + { + "name": "materializeAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "metadataIndexRetractionOp", + "kind": "method", + "arity": 3 + }, + { + "name": "migrate", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateField", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateInternal", + "kind": "method", + "arity": 2 + }, + { + "name": "migrateLegacyZeroNormVfsRootIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "migrationSnapshot", + "kind": "method", + "arity": 0 + }, + { + "name": "neededFamiliesMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "neighbors", + "kind": "method", + "arity": 2 + }, + { + "name": "newId", + "kind": "method", + "arity": 0 + }, + { + "name": "nlp", + "kind": "method", + "arity": 0 + }, + { + "name": "normalizeConfig", + "kind": "method", + "arity": 1 + }, + { + "name": "noteWriteForPersistence", + "kind": "method", + "arity": 0 + }, + { + "name": "now", + "kind": "method", + "arity": 0 + }, + { + "name": "onChange", + "kind": "method", + "arity": 1 + }, + { + "name": "pagination", + "kind": "accessor" + }, + { + "name": "parseMigrationPath", + "kind": "method", + "arity": 1 + }, + { + "name": "parseNaturalQuery", + "kind": "method", + "arity": 1 + }, + { + "name": "pathExists", + "kind": "method", + "arity": 2 + }, + { + "name": "pendingEmbedCount", + "kind": "method", + "arity": 0 + }, + { + "name": "performInit", + "kind": "method", + "arity": 1 + }, + { + "name": "persistPinnedGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "persistSingleOp", + "kind": "method", + "arity": 6 + }, + { + "name": "pickMetadataProbe", + "kind": "method", + "arity": 1 + }, + { + "name": "pickVectorProbe", + "kind": "method", + "arity": 0 + }, + { + "name": "pinGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "planGetEntity", + "kind": "method", + "arity": 3 + }, + { + "name": "planTransact", + "kind": "method", + "arity": 1 + }, + { + "name": "planTxAdd", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRemove", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUnrelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUpdate", + "kind": "method", + "arity": 3 + }, + { + "name": "projectionGauges", + "kind": "method", + "arity": 0 + }, + { + "name": "providerForFamily", + "kind": "method", + "arity": 1 + }, + { + "name": "providerIsMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "providerMigrationStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "queryAggregate", + "kind": "method", + "arity": 2 + }, + { + "name": "queryIndexFamilies", + "kind": "method", + "arity": 1 + }, + { + "name": "readPath", + "kind": "method", + "arity": 2 + }, + { + "name": "ready", + "kind": "accessor" + }, + { + "name": "rebuildIndexesIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "rebuildMetadataIndexOnline", + "kind": "method", + "arity": 0 + }, + { + "name": "reconcileLogDivergence", + "kind": "method", + "arity": 2 + }, + { + "name": "reconstructPath", + "kind": "method", + "arity": 4 + }, + { + "name": "recordStateAt", + "kind": "method", + "arity": 3 + }, + { + "name": "recoverPendingEmbedsFromLog", + "kind": "method", + "arity": 0 + }, + { + "name": "registerShutdownHooks", + "kind": "method", + "arity": 0 + }, + { + "name": "relate", + "kind": "method", + "arity": 1 + }, + { + "name": "related", + "kind": "method", + "arity": 1 + }, + { + "name": "relateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "relationFromGenerationRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "relationshipSubtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "releaseGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "remove", + "kind": "method", + "arity": 1 + }, + { + "name": "removeAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMany", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMigrationBackupSafe", + "kind": "method", + "arity": 0 + }, + { + "name": "repackHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "repairIndex", + "kind": "method", + "arity": 1 + }, + { + "name": "requestFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "requireProviders", + "kind": "method", + "arity": 1 + }, + { + "name": "requireSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveAsOfGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "resolveDiffEndpoint", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHiddenIds", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHNSWPersistMode", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveRawGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveRetentionPolicy", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveVerbEndpointInts", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveVerbIntsToIds", + "kind": "method", + "arity": 1 + }, + { + "name": "restore", + "kind": "method", + "arity": 2 + }, + { + "name": "rrfFusion", + "kind": "method", + "arity": 4 + }, + { + "name": "runAggregationBackfillWalk", + "kind": "method", + "arity": 0 + }, + { + "name": "runAggregationCatchUp", + "kind": "method", + "arity": 0 + }, + { + "name": "runEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "runOracle", + "kind": "method", + "arity": 1 + }, + { + "name": "runRepairIndexPhases", + "kind": "method", + "arity": 5 + }, + { + "name": "scanFacts", + "kind": "method", + "arity": 1 + }, + { + "name": "seedIdsToInts", + "kind": "method", + "arity": 1 + }, + { + "name": "selectorToSeedIds", + "kind": "method", + "arity": 1 + }, + { + "name": "setRetentionBudget", + "kind": "method", + "arity": 1 + }, + { + "name": "setupEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "setupIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "setupStorage", + "kind": "method", + "arity": 0 + }, + { + "name": "similar", + "kind": "method", + "arity": 1 + }, + { + "name": "similarity", + "kind": "method", + "arity": 2 + }, + { + "name": "splitForHighlighting", + "kind": "method", + "arity": 2 + }, + { + "name": "stampBrainFormat", + "kind": "method", + "arity": 0 + }, + { + "name": "stampBrainFormatIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "stampEntityTree", + "kind": "method", + "arity": 0 + }, + { + "name": "stampProjectionWatermarks", + "kind": "method", + "arity": 0 + }, + { + "name": "stats", + "kind": "method", + "arity": 0 + }, + { + "name": "storageAdapter", + "kind": "accessor" + }, + { + "name": "stream", + "kind": "method", + "arity": 0 + }, + { + "name": "streaming", + "kind": "accessor" + }, + { + "name": "subtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "trackField", + "kind": "method", + "arity": 1 + }, + { + "name": "transact", + "kind": "method", + "arity": 2 + }, + { + "name": "transactionLog", + "kind": "method", + "arity": 1 + }, + { + "name": "unrelate", + "kind": "method", + "arity": 1 + }, + { + "name": "unvectorNounForRootMigration", + "kind": "method", + "arity": 1 + }, + { + "name": "update", + "kind": "method", + "arity": 1 + }, + { + "name": "updateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "updateRelation", + "kind": "method", + "arity": 1 + }, + { + "name": "upsertMergeParams", + "kind": "method", + "arity": 2 + }, + { + "name": "use", + "kind": "method", + "arity": 1 + }, + { + "name": "usesDefaultWasmEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "validateIndexConsistency", + "kind": "method", + "arity": 0 + }, + { + "name": "vectorSearchAtGeneration", + "kind": "method", + "arity": 4 + }, + { + "name": "verbsToRelations", + "kind": "method", + "arity": 1 + }, + { + "name": "verbToRelationLike", + "kind": "method", + "arity": 1 + }, + { + "name": "verifyEntityTreeStamp", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyGraphAdjacencyLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyMetadataLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyVectorLive", + "kind": "method", + "arity": 0 + }, + { + "name": "versionedIndexProviders", + "kind": "method", + "arity": 0 + }, + { + "name": "vfs", + "kind": "accessor" + }, + { + "name": "waitForIndexed", + "kind": "method", + "arity": 2 + }, + { + "name": "warm", + "kind": "method", + "arity": 0 + }, + { + "name": "warmupEmbeddings", + "kind": "method", + "arity": 0 + }, + { + "name": "warnIfReadsDegraded", + "kind": "method", + "arity": 1 + }, + { + "name": "wireConnectionsCodec", + "kind": "method", + "arity": 0 + }, + { + "name": "wireGraphIdResolver", + "kind": "method", + "arity": 0 + } + ], + "errors": [ + "BrainyError", + "DerivedArtifactMissingError", + "GraphIndexNotReadyError", + "MetadataIndexNotReadyError", + "MigrationInProgressError", + "ProtectedArtifactError", + "VectorIndexNotReadyError" + ], + "operators": { + "accepted": [ + "between", + "contains", + "endsWith", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "length", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "matches", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf", + "startsWith" + ], + "servedOnIndexPath": [ + "between", + "contains", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf" + ], + "refusedByIndexPath": [ + "endsWith", + "length", + "matches", + "startsWith" + ], + "combinators": [ + "allOf", + "anyOf", + "not" + ] + }, + "fieldAddressing": { + "systemKeyPrefix": "system.", + "systemEntityScalars": [ + "confidence", + "createdAt", + "createdBy", + "id", + "service", + "subtype", + "type", + "updatedAt", + "visibility", + "weight" + ], + "systemRelationScalars": [ + "confidence", + "createdAt", + "createdBy", + "service", + "sourceId", + "subtype", + "targetId", + "updatedAt", + "verb", + "visibility", + "weight" + ], + "plumbingFields": [ + "_rev", + "connections", + "data", + "level", + "vector" + ] + }, + "health": { + "verdicts": [ + "pass", + "warn", + "fail" + ], + "healKinds": [ + "none", + "repair", + "rebuild" + ], + "servingWithholdingInvariants": [ + "index-initialized", + "durable-state-present", + "manifest-residency", + "replay-clean", + "strand-latch" + ] + } +} diff --git a/docs/api/README.md b/docs/api/README.md index 4ca84364..ba49ff48 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -24,7 +24,7 @@ next: ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() // Zero config! await brain.init() // VFS auto-initialized! @@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization ### Db API errors -All exported from `@soulcraft/brainy`: +All exported from `@soulcraftlabs/brainy`: | Error | Thrown by | Meaning | |---|---|---| @@ -1451,6 +1451,34 @@ const count = await brain.getVerbCount() --- +### The canonical count ledger (`StorageAdapter.getCanonicalCounts()`) + +An OPTIONAL method on the `StorageAdapter` interface (implemented by both +built-in adapters), not a method on `Brainy` itself — relevant if you're +writing a custom storage adapter or composing a provider's own +`healthReport()`. O(1), no I/O. Per family (`nouns`/`verbs`): + +```typescript +interface CanonicalCounts { + nouns: { counted: number; all: number } + verbs: { counted: number; all: number } + suspect: boolean +} +``` + +- `counted` mirrors `getNounCount()` / `getVerbCount()` (public + internal tiers). +- `all` is the ALL-visibility scalar — every tier, including system/internal + records — the denominator a derived index's own coverage math is measured + against. +- `suspect` is `true` when an unprovable delete has left `all` unverified since + the last recount; `brain.repairIndex()` clears it with a real canonical walk. + +Adapters without the ledger omit the method; treat absence as "no +denominator," never as zero. See +**[Index Health](../concepts/index-health.md)** for the full story. + +--- + ### Subtype & facet APIs Full guide: **[Subtypes & Facets](../guides/subtypes-and-facets.md)**. @@ -1831,6 +1859,104 @@ const semanticOnly = await brain.getStats({ excludeVFS: true }) --- +### `repairIndex(options?)` → `Promise` + +The ceremony door for index repair. Bare `repairIndex()` is report-driven: it +prunes orphaned containers, recomputes count rollups, reconciles VFS +containment, and rebuilds only a derived-index family whose own health check +asks for it. Pass `options.rebuild` to force one or more families to rebuild +UNCONDITIONALLY — no health check is consulted — when an operator has +independent reason to reconcile a family regardless of what it self-reports. + +```typescript +// Report-driven: only heals what actually needs it +const report = await brain.repairIndex() +console.log(report.healedTotal, report.families) + +// Explicit: force the graph adjacency to rebuild from canonical, unconditionally +await brain.repairIndex({ rebuild: ['graph'] }) + +// Explicit: force all three derived indexes to rebuild +await brain.repairIndex({ rebuild: 'all' }) +``` + +**`RepairReport`:** +- `families: RepairFamilyReport[]` — one row per family checked +- `healedTotal: number` — items healed across every family +- `durationMs: number` + +**`RepairFamilyReport`** (one row): +- `family: string` — e.g. `'orphaned-containers'`, `'count-rollups'`, + `'vfs-containment'`, `'metadata-corruption'`, `'provider:metadata'`, + `'provider:graph'`, `'provider:vector'` +- `checked: boolean` — was this family actually examined (`false` ⇒ see `skipped`) +- `healed: number` — items re-posted/corrected in place (the incremental heal count) +- `missing?: { count: number; sample: string[] }` — exact count plus a capped id + sample when the check can name what diverged (never the full list) +- `rebuilt?: boolean` — a full generational rebuild ran (vs. an incremental heal) +- `detail?: string` / `reason?: string` — narration +- `skipped?: string` — why the family wasn't checked + +Full walkthrough — what each family checks, degraded-but-serving vs. not-ready, +and what `suspect` counts mean — in +**[Index Health](../concepts/index-health.md)**. + +--- + +### Index readiness: typed errors, `healthReport()`, `disableAutoRebuild` + +Every derived-index provider (vector, graph, metadata) may expose a named, +synchronous, O(1) `healthReport()` composed from its own exact ledgers — the +signal Brainy's read gate trusts over sampling or size heuristics. `init()` +brings every provider to serving before it returns; there is no first-query +lazy-rebuild path. A read that reaches a provider whose health report says it +isn't serving throws instead of rebuilding mid-query: + +| Error | Thrown by | Meaning | +|---|---|---| +| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | Graph adjacency isn't serving | +| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | +| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | + +All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish +"index not ready" from a genuine empty result: + +```typescript +import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy' + +try { + const rows = await brain.find({ where: { status: 'active' } }) +} catch (err) { + if (err instanceof MetadataIndexNotReadyError) { + // reconcile: await brain.repairIndex(), then retry + } else { + throw err + } +} +``` + +**`disableAutoRebuild`** no longer defers index construction to the first +query. A needed rebuild always runs at `open()`, regardless of this flag or +dataset size; the flag has no effect on *when* a rebuild runs. Full manual +control lives in `repairIndex({ rebuild: [...] })`, above. + +### `validateIndexConsistency()` → `Promise<...>` + +The deep, async diagnostic counterpart to `healthReport()` — safe to run on a +live brain, but does more work (a provider's `validateInvariants()` may run a +full scan, not just read a ledger). Aggregates the JS metadata index's own +consistency check with every derived-index provider's invariant report. + +```typescript +const validation = await brain.validateIndexConsistency() +if (!validation.healthy) { + console.log(validation.recommendation) // what to run, e.g. repairIndex() + console.log(validation.providers) // each provider's own invariant report, when exposed +} +``` + +--- + ## Lifecycle ### Initialization @@ -2082,7 +2208,7 @@ For the full taxonomy with all 169 types and their descriptions, see: - **📖 Documentation:** [Full Documentation](../) - **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues) - **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions) -- **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy) +- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy) - **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy) --- diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 48064757..83b9e23a 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with .ack | **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) | A pluggable index provider (the 8.0 plugin contract in -`@soulcraft/brainy/plugin`) may replace any of the JS implementations; the +`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the persisted formats above are contract-bound so JS and native implementations can interleave on the same directory. diff --git a/docs/architecture/finite-type-system.md b/docs/architecture/finite-type-system.md index 76492ee5..48a8b1fe 100644 --- a/docs/architecture/finite-type-system.md +++ b/docs/architecture/finite-type-system.md @@ -126,7 +126,7 @@ class TypeAwareMetadataIndex { **The Design**: Specify types clearly in your API calls: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' // Add entity with explicit type await brain.add({ @@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation { **Brainy's Approach**: Extract **typed** concepts: ```typescript -import { NaturalLanguageProcessor } from '@soulcraft/brainy' +import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy' const nlp = new NaturalLanguageProcessor() const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco") @@ -382,7 +382,7 @@ import { getVerbTypes, BrainyTypes, suggestType -} from '@soulcraft/brainy' +} from '@soulcraftlabs/brainy' // Get all available noun types const nounTypes = getNounTypes() diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md index 8b3dc540..6a754b56 100644 --- a/docs/architecture/index-architecture.md +++ b/docs/architecture/index-architecture.md @@ -723,6 +723,14 @@ async stats(): Promise { ### 5. Index Rebuilding (Lazy Loading Support) +> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is +> RETIRED.** `disableAutoRebuild` no longer defers index construction to a +> first query; `brain.init()` now runs every needed rebuild to completion +> before it returns, unconditionally, and a read against a not-serving +> provider throws a typed `*NotReadyError` instead of rebuilding mid-query. +> See `docs/concepts/index-health.md` for the current contract. Left below +> as historical background on the rebuild mechanics. + **Two modes of index loading:** #### Mode 1: Auto-Rebuild on init() (default) diff --git a/docs/architecture/initialization-and-rebuild.md b/docs/architecture/initialization-and-rebuild.md index a1645744..e19bdd9f 100644 --- a/docs/architecture/initialization-and-rebuild.md +++ b/docs/architecture/initialization-and-rebuild.md @@ -1,5 +1,15 @@ # Initialization and Rebuild Processes +> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is RETIRED.** +> `disableAutoRebuild` no longer defers index construction to a first query; +> `brain.init()` now runs every needed rebuild to completion before it +> returns, unconditionally. A read against a not-serving provider throws a +> typed `*NotReadyError` instead of rebuilding mid-query. See +> `docs/concepts/index-health.md` for the current contract; this document's +> line-number references to `src/brainy.ts` also predate the file's current +> size and are unreliable. Left as historical background on the rebuild +> mechanics, not as a current API description. + This document explains how Brainy's four indexes (MetadataIndex, vector index, GraphAdjacencyIndex, DeletedItemsIndex) initialize and rebuild from persisted storage. ## Core Principle: All Indexes Are Disk-Based diff --git a/docs/architecture/multiprocess-storage-mixin.md b/docs/architecture/multiprocess-storage-mixin.md index 46f98398..1593bf8f 100644 --- a/docs/architecture/multiprocess-storage-mixin.md +++ b/docs/architecture/multiprocess-storage-mixin.md @@ -127,7 +127,7 @@ For reference, a clean migration path: `isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for build/install artifact protection. 5. Document the new contract in `concepts/storage-adapters.md`. -6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by +6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by plugins. Estimated work: ~half a day of code, ~2 hours of doc/example updates, diff --git a/docs/architecture/noun-verb-taxonomy.md b/docs/architecture/noun-verb-taxonomy.md index 286464be..3dac6892 100644 --- a/docs/architecture/noun-verb-taxonomy.md +++ b/docs/architecture/noun-verb-taxonomy.md @@ -20,7 +20,7 @@ next: Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu - **Multi-hop Graph Traversals = Relationship Complexity** - **Result: Model data across virtually any industry** -Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. +Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. ## The Power of Standardization: Universal Interoperability diff --git a/docs/architecture/zero-config.md b/docs/architecture/zero-config.md index d42d6784..a35e6416 100644 --- a/docs/architecture/zero-config.md +++ b/docs/architecture/zero-config.md @@ -35,7 +35,7 @@ constructor and `init()`. ## Instant Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // That's it. No config needed. const brain = new Brainy() diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md new file mode 100644 index 00000000..d24dd66b --- /dev/null +++ b/docs/concepts/field-addressing.md @@ -0,0 +1,233 @@ +--- +title: Field addressing: your fields and system fields +slug: concepts/field-addressing +public: true +category: concepts +template: concept +order: 7 +description: The one rule for every query-surface field name — a bare name always means your metadata, system. reaches the ten engine scalars explicitly, and anything else refuses by name. +next: + - guides/namespace-migration + - concepts/consistency-model +--- + +# Field addressing: your fields and system fields + +Every query surface in Brainy — `find()`'s `where`, `orderBy`, aggregation +`groupBy`, and aggregation `source.where` — resolves field names by one rule, +with no exceptions: + +> **A bare field name always means your metadata. `system.` reaches an +> engine scalar, and only when you spell it explicitly.** + +```typescript +await brain.find({ orderBy: 'level' }) // reads entity.metadata.level — YOUR field +await brain.find({ orderBy: 'system.createdAt' }) // reads the engine's createdAt scalar +await brain.find({ orderBy: 'metadata.level' }) // identical to bare 'level' — explicit scope +``` + +There is no priority list, no "try the system field, fall back to metadata" +behavior, and no name that resolves differently depending on what else +happens to exist on your entities. A field called `level`, `score`, +`createdAt`, or `type` in your own `metadata` is read as *your* field, every +time, by its bare name. + +## Why this rule exists + +An internal report from a production deployment found that a user metadata +field literally named `level` was being silently shadowed by the engine's +own internal index layer field of the same name — every sort by `level` +returned insertion order, with no error raised. This rule makes that class of +bug structurally impossible: bare names belong to you, unconditionally, and +anything that isn't yours has to be spelled out. + +## The system scalars + +`system.` addresses exactly ten scalars on an entity — no more, no +fewer: + +| System field | What it is | +|---|---| +| `system.id` | The entity's id | +| `system.type` | The entity's `NounType` | +| `system.subtype` | The per-app sub-classification passed to `add()` | +| `system.createdAt` | When the entity was created | +| `system.updatedAt` | When the entity was last written | +| `system.confidence` | The `confidence` param (0–1) | +| `system.weight` | The `weight` param | +| `system.visibility` | `'public'` / `'internal'` (see the visibility tiers in [Consistency Model](./consistency-model.md)) | +| `system.service` | The multi-tenancy `service` tag | +| `system.createdBy` | Who/what created the entity | + +Relationships mirror the same eight shared scalars (`subtype`, `createdAt`, +`updatedAt`, `confidence`, `weight`, `visibility`, `service`, `createdBy`) +plus three of their own: + +| System field (relationship) | What it is | +|---|---| +| `system.verb` | The relationship's `VerbType` | +| `system.sourceId` | The id of the entity the relationship starts from | +| `system.targetId` | The id of the entity the relationship points to | + +Anything not on these two lists is not a system scalar — `system.` for +any other name refuses (see "Refusal semantics" below), even if that name +sounds like it should be engine-owned. + +## Invisible plumbing — never addressable, in either spelling + +Five names are pure engine internals. They are not reachable as a bare name, +and not reachable as `system.` either — they simply have no place on +the query surface: + +- **`vector`** — the stored embedding. It participates in similarity search + (`query`, `near`, vector `find()`), never in `where`/`orderBy`/`groupBy`. +- **`connections`** — graph adjacency. Reached through `connected` and + `brain.related()`, not through field addressing. +- **`level`** — the internal index layer number used by the nearest-neighbor + graph. It is pure index plumbing with no query-surface meaning at all — + which is exactly why a user field of the same name must never be shadowed + by it. `level` as a bare name is always yours; there is no engine-owned + spelling of it to compete with. +- **`data`** — your entity's content payload, not a scalar. It can be a + string, a number, or an arbitrary object, so sorting or filtering it as a + single comparable value would lie about its actual shape. Content is + reached through the content/text-search APIs (`query`, `searchMode: + 'text'`), not through `where`/`orderBy`. +- **`_rev`** — the per-entity revision counter used for optimistic + concurrency (`ifRev`). It is a CAS token, not a queryable dimension. + +`system.level`, `system.vector`, and `system.data` all refuse for the same +reason: they are not in the ten-scalar system map, full stop. + +## `metadata.` — the explicit spelling of "mine" + +Prefix any field with `metadata.` to say the same thing a bare name already +says, spelled out. The two are interchangeable everywhere a field name is +accepted, including `orderBy`: + +```typescript +await brain.find({ where: { 'customer.tier': 'gold' } }) +await brain.find({ where: { 'metadata.customer.tier': 'gold' } }) // identical +await brain.find({ orderBy: 'metadata.score', order: 'desc' }) // identical to orderBy: 'score' +``` + +Reach for the explicit spelling when it reads more clearly next to a +`system.` field in the same query — for example, sorting by your own `score` +while filtering on `system.confidence`. + +## No special names — the write side + +The same law governs writes: + +> **Data is either in main space, where developers can use anything, or it +> is in `system.*`.** + +There are **no reserved metadata names**. A field called `confidence`, +`type`, `id`, `data`, `content`, or anything else inside your `metadata` bag +is an ordinary user field: it is stored verbatim, indexed, filterable, +sortable, aggregatable, and it survives restarts, index rebuilds, and +time-travel (`asOf`) reads exactly as written — even when an engine scalar +shares its spelling. The engine's values are written only through their +dedicated params (`confidence`, `weight`, `subtype`, `visibility`, …) and +read at `system.`; your bag can never touch them and they can never +shadow your bag. + +```typescript +const id = await brain.add({ + data: 'Ada Lovelace', + type: NounType.Person, + confidence: 0.9, // the ENGINE scalar + metadata: { confidence: 'self-rated' } // YOUR field, same spelling — both live +}) + +await brain.find({ where: { confidence: 'self-rated' } }) // finds it (yours) +await brain.find({ where: { 'system.confidence': 0.9 } }) // finds it (engine's) +``` + +The one spelling a write refuses is a metadata key that literally starts +with `system.` — the explicit address namespace cannot be forged as a user +field name. That refusal is typed and names the fix. + +Value **shape** rules still apply uniformly to every name (they are not name +carve-outs): arrays longer than 10 elements are not turned into posting-list +scalars, and very long values are indexed by hash. + +## Refusal semantics + +A name that resolves to neither your metadata nor a system scalar is a typed +refusal, not a silent empty result and not a guess. Refusals name **both** +candidates, so the fix is always in the error text: + +```typescript +await brain.find({ orderBy: 'createdAt' }) +// UnresolvableFieldError: no metadata field 'createdAt' — did you mean +// system.createdAt or metadata.createdAt? +``` + +`UnresolvableFieldError` is exported from the package root: + +```typescript +import { UnresolvableFieldError } from '@soulcraftlabs/brainy' + +try { + await brain.find({ orderBy: 'createdAt' }) +} catch (err) { + if (err instanceof UnresolvableFieldError) { + // err.message names both candidates — usually enough to fix the call site. + } +} +``` + +A handful of `find()` options are not implemented yet: `cursor`, +`includeRelations`, and `writeOnly`. Rather than accepting them and quietly +ignoring the option, `find()` refuses with `UnsupportedFindOptionError` — +also exported from the package root — so a call site can never believe an +unimplemented option took effect when it didn't. + +## The ordering contract + +`orderBy` behaves identically regardless of which engine (the pure-TypeScript +path or a native accelerator) is serving the query: + +- An entity missing the `orderBy` field, or holding `null` on it, sorts + **LAST — in both `asc` and `desc`**. It is never treated as "smaller than + everything" in one direction and "larger than everything" in the other; it + is simply last, either way. +- Rows are **never dropped** from an ordered read because they lack the + field — a missing value changes position, never presence. +- Ties on the `orderBy` field break by **id ascending**, regardless of the + primary sort direction. + +```typescript +// employees: [{ score: 9 }, { score: 5 }, { /* no score field */ }] +await brain.find({ orderBy: 'score', order: 'desc' }) // [9, 5, missing] — missing is last +await brain.find({ orderBy: 'score', order: 'asc' }) // [5, 9, missing] — missing is STILL last +``` + +## Migrating existing call sites + +If you have call sites written before this rule shipped that rely on a bare +system name — `orderBy: 'createdAt'`, `where: { confidence: { greaterThan: +0.8 } }`, and similar — they now refuse instead of silently resolving to the +engine field. The fix is always in the error: swap the bare name for +`system.` (or `metadata.` if you actually meant your own field +of that name, and it happens to share a name with a system scalar): + +```typescript +// Before: bare 'createdAt' silently meant the engine's timestamp. +await brain.find({ orderBy: 'createdAt' }) + +// After: say which one you meant. +await brain.find({ orderBy: 'system.createdAt' }) // the engine timestamp +await brain.find({ orderBy: 'metadata.createdAt' }) // your own field named createdAt, if you have one +``` + +There is no silent migration path by design — every ambiguous call site +surfaces as a refusal naming its own fix, once, the first time it runs +against the new rule. + +## Where to go next + +- [Consistency Model](./consistency-model.md) — visibility tiers, revision + counters, and the rest of the read/write contract this page's + read-time addressing rule. diff --git a/docs/concepts/generation-fact-log.md b/docs/concepts/generation-fact-log.md new file mode 100644 index 00000000..f9b3e974 --- /dev/null +++ b/docs/concepts/generation-fact-log.md @@ -0,0 +1,117 @@ +--- +title: The Generation Fact Log +slug: concepts/generation-fact-log +public: true +category: concepts +template: concept +order: 6 +description: Every committed write also appends a self-verifying "fact" — an after-image commit record — to an append-only log. What facts are, the crash-safety model, the scanFacts() streaming surface, family stamps, and how index providers consume the log for sequential heals. +next: + - concepts/consistency-model + - guides/snapshots-and-time-travel +--- + +# The Generation Fact Log + +Since 8.4.0, every committed generation also appends a **fact** — a compact record of what each +touched entity or relationship *became* — to an append-only, checksummed log under +`_generations/facts/`. Where the generational history answers *"what did things look like +before?"* (before-images, powering `asOf()` and rollback), the fact log answers *"what happened, +in order?"* — one sequential, self-verifying stream of the store's present being written. + +Nothing about querying changes. The fact log exists for three consumers: + +1. **Index heals and rebuilds** — one sequential read in commit order replaces a per-entity + directory walk over millions of files. +2. **Incremental catch-up** — a derived index that knows which generation it reflects reads *just + the gap*, instead of rebuilding from scratch. +3. **Replay and audit tooling** — anything that wants the store's committed timeline as a stream. + +## What a fact is + +One fact per committed generation: + +- **`generation`** and **`timestamp`** — which commit, when. +- **`ops`** — every write in that commit: `{ kind: 'noun' | 'verb', id, record }` where `record` + holds the entity's full after-image (both stored legs), or **`null` for a tombstone** — a + removal carries no body, by design. +- **`meta`** — the transaction metadata `transact()` was submitted with, when present. +- **`blobHashes`** — content-blob references, for exact reclamation accounting. + +Facts accumulate **from the first write after upgrading** — pre-existing history is not +retroactively converted, and consumers fall back to the enumeration walk when no log exists. + +## Crash safety, in one paragraph + +Facts are appended and fsynced **inside the same durability window as the commit itself**, before +the commit point — so after a crash, the log can only ever be *ahead* of committed truth, never +behind it with a hole. On open, the store reconciles the log back to the committed watermark: +torn tails are detected by per-record checksums and cut; whole records beyond the watermark are +truncated. The invariant every reader can rely on: **an absent generation was never committed; a +present fact was.** `transact()` facts are durable the moment `transact()` returns; single-op +facts share the same group-commit flush as the rest of their generation, so a hard kill loses the +fact and the generation *together* — never a torn state. + +## Reading the log + +```typescript +const scan = brain.scanFacts({ fromGeneration: 1 }) +if (scan) { + // Telemetry up front — progress bars get a denominator from second zero. + console.log(scan.headGeneration, scan.segmentCount, scan.approxFactCount) + + for await (const batch of scan.batches()) { + // Each batch: { facts, firstGeneration, lastGeneration, factCount, byteSize, segmentId } + for (const fact of batch.facts) { + for (const op of fact.ops) { + if (op.record === null) { + // a tombstone: op.id was removed in this generation + } + } + } + } + + console.log(scan.summary()) // { factsYielded, segmentsRead } — the cross-check +} +``` + +- `scanFacts()` returns `null` when the store hosts no fact log (older store, or a storage adapter + without binary append support) — fall back to enumerating entities. +- Scans run against a **snapshot**: facts appended after the scan opens never bleed in, each fact + is yielded exactly once, and a detected gap aborts loudly — never a silent skip. +- `brain.factSegmentPaths()` returns the immutable, *sealed* segment files for zero-copy consumers + (the append-mutable tail is excluded — read it through `scanFacts()`). + +## Family stamps: how a projection proves it's current + +Anything derived from the store — an index, the entity file tree itself — carries a **family +stamp**: a small JSON record of *which committed generation the projection reflects* +(`sourceGeneration`) plus the invariants that verify it whole (exact per-file byte sizes for +bounded families; rollup invariants like entity counts for unbounded ones). At open, coherence is +a **comparison**, not a walk: + +- stamp equals the committed watermark and invariants hold → serve; +- stamp is behind → the projection reads just the gap from the fact log; +- invariants fail → loud, named divergence — `brain.repairIndex()` rebuilds from canonical and + re-stamps. + +The verifier is exported (`verifyFamilyStamp`) so every projection — TypeScript or native — runs +literally the same check. + +## For plugin authors: the storage capability + +Index providers receive the storage adapter, not the brain — so the host wires the log onto it. +Feature-detect and prefer the stream; fall back to enumeration: + +```typescript +const scan = storage.scanFacts?.({ fromGeneration: stamp.sourceGeneration + 1 }) +if (scan) { + // sequential catch-up from the log +} else { + // enumeration walk (older store or adapter) +} +const committed = storage.committedGeneration?.() // the watermark stamps compare against +``` + +Providers must never construct their own reader over the log's files — the open path belongs to +the single writer (it reconciles the log at open); the capability is the sanctioned seam. diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md new file mode 100644 index 00000000..923267df --- /dev/null +++ b/docs/concepts/index-health.md @@ -0,0 +1,217 @@ +--- +title: Index Health +slug: concepts/index-health +public: true +category: concepts +template: concept +order: 8 +description: How Brainy knows whether a derived index can be trusted — exact accounting instead of sampling, the named health report, degraded-but-serving vs. not-ready, and what repairIndex() checks, heals, and rebuilds. +next: + - concepts/generation-fact-log + - guides/inspection +--- + +# Index Health + +Brainy keeps one **canonical** copy of every entity and relationship, and three +**derived** indexes built from it — vector, metadata, and graph — so `find()` can +answer semantically, by filter, and by traversal without re-deriving the answer from +scratch on every query. A derived index is a cache with a serving structure: it can +be present but stale, present but only partially loaded, or fully out of sync with +canonical after a crash. This page is about how Brainy decides whether to trust one, +what it does when it can't, and how you reconcile the two. + +## Exact accounting instead of sampling + +Older health checks worked by inference: does `size()` return something greater +than zero, does a spot-check on one known item come back correct. Both are proxies. +A cold index can report a nonzero count while its actual serving structure never +loaded, and a spot-check only proves the one item it happened to ask about. + +Every derived-index provider may now expose a named, synchronous, O(1) +`healthReport()` — composed from the provider's own **exact ledgers** (real counters +it already maintains on the write path), never a sample or a walk. This is the one +signal Brainy's read gate consults. A provider that doesn't yet expose one falls +back to an honest `isReady()` boolean, and finally to a size heuristic for engines +with neither — but wherever a `healthReport()` exists, it wins. + +Underneath, storage itself keeps an analogous **canonical count ledger**: a +`counted` scalar (the user-facing total — what `getNounCount()` / `getVerbCount()` +return) and an `all` scalar (every tier, including internal records a derived +index's own coverage math needs to compare against). This is the real denominator +a provider's `healthReport()` measures itself by, rather than a total that can only +ever ratchet upward. See [What `suspect` counts mean](#what-suspect-counts-mean) +below for the one case that ledger can't stay exact through on its own. + +## The named report + +A `HealthReport` carries, per provider (`'vector'` / `'graph'` / `'metadata'`): + +- **`healthy`** — `true` iff every *verified* invariant holds. An invariant whose + family has no ledger yet is `unledgered`, never counted either way — unknown, + not passing. +- **`serving`** — can this provider answer a query right now. A failing invariant + graded `heal: 'repair'` or `heal: 'none'` still leaves `serving: true` — this is + **degraded-but-serving**: something is off (say, a stale rollup on an + `employee` record's relationship count) but reads keep working. Only a failure + graded `heal: 'rebuild'` flips `serving` to `false` — **not-ready** — because the + provider itself is telling you its serving structure cannot answer correctly. +- **`invariants`** — each checked condition, with its provenance + (`source: 'ledger'` — an exact count; `'deep'` — a full scan, diagnostic-only; + `'unledgered'` — not yet tracked) and, for a failing one, an exact `missing` + count plus a capped sample of the affected ids — a verdict, never a dump. +- **`generation`** — bumps on every ledger mutation and rebuild, so a caller can + cache a verdict per generation instead of re-deriving it. + +The distinction that matters day to day: `healthy: false` can be entirely benign — +a maintenance window, a divergence `repairIndex()` will clean up on its own +schedule. `serving: false` is not benign. It means this provider is refusing to +answer, on its own word, right now. + +**How a failure gets its grade — the serving law.** A provider grades `heal` by +one question only: *could an answer be wrong?* — never *how expensive is the +fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post +exactly what the ledger names, reads serving throughout); it can never withhold +serving just because healing it takes work. `serving` is withheld only by a +small, named set of rebuild-graded conditions — the index not initialized, its +durable state absent, a manifest naming files that are not resident, a replay +that did not complete cleanly — the states in which an answer could genuinely be +wrong. And a read is only ever refused by the family it actually consults: a +metadata filter is answered by the metadata index alone, vector search by the +vector index, traversal by the graph index — one family's refusal never blocks +another family's reads. + +## Reads refuse — they never rebuild + +A query that reaches a not-serving provider does not trigger a rebuild from inside +the read. Brainy retired that path deliberately: a rebuild kicked off by an ordinary +`find({ where: { status: 'active' } })` call is a dark, unpredictable cost hiding +behind a request that looks like a cheap read. Instead, the read throws a typed, +catchable error naming the reason: + +| Error | Thrown when | Meaning | +|---|---|---| +| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | The graph adjacency index isn't serving — traversal would otherwise return `[]` indistinguishable from "no relationships" | +| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | +| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | + +All three are exported from `@soulcraftlabs/brainy`. Catch them where your application +needs to distinguish "this index isn't ready yet" from "there's genuinely nothing +here" — a health dashboard, a retry policy, an operator alert. The fix is always +the same: reconcile the index, either by reopening the brain (which brings every +provider to serving before `init()` returns — see the next section) or by calling +`repairIndex()` explicitly. + +```typescript +try { + const active = await brain.find({ where: { status: 'active' } }) +} catch (err) { + if (err instanceof MetadataIndexNotReadyError) { + // not a "no results" — the index itself refused; alert or retry after repair + } else { + throw err + } +} +``` + +### Rebuilds happen at open, not on first query + +`brain.init()` runs every needed rebuild to completion **before it returns**, +unconditionally, regardless of dataset size. There is no lazy, first-query +rebuild path anymore — a brain either finishes opening healthy, or it fails +open loudly. `disableAutoRebuild: true` no longer defers index construction to +the first query: it has no effect on *when* a needed rebuild runs. Full manual +control over rebuilds is `repairIndex({ rebuild: [...] })` (below), not this flag. + +## `repairIndex()` — checking and healing + +Bare `repairIndex()` is **report-driven**: it only heals what its own checks say +actually needs it, and it always returns a full per-family receipt. + +```typescript +const report = await brain.repairIndex() +report.healedTotal // total items healed across every family +report.durationMs +report.families // one row per family checked +``` + +Each `RepairFamilyReport` row names what happened: + +- **`checked`** — was this family actually examined (`false` means skipped — + see `skipped` for why). +- **`healed`** — items re-posted or corrected in place. +- **`missing`** — when the check can name what diverged: an exact `count` plus a + capped `sample` of ids. +- **`rebuilt`** — a full generational rebuild ran (as opposed to an incremental + heal). +- **`detail`** / **`reason`** / **`skipped`** — the receipt's narration; a row is + always either checked or explains why it wasn't. Nothing is silent. + +On every call, bare `repairIndex()`: + +1. Prunes orphaned canonical containers left by a partial delete. +2. Recomputes the count rollups from one canonical walk (unconditional — this is + also what clears a `suspect` ledger; see below). +3. Reconciles VFS containment edges, if the VFS is initialized. +4. Runs the metadata index's own corruption detection pass. +5. Consults each of the three derived-index providers' own health check and + rebuilds only a family whose failing invariant actually asks for it + (`heal: 'rebuild'`) — never a provider that reports `healthy` or a lesser + grade. + +### The explicit rebuild door + +`options.rebuild` skips the health check and rebuilds one or more families +**unconditionally** — the operator override for when you have independent reason +to distrust a family regardless of what it self-reports (a suspicious deploy, a +storage-layer incident, a support ticket that doesn't match what the health report +says): + +```typescript +// Force the graph adjacency to rebuild from canonical, no invariant consulted +await brain.repairIndex({ rebuild: ['graph'] }) + +// Force all three derived indexes +await brain.repairIndex({ rebuild: 'all' }) +``` + +A family named this way is recorded with `rebuilt: true` and +`reason: 'explicit rebuild requested'`, and is skipped by the normal +health-driven pass in the same call — it was already rebuilt unconditionally. + +Reach for the explicit door when you need certainty regardless of self-report; +reach for bare `repairIndex()` for routine maintenance and after any incident +where you're not sure which family (if any) needs it. + +## What `suspect` counts mean + +Storage's canonical count ledger increments the ALL-visibility total on every new +record and decrements it on every *proven* delete — one where the record was read, +or the caller supplied its prior image. A delete that cannot prove what it removed +existed doesn't guess: it flags the ledger `suspect` (an operator-visible +`console.warn`, narrated once per session, not once per delete) rather than risk +decrementing a total that was never incremented for that record in the first +place. This is intentionally rare — it's a defensive fallback for callers on an +unusual removal path, not a per-delete cost. + +`suspect` is not directly exposed on any `Brainy` method today — it lives on the +`StorageAdapter`'s optional `getCanonicalCounts()`, primarily consulted by +`repairIndex()`'s recount step and by custom storage adapters composing their own +`healthReport()`. What matters for an application: a `suspect` ledger is not +incorrect, just *unverified since the last recount* — and `repairIndex()`'s +unconditional count-rollup step (step 2, above) recomputes the ALL scalars from a +real canonical walk on every call, clearing the flag with proof either way. + +## Practical guidance + +- **On a normal restart**, do nothing — `init()` brings every provider to + serving before it returns, or fails loudly. +- **On a `*NotReadyError`** from a live read, reconcile with `repairIndex()` + (report-driven is almost always sufficient) and retry. +- **After an incident** where you distrust a specific family regardless of what + it reports healthy — a storage-layer fault, a suspicious restore — use the + explicit door: `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] })`. +- **To audit before trusting a report**, `brain.auditGraph()` walks every stored + relationship and proves (or disproves) that reads return canonical truth, + independent of what any provider self-reports — see + [Inspecting a Live Brainy](../guides/inspection.md). diff --git a/docs/concepts/storage-adapters.md b/docs/concepts/storage-adapters.md index af6d068f..82aa01e8 100644 --- a/docs/concepts/storage-adapters.md +++ b/docs/concepts/storage-adapters.md @@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from to call `acquireWriterLock()` at init. ```typescript -import { FileSystemStorage } from '@soulcraft/brainy' +import { FileSystemStorage } from '@soulcraftlabs/brainy' export class MmapFileSystemStorage extends FileSystemStorage { public supportsMultiProcessLocking(): boolean { @@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom network backend), extend `BaseStorage` directly: ```typescript -import { BaseStorage } from '@soulcraft/brainy' +import { BaseStorage } from '@soulcraftlabs/brainy' export class MyCloudStorage extends BaseStorage { // BaseStorage's default no-op implementations of the multi-process @@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`, `hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM import (verify in your plugin's `dist/`: `import { FileSystemStorage } from -'@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype +'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype chain at runtime resolves to whatever Brainy version your consumer has installed. @@ -109,8 +109,8 @@ installed. the prototype chain at the consumer-app level: - **Stale `node_modules`** — a lingering install from before the consumer - upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but - `node_modules/@soulcraft/brainy` is still 7.20.x. + upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but + `node_modules/@soulcraftlabs/brainy` is still 7.20.x. - **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy version older than the package.json range, and `bun install` honors the lockfile. @@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint: methods on its prototype chain. Writer locking and the flush-request RPC are disabled for this directory. Likely fix: clean install (`rm -rf node_modules bun.lockb && bun install`) or rebuild your container image to refresh -`@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. +`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. ``` ## Authoring a new storage adapter — minimum checklist @@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh install time — fix install, not your plugin. 6. **Pin your peer dep generously.** `"peerDependencies": { - "@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin + "@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin to an exact patch unless you're tracking a known regression. ## Future direction @@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change. heartbeat semantics, what the lock protects. - [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the read-only mode. -- `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the +- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the authoritative type signatures for every method this page references. diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index e58eb2ce..616c8fc4 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -11,12 +11,18 @@ No batch jobs. No scheduled recalculations. Aggregates stay current with every w **Defining over existing data:** if you define an aggregate on a store that already holds matching entities, Brainy backfills it from those entities on the first query (a one-time scan, then purely incremental). So `defineAggregate()` behaves the same whether you define it before -or after the data exists — including when a persisted brain reopens already populated. +or after the data exists. + +**Reopening a persisted brain:** aggregate state persists across restarts. Re-defining the +same aggregate at boot (the normal declarative pattern) adopts the persisted state directly — +no rescan. A backfill scan runs only when the definition actually changed, when no persisted +state exists, or when the state failed to load; and however many aggregates need backfilling, +they share a single scan. ## Quick Start ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/external-backups-and-sparse-storage.md b/docs/guides/external-backups-and-sparse-storage.md new file mode 100644 index 00000000..f28dcfd7 --- /dev/null +++ b/docs/guides/external-backups-and-sparse-storage.md @@ -0,0 +1,99 @@ +--- +title: External Backups & Sparse Storage +slug: guides/external-backups +public: true +category: guides +template: guide +order: 10 +description: How to back up a brain directory with external tools (tar, rsync, cp) without exploding sparse files — why a store can show 100+ GB "apparent" size on a small disk, which files are sparse, and how persist()/restore() handle it for you. +next: + - guides/snapshots-and-time-travel + - concepts/storage-adapters +--- + +# External Backups & Sparse Storage + +The built-in snapshot path — [`db.persist()` and `brain.restore()`](/docs/guides/snapshots-and-time-travel) — +already handles everything on this page for you. Read this when you back up a brain directory with +**external tools**: `tar`, `rsync`, `cp`, `scp`, or a filesystem-level backup agent. + +## The one-sentence rule + +> **Always use the sparse-aware flag**: `tar czSf` (capital `S`), `rsync --sparse`, +> `cp --sparse=always`. A naive copy can turn a 2 GB store into a 100+ GB one — or fail +> the disk entirely. + +## Why: some files are sparse + +When a native accelerator plugin is active, parts of the index live in **memory-mapped files** +created at a large fixed virtual size — the file's *apparent* size — while the filesystem only +allocates blocks that were actually written. A brand-new id-mapper file can report tens of +gigabytes in `ls -l` while occupying a few megabytes on disk. + +Check the difference yourself: + +```bash +ls -lh brain-data/_id_mapper/ # APPARENT size (can be huge) +du -sh brain-data/ # ALLOCATED size (the real footprint) +``` + +The sparse candidates in a brain directory: + +| Path | What it is | +|---|---| +| `_id_mapper/` | The native id-mapper's mmap files (large fixed virtual size) | +| `_blobs/` | Native index files (vector base, segments) — may be mmap-backed | + +Everything else (entities, `_system`, `_generations`, `_cas` content blobs) is ordinary dense data. + +## Doing it right + +**tar** — the `S` flag detects holes and stores only real data: + +```bash +tar czSf brain-backup.tgz /data/brain +# restore preserves the holes: +tar xzSf brain-backup.tgz -C /data/ +``` + +**rsync**: + +```bash +rsync -a --sparse /data/brain/ backup-host:/backups/brain/ +``` + +**cp**: + +```bash +cp -a --sparse=always /data/brain /backups/brain +``` + +**What goes wrong without the flag:** the copy *materializes* every hole as real zero bytes. +A store whose apparent size exceeds the target disk fails with `ENOSPC` partway through — and a +copy that *does* fit silently costs the full apparent size in storage and transfer time. + +## What the built-in paths do (so you don't have to) + +- **`db.persist(path)`** snapshots via **hard links** — instant and space-shared, since every data + file is immutable-by-rename. The handful of append-in-place files (the transaction log, the + commit fact log's tail segment) and mmap-mutated directories (`_id_mapper/`) are **byte-copied** + instead, so a post-snapshot write can never reach through a shared inode into your backup. +- **`brain.restore(path, { confirm: true })`** is **non-destructive and sparse-aware**: the snapshot + is copied into a staging area *before* any live data is touched (all-zero blocks stay holes), and + only after the copy fully succeeds does an atomic swap move it into place. A failed copy — + including `ENOSPC` — leaves the live store exactly as it was. A crash mid-swap completes forward + on the next open. + +## Live-store caveats for external tools + +1. **Prefer snapshotting a `persist()` output, not the live directory.** `persist()` produces a + crash-consistent, immutable snapshot; running `tar` against a live, actively-written directory + can capture a torn mid-write state. If you must archive live, stop writes first (or accept that + the archive is only as consistent as the moment's flush state). +2. **Never prune or "clean up" files inside a brain directory.** Index files that look stale or + redundant are load-bearing; the store protects its declared index families from in-process + deletion, but an external `rm` bypasses that fence. If space is the concern, `du -sh` first — + the allocated size is usually far smaller than it looks. +3. **Verify restores by opening them.** `Brainy.load(path)` opens any snapshot or restored + directory read-only — the store verifies its own coherence at open and reports loudly if + anything is missing or torn. diff --git a/docs/guides/find-limits.md b/docs/guides/find-limits.md index 66418b80..4c7fd252 100644 --- a/docs/guides/find-limits.md +++ b/docs/guides/find-limits.md @@ -40,6 +40,10 @@ Brainy picks `maxLimit` from the first of these that's available: Worked example: a 4 GB Cloud Run container picks priority 3 → `floor(4 GB × 0.25 / 25 KB) = floor(40 960) = 40 000` results. A 900 MB free-memory box on priority 4 gets `floor(900 MB / 25 KB) = ~36 000`. +The cap is fixed at construction and never changes at runtime. Query timing is recorded +for diagnostics only — a burst of slow queries cannot silently shrink the cap, and the +auto-detected tiers (3 and 4) never go below a floor of 10 000. + > **Calibration note.** Pre-7.30.2 used 100 KB per result instead of 25 KB, which produced caps that were 4× too tight for typical workloads (an 8 KB / result reality). 7.30.2 recalibrated to match observed entity sizes; existing `limit: 10_000` safety patterns now pass silently on any reasonably-sized box. ## What happens when you exceed the cap diff --git a/docs/guides/framework-integration.md b/docs/guides/framework-integration.md index 984466c5..8f85da00 100644 --- a/docs/guides/framework-integration.md +++ b/docs/guides/framework-integration.md @@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server: -- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'` +- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'` - **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node - **Cleaner code**: No browser polyfills, no conditional client/server imports - **Better DX**: One instance shared across your server routes @@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers ### Install Brainy ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Integration ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Run on the server (API route, server component, backend service) // new Brainy() auto-detects filesystem persistence on Node @@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod ```javascript // lib/brain.server.js -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests: ```javascript // server/brain.js (server-only module) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun): ```typescript // server: api/search -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() // auto-detects filesystem persistence on Node await brain.init() @@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or ```javascript // lib/brain.server.js (imported only by server code) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul ```javascript // src/lib/server/brain.js (server-only — note the .server suffix) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -432,7 +432,7 @@ import { defineConfig } from 'vite' export default defineConfig({ ssr: { - external: ['@soulcraft/brainy'] + external: ['@soulcraftlabs/brainy'] } }) ``` @@ -440,7 +440,7 @@ export default defineConfig({ ```javascript // rollup.config.js (server bundle) export default { - external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto'] + external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto'] } ``` @@ -466,7 +466,7 @@ export async function load({ url }) { ```javascript // For build-time usage (runs in Node during the build) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' export async function generateStaticProps() { const brain = new Brainy({ @@ -513,7 +513,7 @@ export async function generateStaticProps() { ### Issue: Large client bundle size **Cause**: A client module is pulling in Brainy. -**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle. +**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle. ### Issue: SSR hydration mismatch **Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup. diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index e0cd94ed..ffabe55c 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING: ## The Ultimate Simplicity ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -300,7 +300,10 @@ await brain.import(data, { // Deduplication enableDeduplication: true, // Check for duplicate entities (default: true) deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85) - // Note: Auto-disabled for imports >100 entities + // Notes: false disables BOTH the inline merge and the background pass that + // runs ~5 min after the last import (merged duplicates are deleted). + // The inline pass auto-disables for imports >100 entities (O(n²) cost); + // the background pass still covers those unless the flag is false. // Performance chunkSize: 100, // Batch size for processing (default: varies by operation) diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md index 66f50713..18c3cb9a 100644 --- a/docs/guides/import-progress-examples.md +++ b/docs/guides/import-progress-examples.md @@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats** ### Basic Progress Tracking ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import * as fs from 'fs' const brain = await Brainy.create() diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 5a850a86..3bc26dae 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -7,7 +7,7 @@ ## Basic Import ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -86,11 +86,22 @@ await brain.import(file, { ```typescript await brain.import(file, { - enableDeduplication: true, // Check for duplicates (default: false) + enableDeduplication: true, // Check for duplicates (default: true) deduplicationThreshold: 0.85 // Similarity threshold (default: 0.85) }) ``` +Deduplication merges entities judged duplicates — the non-primary records are +**deleted**. Set `enableDeduplication: false` to disable it entirely: the flag +gates both the inline merge during import and the background pass that runs +about 5 minutes after the last import. + +```typescript +await brain.import(file, { + enableDeduplication: false // No merging, inline or background +}) +``` + ### Import Tracking Track and organize imports by project: @@ -176,7 +187,7 @@ await brain.import(file, { ## Complete Example ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import * as fs from 'fs' async function importCatalog() { diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 015c5118..8560b543 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI. ## Programmatic inspection ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } @@ -166,6 +166,34 @@ brainy inspect diff /data/brain-prod /data/brain-staging Sample-based — for a full diff, dump both with `inspect dump` and compare the JSONL. +## Auditing graph-read truth + +`brain.auditGraph()` (8.6.0+) proves — or disproves — that relationship reads +return canonical truth on a given brain, without mutating anything. It walks +every stored relationship record, asks the same read path your application +uses (`related()`, VFS `readdir`) with every visibility tier included, and +classifies every discrepancy: + +```typescript +const report = await brain.auditGraph() + +report.coherent // true = related()/readdir can be trusted on this brain +report.missingFromReadsCount // records the read path omits — stale index +report.danglingEndpointsCount // relationships whose endpoint entity is gone +report.readOnlyCount // read-path edges with NO stored record — ghosts +report.visibilityHiddenCount // internal/system edges hidden by design (not a fault) +``` + +Counts are always exact; the example lists (`missingFromReads`, +`danglingEndpoints`, `readOnlyVerbIds`) are capped at `maxExamples` +(default 100) and `truncatedExamples` says so when they are. + +Run it after any engine upgrade, restore, or migration. If it reports +discrepancies, run `brain.repairIndex()` and audit again — a `coherent` +report after the repair is the verified statement that the heal worked. +Cost: one relationship-record walk plus one indexed read per distinct +source entity — safe on a live brain. + ## Repairing a corrupted store If invariants fail and you suspect index corruption, `inspect repair` diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 0a36f632..20d40ea2 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -21,21 +21,21 @@ next: ## Install ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` Or with your preferred package manager: ```bash -bun add @soulcraft/brainy -yarn add @soulcraft/brainy -pnpm add @soulcraft/brainy +bun add @soulcraftlabs/brainy +yarn add @soulcraftlabs/brainy +pnpm add @soulcraftlabs/brainy ``` ## Verify ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -52,7 +52,7 @@ npm install @soulcraft/cor ``` ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) await brain.init() // native providers registered during init @@ -71,7 +71,7 @@ remains available on npm if you need it. Brainy ships with full TypeScript types. No `@types/` package needed: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/migration-3.36.0.md b/docs/guides/migration-3.36.0.md index 8b1f239e..5f00534a 100644 --- a/docs/guides/migration-3.36.0.md +++ b/docs/guides/migration-3.36.0.md @@ -66,7 +66,7 @@ const results = await brain.search("query") **New diagnostics for capacity planning and performance tuning.** ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')} ### Step 1: Update Package ```bash -npm install @soulcraft/brainy@latest +npm install @soulcraftlabs/brainy@latest ``` ### Step 2: Restart Your Application @@ -134,7 +134,7 @@ npm run start ### Check Adaptive Sizing is Working ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -218,7 +218,7 @@ For debugging or compatibility testing: If you need to rollback to v3.35.0: ```bash -npm install @soulcraft/brainy@3.35.0 +npm install @soulcraftlabs/brainy@3.35.0 ``` **Note:** We don't anticipate any issues, but rollback is straightforward if needed. @@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) { ## Next Steps -1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest` +1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest` 2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements 3. 🎯 **Tune:** Adjust based on recommendations (if needed) 4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning diff --git a/docs/guides/model-loading.md b/docs/guides/model-loading.md index e5b7b1d6..cc1b2b6a 100644 --- a/docs/guides/model-loading.md +++ b/docs/guides/model-loading.md @@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings. ```bash # Bun as a runtime — supported and recommended -bun add @soulcraft/brainy +bun add @soulcraftlabs/brainy bun run server.ts ``` diff --git a/docs/guides/namespace-migration.md b/docs/guides/namespace-migration.md new file mode 100644 index 00000000..f7d2c7f7 --- /dev/null +++ b/docs/guides/namespace-migration.md @@ -0,0 +1,99 @@ +--- +title: Migrating to 9.0 — your fields and system fields +slug: guides/namespace-migration +public: true +category: guides +template: guide +order: 1 +description: The simple story of the 9.0 field-addressing change and the mechanical checklist for updating your call sites — every miss fails loudly with the fix in the error. +next: + - concepts/field-addressing +--- + +# Migrating to 9.0 — your fields and system fields + +The one-sentence version: **your data's field names are now completely +yours, the engine's own fields all live behind one `system.` prefix, and +nothing in between can silently go wrong anymore.** + +## What changed, simply + +**1. Any field name just works.** Before 9.0 the engine quietly owned +certain names. A field called `level` could be shadowed by the engine's +internal index layer of the same name (sorts silently returned insertion +order); names like `confidence` or `subtype` were rejected inside +`metadata`; names like `content` or `id` were silently never indexed, so +filtering on them returned nothing. All of that is gone. Any name — +`level`, `confidence`, `type`, `id`, `content`, anything — is stored +exactly as written and works with every feature: filtering, sorting, +grouping, aggregation, search, and time-travel reads. + +**2. The engine's fields moved behind `system.`.** The engine still keeps +its own per-record bookkeeping — creation time, type, confidence, and so +on. Those are reached one way only now: spelled out, e.g. +`system.createdAt`, `system.type`. They are just as queryable and sortable +as before. `orderBy: 'createdAt'` means *your* field named `createdAt`; +`orderBy: 'system.createdAt'` means the engine's timestamp. No guessing, +no priority rules. + +**3. Storage keeps the two physically separate.** New records store your +metadata in its own nested compartment, so a user field named +`confidence` and the engine's confidence live side by side, both intact, +through restarts, index rebuilds, and `asOf()` history. Old records stay +readable forever; nothing rewrites your data. + +**4. Mistakes are loud.** An ambiguous or unknown field name is a typed +error naming the fix. Unimplemented options refuse instead of being +ignored. The only forbidden name in your metadata is one literally +starting with `system.`. + +## The mechanical checklist + +Every missed site fails **loudly** with the correction in the error +message — nothing silently changes meaning. Sweep these patterns: + +| Before (8.x) | After (9.0) | +|---|---| +| `orderBy: 'createdAt'` (meaning the engine timestamp) | `orderBy: 'system.createdAt'` | +| `where: { subtype: 'invoice' }` (the engine subtype) | `where: { 'system.subtype': 'invoice' }` | +| `where: { confidence: { greaterThan: 0.8 } }` (the engine scalar) | `where: { 'system.confidence': { greaterThan: 0.8 } }` | +| `groupBy: ['noun']` or `groupBy: ['type']` | `groupBy: ['system.type']` | +| `where: { visibility: 'internal' }` / `{ service: … }` (engine values) | `'system.visibility'` / `'system.service'` | +| `metadata: { confidence: 0.9 }` expecting a throw or a lift to the engine scalar | it is YOUR field now — set the engine scalar via the `confidence` param | +| `new Brainy({ reservedFieldPolicy: … })` | remove the option (it throws with this note) | +| `find({ cursor })` / `includeRelations` / `writeOnly` | refuse with `UnsupportedFindOptionError` — they were silently ignored before | + +If a bare name in a query was genuinely *your* field all along (`orderBy: +'score'`, `where: { status: 'active' }`), **change nothing** — bare names +mean your fields, always. + +## What happens at first open + +Each existing database rebuilds its derived indexes once, automatically, +at the first open on 9.0 (index epoch 3 — the index keys split the two +namespaces). One-time cost, observable via `getIndexStatus()`; no manual +step, and your stored data is not modified. + +## For tooling and raw-record readers + +If you read raw stored records (fact-log scanners, export tooling), use +the exported shape-aware splitters — they handle both record eras: + +```typescript +import { splitNounMetadataRecord } from '@soulcraftlabs/brainy' +const { reserved, custom } = splitNounMetadataRecord(rawRecord) +// reserved = engine fields · custom = the user's bag, ANY names +``` + +Feature detection (never version-sniff): + +```typescript +import * as brainy from '@soulcraftlabs/brainy' +const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1' +``` + +## Where to go next + +- [Field addressing](../concepts/field-addressing.md) — the full contract: + the ten system scalars, the relation mirror, refusal semantics, and the + cross-engine ordering guarantees. diff --git a/docs/guides/nextjs-integration.md b/docs/guides/nextjs-integration.md index ab55e51f..25d6062d 100644 --- a/docs/guides/nextjs-integration.md +++ b/docs/guides/nextjs-integration.md @@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou ```bash npx create-next-app@latest my-brainy-app cd my-brainy-app -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Setup @@ -18,7 +18,7 @@ npm install @soulcraft/brainy // app/components/BrainyProvider.jsx 'use client' import { createContext, useContext, useEffect, useState } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const BrainyContext = createContext() @@ -271,7 +271,7 @@ export default function SearchPage() { ```javascript // app/api/search/route.js (App Router) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -332,7 +332,7 @@ export async function GET() { ```javascript // pages/api/search.js (Pages Router) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -374,7 +374,7 @@ export default async function handler(req, res) { ```javascript // app/api/data/route.js -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -418,7 +418,7 @@ export async function POST(request) { ```jsx // app/actions/brainy.js 'use server' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -630,7 +630,7 @@ CMD ["npm", "start"] /** @type {import('next').NextConfig} */ const nextConfig = { experimental: { - serverComponentsExternalPackages: ['@soulcraft/brainy'] + serverComponentsExternalPackages: ['@soulcraftlabs/brainy'] }, webpack: (config, { isServer }) => { if (!isServer) { @@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) { // app/contexts/BrainyContext.jsx 'use client' import { createContext, useContext, useReducer, useEffect } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const BrainyContext = createContext() @@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider' import { Search } from '../app/components/Search' // Mock Brainy -jest.mock('@soulcraft/brainy', () => ({ +jest.mock('@soulcraftlabs/brainy', () => ({ Brainy: jest.fn().mockImplementation(() => ({ init: jest.fn().mockResolvedValue(undefined), find: jest.fn().mockResolvedValue([ diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index a21a8af0..2984998b 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin Every distributed-job scheduler eventually wants this exact loop: ```ts -import { Brainy, RevisionConflictError } from '@soulcraft/brainy' +import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy' const LOCK_ID = '...uuid for this job slot...' @@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved: ```ts -import { GenerationConflictError } from '@soulcraft/brainy' +import { GenerationConflictError } from '@soulcraftlabs/brainy' async function addIfMissingByEmail(email: string, data: string) { for (let attempt = 0; attempt < 5; attempt++) { @@ -180,3 +180,35 @@ Brainy 8.0 has exactly two write-coordination counters, at two granularities: They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md). A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from. + +## The transact envelope: batch size, budget, and bulk imports + +`transact()` applies its batch atomically under one commit — which means the whole batch +shares one **apply budget**. Since 8.7.0 the budget scales with the batch: +`max(30 s, opCount × 2 s)`, or exactly what you pass as `timeoutMs`. A tripped budget rolls +the entire batch back (nothing partial survives) and throws a retryable +`TransactionTimeoutError` that names the operation it stopped at, the batch size, and the +elapsed vs budgeted time — a diagnosis, not just a failure: + +``` +Transaction timed out at operation 41/120 ('add') — 246012ms elapsed, budget 240000ms. +The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch. +``` + +Practical envelope guidance for bulk work: + +1. **Precompute embeddings outside the commit path.** Embedding inside `transact()` spends + the budget on model inference. Use `brain.embedBatch(texts)` and pass each vector via + the op's `vector` field — the commit then pays only storage costs, and a retried batch + never re-pays inference. (The win is *where* the inference happens, not raw embedding + throughput: on the default WASM engine, batch and sequential embedding measure + comparably, ~160 ms/text; native embedding providers may batch faster.) +2. **Chunk very large imports** into batches of a few hundred ops with one `transact()` + each. You lose whole-import atomicity but keep per-chunk atomicity, bounded memory, and + resumability — pair with `ifAbsent` upserts so a retried chunk is idempotent. +3. **Slow disks change the math, not the contract.** On network-attached storage a single + op can cost ~2 s (canonical write + fsync + index maintenance). The scaled default + absorbs that; pass an explicit `timeoutMs` only when you know better than the scale. +4. **`addMany`/`relateMany` are the convenience tier** — they chunk and batch-embed for + you, with per-item error reporting instead of batch atomicity. Choose by what you need: + atomic-all-or-nothing → `transact()`; resilient bulk load → `addMany`. diff --git a/docs/guides/quick-start.md b/docs/guides/quick-start.md index 097c55fe..d9a4e896 100644 --- a/docs/guides/quick-start.md +++ b/docs/guides/quick-start.md @@ -18,13 +18,13 @@ Get Brainy running in under a minute. ## 1. Install ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ## 2. Initialize ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -67,7 +67,7 @@ await brain.relate({ ## 5. Query with Triple Intelligence ```typescript -import type { Result } from '@soulcraft/brainy' +import type { Result } from '@soulcraftlabs/brainy' // All three search paradigms in one call const results: Result[] = await brain.find({ diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md index 6f7b150a..490aecab 100644 --- a/docs/guides/snapshots-and-time-travel.md +++ b/docs/guides/snapshots-and-time-travel.md @@ -9,6 +9,7 @@ description: Recipes for the Db API — instant backups with persist(), restore, next: - concepts/consistency-model - guides/optimistic-concurrency + - guides/external-backups --- # Snapshots & Time Travel @@ -41,6 +42,10 @@ bytes. Cross-device targets fall back to per-file byte copies, and persisting an in-memory brain serializes it to the same directory layout — a real, durable store. +> Archiving a brain directory with **external tools** (`tar`, `rsync`, `cp`)? +> Some index files are sparse and can explode to their apparent size under a +> naive copy — see [External Backups & Sparse Storage](/docs/guides/external-backups). + Two things to know: - `persist()` requires the view to still be the store's **latest** @@ -339,8 +344,12 @@ For per-entity write coordination (rather than whole-store history), the ## Keeping history bounded Under Model-B every write is a generation, so history can grow quickly — -Brainy auto-compacts on every `flush()`/`close()` under the **`retention`** -knob (configured on the constructor): +Brainy auto-compacts at `close()` (time-bounded per pass) under the +**`retention`** knob (configured on the constructor). Since 8.9.0, `flush()` +never compacts: flushing is durability work and costs only what the current +window's writes cost, regardless of history backlog. A long-lived writer that +never closes keeps its history until its next explicit `compactHistory()` — +schedule one in your maintenance window if you run bounded retention: ```typescript // Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows, @@ -354,10 +363,13 @@ new Brainy({ retention: 'all' }) new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } }) ``` -Reclaim manually at any time (the same caps): +Reclaim manually at any time (the same caps, plus an optional per-pass time +budget for maintenance windows — an early stop is a consistent prefix and the +next pass resumes): ```typescript await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 }) +await brain.compactHistory({ maxBytes: 512 * 1024 ** 2, timeBudgetMs: 10_000 }) ``` Compaction never breaks a pinned read — record-sets are reclaimed only when diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md index 9f2e2e5b..27dabe75 100644 --- a/docs/guides/standard-import-progress.md +++ b/docs/guides/standard-import-progress.md @@ -11,7 +11,7 @@ ### One Interface for Everything ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = await Brainy.create() @@ -78,7 +78,7 @@ interface ImportProgress { ```typescript import { useState } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' function UniversalImportProgress({ file }: { file: File }) { const [progress, setProgress] = useState({ @@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) { ```typescript import ora from 'ora' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function importWithProgress(filePath: string) { const spinner = ora('Starting import...').start() diff --git a/docs/guides/storage-adapters.md b/docs/guides/storage-adapters.md index 06ec9f3a..a4224bc8 100644 --- a/docs/guides/storage-adapters.md +++ b/docs/guides/storage-adapters.md @@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map). ## Quick start ```ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Filesystem (recommended for any persistent workload): const brain = new Brainy({ @@ -134,7 +134,7 @@ config; the `type` is optional. If you want to skip the factory: ```ts -import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy' +import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy' const fsStorage = new FileSystemStorage('./brainy-data') const memStorage = new MemoryStorage() diff --git a/docs/guides/subtypes-and-facets.md b/docs/guides/subtypes-and-facets.md index ff5de320..74311528 100644 --- a/docs/guides/subtypes-and-facets.md +++ b/docs/guides/subtypes-and-facets.md @@ -34,7 +34,7 @@ Three layers solve this: ### Write ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -240,7 +240,7 @@ await brain.migrateField({ A realistic adoption sequence for a brain that started without these primitives: ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } }) await brain.init() diff --git a/docs/guides/upgrading-7-to-8.md b/docs/guides/upgrading-7-to-8.md index a3c64fb9..53aa2a5c 100644 --- a/docs/guides/upgrading-7-to-8.md +++ b/docs/guides/upgrading-7-to-8.md @@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you. ## TL;DR -- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.** +- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.** If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, with no operator action. - Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. @@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and open it.** ```ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Opening the store is all that is required — recovery runs during init(). const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) @@ -182,5 +182,5 @@ and opening each store is sufficient. The recovery is copy-only, so no rollback of the recovery itself is ever needed. If you need to roll back the **whole** 7→8 upgrade, restore the directory from your pre-upgrade backup (retained automatically while recovery is incomplete, or -your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old +your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old branch layout in place, so a directory-level restore is the rollback path. diff --git a/docs/guides/vue-integration.md b/docs/guides/vue-integration.md index 7f7c6a06..34d18ebf 100644 --- a/docs/guides/vue-integration.md +++ b/docs/guides/vue-integration.md @@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N npm create vue@latest my-brainy-app cd my-brainy-app npm install -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Setup @@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun ```javascript // server/utils/brain.js (server-only — Nitro never bundles this into the client) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue' export default defineConfig({ plugins: [vue()], ssr: { - external: ['@soulcraft/brainy'] + external: ['@soulcraftlabs/brainy'] } }) ``` diff --git a/docs/neural-extraction.md b/docs/neural-extraction.md index 989b1b60..cfb6d764 100644 --- a/docs/neural-extraction.md +++ b/docs/neural-extraction.md @@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c ### Method 1: Brain Instance (Recommended) ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', { import { SmartExtractor, SmartRelationshipExtractor -} from '@soulcraft/brainy' +} from '@soulcraftlabs/brainy' // Or use subpath imports: -import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor' +import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor' const brain = new Brainy() await brain.init() @@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, { **Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration. ```typescript -import { SmartExtractor, FormatContext } from '@soulcraft/brainy' +import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy' const extractor = new SmartExtractor(brain, { minConfidence: 0.7, // Threshold @@ -229,7 +229,7 @@ interface ExtractionResult { **Relationship type classifier.** Determines verb/relationship types between entities. ```typescript -import { SmartRelationshipExtractor } from '@soulcraft/brainy' +import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' const relExtractor = new SmartRelationshipExtractor(brain, { minConfidence: 0.6, @@ -286,7 +286,7 @@ const rel = await relExtractor.infer( **Full extraction orchestrator.** Handles candidate detection, classification, and deduplication. ```typescript -import { NeuralEntityExtractor } from '@soulcraft/brainy' +import { NeuralEntityExtractor } from '@soulcraftlabs/brainy' const extractor = new NeuralEntityExtractor(brain) @@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location) ### Example 2: Excel Data Classification ```typescript -import { SmartExtractor } from '@soulcraft/brainy' +import { SmartExtractor } from '@soulcraftlabs/brainy' const extractor = new SmartExtractor(brain) @@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) { ### Example 3: Relationship Extraction ```typescript -import { SmartRelationshipExtractor } from '@soulcraft/brainy' +import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' const relExtractor = new SmartRelationshipExtractor(brain) diff --git a/docs/path-registry.md b/docs/path-registry.md new file mode 100644 index 00000000..8a55004c --- /dev/null +++ b/docs/path-registry.md @@ -0,0 +1,86 @@ +# The Path Registry — brainy's twin table + +The brainy half of the cross-engine Path Registry (the native accelerator +maintains the master list; IDs are shared and stable — `LC3`, `DP7`, … are +citable in commits, board rounds, release notes, and pins). Every row owes +five things: **service class** (INDEX-SERVED | BOUNDED-FALLBACK, announced | +TYPED REFUSAL), **latency budget** at 1k/10k/100k/1M (design bar: billions), +**lifecycle behavior**, **failure narration**, and a **test pin**. A path not +in this registry does not ship; an unregistered path is a red gate in the +scan audit. + +**The availability bar governing every row: user-visible downtime is +seconds, at restart only.** Migration, heal, compaction, embedding, and +retention run behind the doors — yielding, budget-capped, narrated. No path +may hold the doors while it does housekeeping. + +Status legend: ✅ contracted + pinned (test cited) · 🟡 partial (what holds +and what's missing, stated) · 🔴 owed (named, never silent). + +## LC — Lifecycle + +| ID | Brainy row | Status | +|----|-----------|--------| +| LC1 | Same-version reopen adopts everything: brain-format epoch match → zero rebuilds; aggregation state adopts by stamp; persisted indexes load. | ✅ `tests/unit/brainy/brain-format-handshake` + `migration-deference` (no-drift reopen never rebuilds) | +| LC2 | New empty brain: doors immediate. | ✅ exercised by every suite's setup | +| LC3 | Upgrade, same epoch: as LC1 — new code on unchanged formats owes nothing at open. | ✅ same pins as LC1 (epoch equality is the gate) | +| LC4 | Upgrade with epoch migration: TODAY brainy's epoch rebuild runs at open before doors. | 🔴 **owed — the sev's lockout row.** The doors-open-serving-old-structures design (yielding installments + atomic swap) lands measured-and-gated behind the service-class pair, per the lifecycle-sprint choreography. Acceptance case: the 9,184-row hours-lockout. | +| LC5 | Crash recovery: bounded, resumable, narrated. Aggregation leg ✅ (behind-stamp → incremental catch-up off the fact log + time-travel reconciliation, capped at 5,000 affected before an ANNOUNCED rescan). Vector/metadata legs ride epoch machinery (rebuild-from-canonical, narrated). | 🟡 aggregation pinned (`tests/integration/aggregation-lifecycle-catchup`); the rebuild legs are narrated but not yet installment-yielding (couples to LC4) | +| LC6 | Shutdown under load: close() drains the background flush flight, tears down cadence timers, runs ONE time-bounded compaction pass (~5s budget, resumable). | 🟡 pinned for flush/compaction (8.9.0 suites); SIGTERM drain budget not yet declared | +| LC7 | Rollback/downgrade: an N−1 build opening an N brain. | 🔴 owed — no declared read-compat window or typed refusal today (epoch mismatch triggers a rebuild, not a refusal; v2 nested-bag records read as a phantom user field on pre-law builds). Needs the declared-window contract. | +| LC8 | Relocatable brain directory: no absolute paths in artifacts; persist()/load() round-trips. | 🟡 persist/load pinned; byte-for-byte relocation depot cases are the pair gate's (shared corpora) | +| LC9 | Double-open: second writer gets a typed lock refusal (PID-liveness + heartbeat stale detection; `force` escape hatch logs loudly). | ✅ writer-lock suites (8.7.1) | + +## DP — Data plane + +| ID | Brainy row | Status | +|----|-----------|--------| +| DP1 | `get()` by id: direct storage read + hydrate. INDEX-SERVED (id-mapped). Milliseconds at every scale. | ✅ exercised everywhere; budget rides the pair speed table | +| DP2 | `find({query})`: embed + vector search. The embed dominates (native side owns the budget); JS HNSW serves the search leg. | 🟡 300ms-class p95 is the pair speed-table row; brainy-alone budget declared there | +| DP3 | Filtered/sorted list: column top-K when the field is columnized (INDEX-SERVED, zero canonical reads on the sorted page — value pairs come from ONE batched metadata-record pass); no-column fallback is BOUNDED-ANNOUNCED (one batch pass, announces once per field past 500 rows); unknown field → TYPED REFUSAL naming both candidate spellings. | ✅ `tests/unit/utils/metadataIndex-sort-callshape` (zero per-row reads, batch-only — latency-blind) + `metadataIndex-nested-orderby` (dotted keys serve-or-refuse) + `tests/integration/orderby-sort-bug` | +| DP4 | Aggregation/stats: ALWAYS answers. Write-time incremental; behind-stamp reconciles incrementally; genuine rebuilds go through the native parallel door or the paged JS walk; nothing ever latches off; before-image-less deletes flag a LOUD rescan, never a silent skip. | ✅ `tests/integration/aggregation-lifecycle-catchup` + `tests/unit/aggregation/aggregation-provider-rebuild` | +| DP5 | Graph traversal: `related()` paged via adjacency; whole-graph analytics carry declared cost. | 🟡 paged reads pinned; analytics cost-class declaration owed (rides VENUE-GRAPH-TRUST audit tool) | +| DP6 | Single write: ack at the canonical commit; visibility committed at ack (the atomic vector update kills the remove→add dark window); maintenance NEVER holds the ack (background flush cadence — THE ACK LAW pins: a hung flush cannot block a write, a hung EMBEDDER cannot block a write). | ✅ `tests/unit/brainy/persistence-policy` + `tests/unit/hnsw/update-item-atomic` + `tests/integration/deferred-embedding` | +| DP7 | Bulk ingest: sustained rate holds flat — per-write maintenance taxes must not grow with brain size (A4 removed caller-flush convoys; deferred embedding removes the per-write embed tax where opted). | 🟡 the decay-curve row is a pair speed-table RED GATE; brainy-alone sustained-rate run rides the same corpora | +| DP8 | Read under write pressure: no flicker window — a row that exists is never invisible to recall, even transiently (same-vector re-index is a no-op; changed-vector swaps in place, node never leaves the index; deferred updates serve the OLD vector until the atomic swap — stale-beats-absent). | ✅ brainy leg pinned (`tests/unit/hnsw/update-item-atomic` 9/9 + `deferred-embedding` stale-beats-absent); the symmetry property suite + runtime sentinels remain the B4 program | +| — | **As-of semantic recall** (time-travel vector search): `asOf(G).find()` serves the vectors AS THEY STOOD at G — byte-exact past vectors, tombstone masking, the deferred-embed cell honest on the vector leg, TYPED refusal beyond the head. Brainy-alone leg = ephemeral at-generation materialization (documented O(n log n at G) build, bounded); the at-scale leg rides the accelerated provider's as-of index. | ✅ `tests/integration/asof-semantic-recall` 4/4 (registry ID pending the master table's mint) | +| — | **The lazy-open gate honors EVERY provider's not-ready report** (a not-ready metadata provider can no longer latch the silent-empty state under `disableAutoRebuild`). | ✅ `tests/unit/brainy/lazy-notready-honor` | + +## MT — Maintenance (never in the door path) + +| ID | Brainy row | Status | +|----|-----------|--------| +| MT1 | Flush/checkpoint: ENGINE-OWNED cadence (write-count/interval/idle triggers, single-flight, background, loud on failure; callers never flush in hot paths; `flush()` stays as an awaitable barrier). | ✅ `tests/unit/brainy/persistence-policy` | +| MT2 | Compaction: never on flush (durability-only law, 8.9.0); close-time pass time-budgeted + resumable; explicit `compactHistory({timeBudgetMs})`. | ✅ 8.9.0 suites | +| MT3 | Index upkeep (mapper folds, delta promotion): native-side machinery; brainy's JS legs are small and synchronous-cheap. | 🟡 declared; yield audit rides the pair | +| MT4 | Heal/rebuild walks (`repairIndex`, backfill walks): paged; failure latches with cooldown; NOT yet yield-to-foreground installments. | 🔴 owed — the priority-isolation clause (couples to LC4; same choreography) | +| MT5 | Deferred embedding worker: ack at durability, durable pending markers (written BEFORE the commit — orphan-safe), crash-recovered at open via a bounded prefix listing, single-flight, 60s hang guard, `awaitPendingEmbeds()` barrier + `pendingEmbeds` gauge. VFS write paths adopt it end-to-end. | ✅ `tests/integration/deferred-embedding` 5/5 | +| MT6 | Retention/archival walks: retention `'all'` does nothing by design; bounded-retention reclaim is close-time/explicit only. | 🟡 8.9.0 behavior pinned; archival profile is the co-frozen D1+D3 unit | + +## FM — Failure modes + +| ID | Brainy row | Status | +|----|-----------|--------| +| FM1 | Disk full / IO error mid-op: transaction rollback + typed error; failed rollback → StoreInconsistentError quarantines writes until repairIndex(). | 🟡 rollback paths pinned; explicit disk-full depot case owed | +| FM2 | Memory pressure: query limits + reserved-memory config; unified cache eviction. | 🟡 declared budgets; cascade pin owed | +| FM3 | Torn/corrupt file on open: malformed brain-format marker → safe rebuild (never trusting a bad epoch); corrupt records surface loudly. | 🟡 marker pin ✅ (`brain-format-handshake`); broader quarantine is native-side | +| FM4 | Native module unavailable: plugin load failure is LOUD (version-coupling law throws on range mismatch — never silently version-drifted); JS engine serves with its own declared budgets, named as the active backend in op names. | ✅ `tests/unit/plugin-version-coupling` + op-name stamping | + +## FL — Fleet + +| ID | Brainy row | Status | +|----|-----------|--------| +| FL1 | Cold open on demand: LC1's adopt-everything open; warm() available for eager paths. | 🟡 open cost pinned at LC1; millisecond budget rides the speed table | +| FL2–FL4 | Boot storm / upgrade wave / isolation: fleet-layer policies over LC1/LC4 — engine leg = budgeted opens + LC4's behind-doors migration. | 🔴 owed with LC4 | +| FL5 | Brain as product object: create instant (LC2) · erase = `clear()` explicit + complete · export = portable-graph, canon-complete mode available. | ✅ clear-persistence + portable-graph + canonical-enumeration suites | + +## Status summary + +Contracted + pinned this train: **DP3, DP4, DP6, DP8(brainy leg), MT1, +MT5, LC5(aggregation), the lazy-open not-ready gate, LC1/LC3/LC9, FM4, +FL5** — each with the cited test. Owed, in production-risk order, all +coupled to the priority-isolation program the lifecycle sev opened: **LC4 +(doors-open migration), MT4 (yielding heals), LC7 (downgrade contract), +LC6 (SIGTERM budget), FL2–FL4, FM1/FM2 depot cases, B4 symmetry suite + +sentinels.** Rows move from owed to contracted only with a cited test — +none lands by prose. diff --git a/docs/performance-envelopes.md b/docs/performance-envelopes.md new file mode 100644 index 00000000..d29677e3 --- /dev/null +++ b/docs/performance-envelopes.md @@ -0,0 +1,83 @@ +--- +title: Performance Envelopes +slug: guides/performance-envelopes +public: true +category: guides +template: guide +order: 40 +description: Measured per-operation latency envelopes at stated scales — what to expect, on what hardware, and exactly how each number was produced. +next: + - guides/find-limits +--- + +# Performance Envelopes + +Every number on this page is **measured, never projected** — produced by the script +cited at the bottom, against the built package (the artifact you install), on the stated +hardware. Each entry says what was measured, at what scale, on which storage backend. +When a release touches a measured path, that operation is re-measured and this page +updates in the same release. + +Two scopes to keep straight: + +- **These envelopes are the pure-JS engine** (no native accelerator registered) on + filesystem storage. This is the floor every deployment gets from `npm install` alone. +- **Accelerated deployments** (the optional native provider) publish their own numbers — + this page never claims them. + +## Read operations + +Reads are where the architecture pays off: after the write path has done its indexing +work, queries answer from purpose-built indexes without scanning. + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `get(id)` (warm) | p50 < 0.1ms | p50 < 0.1ms | served from cache/metadata index | +| `find` (metadata: indexed equality + range, limit 100) | p50 1.0ms · p95 1.8ms | p50 7.0ms · p95 8.9ms | column-store bitmap paths | +| `related(id)` (per-node adjacency) | p50 < 0.1ms · p95 0.2ms | p50 < 0.1ms | LSM adjacency index — O(degree), scale-independent | +| `find` (semantic: embed + HNSW, 1k docs) | p50 178ms · p95 393ms | — | dominated by WASM query embedding (measured on a machine under concurrent load — treat the p95 as an upper bound); the vector search itself is single-digit ms | + +## Write operations + +Under Model-B **every write is its own durable generation** — a single-op `add` pays +serialization, before-image staging, and fsync before it acks. That durability is priced +into the write path visibly, by design: + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `add` (single-op) | p50 167ms · p95 171ms | p50 165ms · p95 172ms | full durable generation per write — flat across scale | +| `addMany` (bulk) | ~163ms/entity | ~187ms/entity | **currently per-item commits** — see the honest note below | +| `relateMany` | ~0.8ms/edge | ~0.9ms/edge | edges batch efficiently today | +| `flush` (steady-state, 1 pending write) | p50 8ms · p95 10ms | p50 45ms · p95 52ms | durability-only since 8.9.0 — cost no longer depends on history backlog or retention mode | + +**The honest note on bulk writes:** `addMany` today commits each item as its own +generation (the same durability as single-op `add`, serialized by the single-writer +lock), so bulk-load cost is N × single-op cost. Batched chunk commits (one generation +and one fsync window per chunk, as `removeMany` already does) are designed into the +unified-commit work on the current roadmap. Until that ships, size bulk imports +accordingly — 10k entities is minutes, not seconds, on filesystem storage. + +## Open / close + +| Operation | 1,000 entities | 10,000 entities | Notes | +|---|---|---|---| +| `open` (empty store) | ~560ms | ~190ms | includes embedder initialization | +| `open` (warm, populated, clean shutdown) | 763ms | 4.9s | pure-JS vector index load dominates and grows with entity count; the native accelerator exists precisely to remove this | +| `close` | bounded | bounded | auto-compaction pass is time-bounded (~5s max) since 8.9.0 | + +A store that was NOT cleanly closed pays index rebuilds on top of the warm-open +number (tens of seconds at 10k) — clean shutdown is worth engineering for. + +## How these were produced + +- **Hardware**: Intel Core i9-14900HX (32 threads), 62GB RAM, NVMe, Linux, Node v22. +- **Backend**: `storage: { type: 'filesystem' }`, pure JS (no native providers). +- **Embeddings**: deterministic stub for non-semantic ops (isolates engine cost); + the real WASM embedder for the semantic row (that's what you'll run). +- **Method**: p50/p95 over 50–200 samples per op against the built `dist/`; + the measuring script ships in the repo history and re-runs per release. + +Numbers on different hardware will differ; the *shape* (sub-2ms indexed reads, +~160ms embedding-bound semantic queries, durability-priced writes) is the envelope +you should hold your deployment against. If your measurements diverge from these +shapes by an order of magnitude, something is wrong — file it. diff --git a/docs/transactions.md b/docs/transactions.md index fce7d10e..cbea39c0 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing }) ### Basic Add Operation ```typescript -import { Brainy } from '@soulcraft/brainy' -import { NounType } from '@soulcraft/brainy/types' +import { Brainy } from '@soulcraftlabs/brainy' +import { NounType } from '@soulcraftlabs/brainy/types' const brain = new Brainy() await brain.init() @@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked ```typescript import { describe, it, expect } from 'vitest' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' describe('Transaction Tests', () => { it('should rollback on failure', async () => { diff --git a/docs/universal-display-augmentation.md b/docs/universal-display-augmentation.md index da42874c..464b91fb 100644 --- a/docs/universal-display-augmentation.md +++ b/docs/universal-display-augmentation.md @@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati ### Basic Usage ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brainy = new Brainy() await brainy.init() diff --git a/docs/vfs/PROJECTION_STRATEGY_API.md b/docs/vfs/PROJECTION_STRATEGY_API.md index 380862e1..f1319d5b 100644 --- a/docs/vfs/PROJECTION_STRATEGY_API.md +++ b/docs/vfs/PROJECTION_STRATEGY_API.md @@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low): ### Step 1: Create the Strategy Class ```typescript -import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' -import { Brainy } from '@soulcraft/brainy' -import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs' +import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' +import { Brainy } from '@soulcraftlabs/brainy' +import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs' export class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' @@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy { ### Step 2: Register the Strategy ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { PriorityProjection } from './PriorityProjection' const brain = new Brainy() @@ -537,7 +537,7 @@ Use the projection's resolve cache: ```typescript import { describe, it, expect, beforeAll } from 'vitest' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { PriorityProjection } from './PriorityProjection' describe('PriorityProjection', () => { @@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) { 3. Use appropriate limits: Don't fetch more than needed ### Type errors -1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'` +1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'` 2. Use `as VFSEntity` when mapping results 3. Check BaseProjectionStrategy import diff --git a/docs/vfs/QUICK_START.md b/docs/vfs/QUICK_START.md index 8b0efce6..4a1f83dc 100644 --- a/docs/vfs/QUICK_START.md +++ b/docs/vfs/QUICK_START.md @@ -14,11 +14,11 @@ A file explorer that: ## ⚡ Step 1: Basic Setup (1 minute) ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // ✅ CORRECT: Use filesystem storage for production const brain = new Brainy({ @@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns: ```tsx import React, { useState, useEffect } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' export function FileExplorer() { const [brain, setBrain] = useState(null) @@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next: ### "Module not found" errors ```bash # Make sure you're using the right import -npm ls @soulcraft/brainy # Check version -npm install @soulcraft/brainy@latest # Update if needed +npm ls @soulcraftlabs/brainy # Check version +npm install @soulcraftlabs/brainy@latest # Update if needed ``` ### "VFS not initialized" errors diff --git a/docs/vfs/README.md b/docs/vfs/README.md index b95f0d7b..a94910c9 100644 --- a/docs/vfs/README.md +++ b/docs/vfs/README.md @@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne ## Quick Start ```javascript -import { VirtualFileSystem } from '@soulcraft/brainy/vfs' +import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs' // Initialize the VFS const vfs = new VirtualFileSystem({ @@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system: ## Installation ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ## Requirements diff --git a/docs/vfs/ROADMAP.md b/docs/vfs/ROADMAP.md index 93c5b901..c8d15cd2 100644 --- a/docs/vfs/ROADMAP.md +++ b/docs/vfs/ROADMAP.md @@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows. ```typescript // Planned (research phase) -import { mountVFS } from '@soulcraft/brainy/vfs/fuse' +import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse' await mountVFS(vfs, { mountPoint: '/mnt/brainy', @@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested ### Express.js Static Middleware ```typescript // Wanted: Community contribution -import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express' +import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express' app.use('/files', createStaticMiddleware(vfs, { index: ['index.html', 'index.md'], @@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, { ### VSCode Extension ```typescript // Wanted: Community contribution -import { VFSProvider } from '@soulcraft/brainy/vfs/vscode' +import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode' const provider = new VFSProvider(vfs) vscode.workspace.registerFileSystemProvider('brainy', provider) diff --git a/docs/vfs/SEMANTIC_VFS.md b/docs/vfs/SEMANTIC_VFS.md index 9298c822..f34ee9ae 100644 --- a/docs/vfs/SEMANTIC_VFS.md +++ b/docs/vfs/SEMANTIC_VFS.md @@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true Create your own semantic dimensions: ```typescript -import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' +import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' diff --git a/docs/vfs/VFS_API_GUIDE.md b/docs/vfs/VFS_API_GUIDE.md index e0c6a94c..5dcaaeb8 100644 --- a/docs/vfs/VFS_API_GUIDE.md +++ b/docs/vfs/VFS_API_GUIDE.md @@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha ## Quick Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Initialize Brainy const brain = new Brainy({ @@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123') VFS uses standard POSIX-style errors: ```typescript -import { VFSError, VFSErrorCode } from '@soulcraft/brainy' +import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy' try { await vfs.readFile('/nonexistent.txt') diff --git a/docs/vfs/VFS_CORE.md b/docs/vfs/VFS_CORE.md index 1eeaf9f8..c1d502c0 100644 --- a/docs/vfs/VFS_CORE.md +++ b/docs/vfs/VFS_CORE.md @@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities: #### GitBridge Usage ```javascript // Import and instantiate GitBridge -import { GitBridge } from '@soulcraft/brainy' +import { GitBridge } from '@soulcraftlabs/brainy' const gitBridge = new GitBridge(vfs, brain) // Export VFS to Git repository structure @@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare ## Complete Example ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function vfsExample() { // Initialize diff --git a/docs/vfs/VFS_GRAPH_TYPES.md b/docs/vfs/VFS_GRAPH_TYPES.md index 3c1f30f0..478bef7f 100644 --- a/docs/vfs/VFS_GRAPH_TYPES.md +++ b/docs/vfs/VFS_GRAPH_TYPES.md @@ -196,5 +196,5 @@ await brain.relate({ Always import and use the type enums: ```javascript -import { NounType, VerbType } from '@soulcraft/brainy' +import { NounType, VerbType } from '@soulcraftlabs/brainy' ``` \ No newline at end of file diff --git a/docs/vfs/VFS_INITIALIZATION.md b/docs/vfs/VFS_INITIALIZATION.md index 97e6b0bf..fd12fc71 100644 --- a/docs/vfs/VFS_INITIALIZATION.md +++ b/docs/vfs/VFS_INITIALIZATION.md @@ -5,7 +5,7 @@ The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed! ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Create and initialize Brainy const brain = new Brainy({ @@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e ## Complete Example ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function useVFS() { // Initialize Brainy @@ -100,7 +100,7 @@ useVFS().catch(console.error) ## TypeScript Usage ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' class FileManager { private brain: Brainy diff --git a/docs/vfs/building-file-explorers.md b/docs/vfs/building-file-explorers.md index 6bb31871..7514c12e 100644 --- a/docs/vfs/building-file-explorers.md +++ b/docs/vfs/building-file-explorers.md @@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues: ### Method 1: Use `getDirectChildren()` (Recommended) ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -97,7 +97,7 @@ Here's a complete example using React: ```tsx import React, { useState, useEffect } from 'react' -import { VirtualFileSystem } from '@soulcraft/brainy' +import { VirtualFileSystem } from '@soulcraftlabs/brainy' interface FileNode { name: string @@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) { If you must build trees manually from flat lists, use the `VFSTreeUtils`: ```typescript -import { VFSTreeUtils } from '@soulcraft/brainy/vfs' +import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs' // Get all entities somehow const allEntities = await vfs.getDescendants('/root') diff --git a/examples/bluesky-distributed-setup.js b/examples/bluesky-distributed-setup.js index 9e83cf25..e3b33506 100644 --- a/examples/bluesky-distributed-setup.js +++ b/examples/bluesky-distributed-setup.js @@ -7,7 +7,7 @@ * the Bluesky firehose with Brainy's distributed architecture */ -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { WebSocket } from 'ws' // ===================================================== diff --git a/examples/monitor-cache-performance.ts b/examples/monitor-cache-performance.ts index 9d50d476..87c965a2 100644 --- a/examples/monitor-cache-performance.ts +++ b/examples/monitor-cache-performance.ts @@ -14,7 +14,7 @@ * ts-node examples/monitor-cache-performance.ts */ -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // ANSI color codes for pretty output const colors = { diff --git a/integrations/README.md b/integrations/README.md index aa3d795b..de156623 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu ## Quick Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature. ### Minimal (in-memory): ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions()) ```typescript import express from 'express' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = express() const brain = new Brainy({ @@ -232,7 +232,7 @@ app.listen(3000, () => { ```typescript import { Hono } from 'hono' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = new Hono() diff --git a/integrations/google-sheets/README.md b/integrations/google-sheets/README.md index b2b0af3a..8309a30a 100644 --- a/integrations/google-sheets/README.md +++ b/integrations/google-sheets/README.md @@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings. The simplest way to enable all integrations: ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -112,7 +112,7 @@ With Express: ```javascript import express from 'express' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = express() const brain = new Brainy({ integrations: true }) diff --git a/package-lock.json b/package-lock.json index c4e4f9ab..c4f66561 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "@soulcraft/brainy", - "version": "8.3.3", + "name": "@soulcraftlabs/brainy", + "version": "10.4.4", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@soulcraft/brainy", - "version": "8.3.3", + "name": "@soulcraftlabs/brainy", + "version": "10.4.4", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index ee74980b..06ce0253 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { - "name": "@soulcraft/brainy", - "version": "8.3.3", + "name": "@soulcraftlabs/brainy", + "version": "10.4.4", + "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", @@ -126,15 +127,16 @@ "license": "MIT", "private": false, "publishConfig": { - "access": "public" + "access": "public", + "registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" }, - "homepage": "https://github.com/soulcraftlabs/brainy", + "homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy", "bugs": { - "url": "https://github.com/soulcraftlabs/brainy/issues" + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues" }, "repository": { "type": "git", - "url": "git+https://github.com/soulcraftlabs/brainy.git" + "url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git" }, "files": [ "dist/**/*.js", diff --git a/scripts/buildEmbeddedPatterns.ts b/scripts/buildEmbeddedPatterns.ts index 73e51224..c046df45 100644 --- a/scripts/buildEmbeddedPatterns.ts +++ b/scripts/buildEmbeddedPatterns.ts @@ -10,6 +10,7 @@ import { TransformerEmbedding } from '../src/utils/embedding.js' import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' +import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -97,13 +98,22 @@ async function buildEmbeddedPatterns() { // Convert to base64 for embedding in TypeScript const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - + + // Deterministic stamp: derived from the git commit time of this + // generator's inputs, never from wall-clock time — two builds of the + // same source tree must produce byte-identical output. + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') + const generatedStamp = resolveDeterministicStamp( + [path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath], + outputPath + ) + // Generate TypeScript file with everything embedded const tsContent = `/** * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${new Date().toISOString()} + * Generated: ${generatedStamp} * Patterns: ${libraryData.patterns.length} * Coverage: 94-98% of all queries * @@ -197,7 +207,6 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length} ` // Write the TypeScript file - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/buildTypeEmbeddings.ts b/scripts/buildTypeEmbeddings.ts index 61bcf238..688d6ac1 100644 --- a/scripts/buildTypeEmbeddings.ts +++ b/scripts/buildTypeEmbeddings.ts @@ -11,6 +11,7 @@ import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' import { NounType, VerbType } from '../src/types/graphTypes.js' +import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -373,12 +374,24 @@ async function buildTypeEmbeddings() { const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') + // Deterministic stamp: derived from the git commit time of this + // generator's inputs, never from wall-clock time — two builds of the + // same source tree must produce byte-identical output. + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') + const generatedStamp = resolveDeterministicStamp( + [ + path.join(__dirname, 'buildTypeEmbeddings.ts'), + path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts') + ], + outputPath + ) + // Generate TypeScript file const tsContent = `/** * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${new Date().toISOString()} + * Generated: ${generatedStamp} * Noun Types: ${nounTypes.length} * Verb Types: ${verbTypes.length} * @@ -395,7 +408,7 @@ export const TYPE_METADATA = { verbTypes: ${verbTypes.length}, totalTypes: ${totalTypes}, embeddingDimensions: ${embeddingDim}, - generatedAt: "${new Date().toISOString()}", + generatedAt: "${generatedStamp}", sizeBytes: { embeddings: ${buffer.byteLength}, base64: ${base64.length} @@ -494,7 +507,6 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n ` // Write the TypeScript file - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs new file mode 100644 index 00000000..be73d4ca --- /dev/null +++ b/scripts/emit-contract-manifest.mjs @@ -0,0 +1,128 @@ +#!/usr/bin/env node +/** + * Emit this build's API-contract manifest to docs/api-contract.json. + * + * WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the + * code the first time somebody adds one. This reads the surface the build + * actually exposes — the prototype's own methods and accessors, the exported + * error classes, the `where` operator sets, the field-addressing vocabulary, + * the health verdicts — so a diff between two engines' manifests is a diff + * between two engines, never between two authors. + * + * Requirement marking (required / optional per door) is NOT derivable from the + * surface — it is a commitment, recorded with the contract's owner rather than + * here. This manifest carries the surface; the promise lives with the contract. + * + * Usage: node scripts/emit-contract-manifest.mjs [--check] + * --check exits non-zero when the committed manifest is stale. + */ + +import { writeFileSync, readFileSync, existsSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..') +const OUT = join(ROOT, 'docs', 'api-contract.json') + +const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js')) +const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js')) +const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js')) +const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js')) + +/** Every own method and accessor on the class's prototype, minus the private ones. */ +function surfaceOf(ctor) { + const doors = [] + for (const name of Object.getOwnPropertyNames(ctor.prototype)) { + if (name === 'constructor' || name.startsWith('_')) continue + const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name) + if (!descriptor) continue + if (typeof descriptor.value === 'function') { + doors.push({ name, kind: 'method', arity: descriptor.value.length }) + } else if (descriptor.get) { + doors.push({ name, kind: 'accessor' }) + } + } + return doors.sort((a, b) => a.name.localeCompare(b.name)) +} + +const errors = Object.entries(errorsModule) + .filter(([name, value]) => typeof value === 'function' && /Error$/.test(name)) + .map(([name]) => name) + .sort() + +// The operator sets, read from the engine's own refusal message so the +// manifest can never disagree with the validator. +const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8') +const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set\(\[([\s\S]*?)\]\)/) +if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess') +const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() + +const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8') +const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) => + // Proven by the refusal path: these are the tokens with no case in the + // index's operator switch, so they fall to its default and are refused. + !new RegExp(`case '${op}':`).test(indexSource) +) +const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) + +const manifest = { + contractVersion: versionModule.contractVersion(), + engine: '@soulcraftlabs/brainy', + compatibility: { + minor: + 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', + major: + 'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused' + }, + doors: surfaceOf(Brainy), + errors, + operators: { + accepted, + servedOnIndexPath: servedOnIndex, + refusedByIndexPath: refusedByIndex, + combinators: ['allOf', 'anyOf', 'not'] + }, + fieldAddressing: { + systemKeyPrefix: 'system.', + systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(), + systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(), + plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort() + }, + health: { + verdicts: ['pass', 'warn', 'fail'], + healKinds: ['none', 'repair', 'rebuild'], + servingWithholdingInvariants: [ + 'index-initialized', + 'durable-state-present', + 'manifest-residency', + 'replay-clean', + 'strand-latch' + ] + } +} + +const rendered = `${JSON.stringify(manifest, null, 2)}\n` + +if (process.argv.includes('--check')) { + if (!existsSync(OUT)) { + console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`) + process.exit(1) + } + if (readFileSync(OUT, 'utf-8') !== rendered) { + console.error( + `docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` + + `the addition (minor = additive; a removal is a contract major).` + ) + process.exit(1) + } + console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`) + process.exit(0) +} + +writeFileSync(OUT, rendered) +console.log( + `Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` + + `${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` + + `${manifest.operators.accepted.length} operators ` + + `(${manifest.operators.refusedByIndexPath.length} refused by the index path).` +) diff --git a/scripts/gate/README.md b/scripts/gate/README.md new file mode 100644 index 00000000..0a8afab0 --- /dev/null +++ b/scripts/gate/README.md @@ -0,0 +1,85 @@ +# Gate Guards + +Two standalone scripts that stand between a test/build gate and a false +verdict: one refuses to let the gate start on a noisy machine, the other +refuses to let a truncated or crashed vitest run be read as green. + +## Why these exist + +Both guards exist because of the 2026-08-13 lost-day ledger: a gate ran on +a machine under load, and separately a vitest worker pool died mid-suite +while still printing a plausible-looking summary line, and in both cases +the bad result was trusted and acted on for the better part of a day before +anyone noticed. Neither failure mode announces itself — a loaded machine +still finishes and reports numbers, and a truncated test run still prints a +`Test Files` / `Tests` line — so both guards check the evidence explicitly +rather than trusting that a gate finishing means the gate was valid. + +## gate-preflight.sh + +Run before any gate lane starts. Exits 1 the moment the machine isn't +gate-clean, with one `FATAL:` line per violation naming the exact offender +(the pid and command, the path, the measured value). Prints one `OK:` line +per check that passes. `WARNING:` lines mark checks that were skipped, not +failures. + +Checks: + +| # | Check | Default threshold | Override | +|---|-------|--------------------|----------| +| a | 1-minute load average | `nproc / 2` | `GATE_MAX_LOAD` | +| b | any non-allowlisted process over 50% of one core | 50% | `GATE_ALLOW_REGEX` (extra pattern matched against the process's args) | +| c | cpu0 scaling governor must be `performance` | — | none (warns and skips if the sysfs path is absent) | +| d | free space on `/` and `/tmp` | 10G each | `GATE_SKIP_DISK_CHECK=1` to skip entirely | + +The allowlist for check (b) is always: this script's own process tree +(its ancestors and its direct child processes), `sshd`, `systemd`, and +kernel threads (recognizable by args wrapped in brackets, e.g. +`[kworker/0:1]`). `GATE_ALLOW_REGEX` extends it — it does not replace it. + +## vitest-verdict-check.sh + +Run after every vitest lane, against that lane's captured log. Fails +loudly, quoting the exact line or string that tripped it, when the log's +own summary can't be trusted: + +- no `Test Files` (or, in `--count-tests` mode, `Tests`) summary line is + present at all +- the parenthesized total in that line doesn't match what was expected +- fewer files/tests are accounted for (passed + failed + skipped) than the + total claims — a truncated run +- the log contains `Unhandled Error` or `Timeout calling` anywhere — a dead + worker pool, regardless of what the summary line claims + +``` +vitest-verdict-check.sh +vitest-verdict-check.sh --count-tests +``` + +The first form checks `Test Files` for an exact match. The second checks +`Tests` for a minimum (a floor, not an exact count, since the total number +of individual tests moves more often than the number of test files). + +## Wiring into a CI lane + +```sh +# Before any lane that will report a verdict: +scripts/gate/gate-preflight.sh || exit 1 + +# Run the suite, capturing its output: +npx vitest run tests/unit 2>&1 | tee /tmp/unit.log + +# After every vitest lane, check the log against the actual file count: +EXPECTED_FILES=$(ls tests/unit/**/*.test.ts | wc -l) +scripts/gate/vitest-verdict-check.sh /tmp/unit.log "$EXPECTED_FILES" || exit 1 +``` + +## Exit-code contract + +| Script | Exit 0 | Exit 1 | +|--------|--------|--------| +| `gate-preflight.sh` | machine is gate-clean | one or more `FATAL:` violations printed | +| `vitest-verdict-check.sh` | log's summary is trustworthy and matches | usage error, missing/unreadable log, or one or more `FATAL:` violations printed | + +Non-zero from either script means: do not trust the gate that was about to +run, or the result of the one that just ran. diff --git a/scripts/gate/gate-preflight.sh b/scripts/gate/gate-preflight.sh new file mode 100755 index 00000000..c6208f49 --- /dev/null +++ b/scripts/gate/gate-preflight.sh @@ -0,0 +1,206 @@ +#!/bin/bash +set -euo pipefail + +# Brainy Gate Preflight +# Refuses to let a test/build gate run on a machine that isn't clean enough +# to trust the numbers it produces. See scripts/gate/README.md for why (the +# 2026-08-13 lost-day ledger). +# +# Checks: 1-minute load average, any non-allowlisted process pinning a core, +# the cpu0 scaling governor, and free space on / and /tmp. +# +# Exit 0 and print one OK line per passing check when the machine is clean. +# Exit 1 and print one FATAL line per violation, naming the offender, when +# it is not. +# +# Known trap: a helper function whose last executed statement is a `while` +# (or any command whose own exit status happens to be nonzero) hands that +# status back as the function's return value. Called as a plain statement, +# that silently kills this script under `set -e`. Every helper below ends +# on an explicit `return 0` as its own statement, never on a loop or test. +# +# The same failure mode hides in plainer-looking lines too: `var=$(cmd)` is +# a bare assignment, so `set -e` DOES treat a nonzero `cmd` (or, under +# `pipefail`, a nonzero stage anywhere in `cmd`'s pipeline) as a failure of +# that statement and kills the script right there — even mid-loop, even +# when the "failure" is routine (a process that exited before a second +# lookup, a path that doesn't exist). Every such assignment below is paired +# with an explicit `|| var=""` fallback so a routine miss degrades to an +# empty value instead of an exit. + +VIOLATIONS=0 +ANCESTOR_PIDS="" + +fatal() { + echo "FATAL: $1" + VIOLATIONS=$((VIOLATIONS + 1)) +} + +ok() { + echo "OK: $1" +} + +# Walks this process's parent chain up to pid 1, then takes one snapshot of +# its direct children (the ps/read pipeline in check_processes), and +# records both in ANCESTOR_PIDS — so the process-scan below can recognize +# its own tree (the shell/terminal/session that launched it, plus its own +# helper commands) instead of flagging it. Children are captured once, up +# front, rather than re-queried per row later, so a helper command that has +# already exited by the time it's looked up can't be mistaken for a miss. +build_ancestor_pids() { + local pid="$$" + local ppid child + ANCESTOR_PIDS=" $pid " + while [ "$pid" != "1" ]; do + ppid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') || ppid="" + if [ -z "$ppid" ]; then + break + fi + ANCESTOR_PIDS="${ANCESTOR_PIDS}${ppid} " + pid="$ppid" + done + + while IFS= read -r child; do + [ -z "$child" ] && continue + ANCESTOR_PIDS="${ANCESTOR_PIDS}${child} " + done < <(ps --ppid "$$" -o pid= 2>/dev/null || true) + + return 0 +} + +# (a) 1-minute load average vs. threshold (default: nproc / 2). +check_load() { + local max_load="${GATE_MAX_LOAD:-}" + if [ -z "$max_load" ]; then + max_load=$(( $(nproc) / 2 )) + if [ "$max_load" -lt 1 ]; then + max_load=1 + fi + fi + + local load_1m + load_1m=$(cut -d' ' -f1 /proc/loadavg) + + if awk -v l="$load_1m" -v m="$max_load" 'BEGIN { exit !(l > m) }'; then + fatal "1-minute load average ${load_1m} exceeds threshold ${max_load} (GATE_MAX_LOAD=${max_load})" + else + ok "1-minute load average ${load_1m} is within threshold ${max_load}" + fi + return 0 +} + +# (b) any process outside the allowlist pinning more than half a core. +# Parsed with `read` into named fields, not an awk/cut chain — a fixed-column +# awk/cut split on `ps` output duplicated fields the first time this was +# tried, because process args vary in word count. `read` with a fixed list +# of variables dumps everything left over into the last one (args), which +# handles that correctly. +check_processes() { + local max_pcpu=50 + local extra_regex="${GATE_ALLOW_REGEX:-}" + local violation_found=0 + local line pcpu pid args pcpu_int + + while IFS= read -r line; do + [ -z "$line" ] && continue + read -r pcpu pid args <<< "$line" + + # Kernel threads report their comm in brackets, e.g. "[kworker/0:1]". + case "$args" in + \[*\]) continue ;; + esac + + # This script's own tree: its ancestors (shell, terminal, session) and + # its direct children, both captured once by build_ancestor_pids. + case " $ANCESTOR_PIDS " in + *" $pid "*) continue ;; + esac + + case "$args" in + *sshd*|*systemd*) continue ;; + esac + + if [ -n "$extra_regex" ] && [[ "$args" =~ $extra_regex ]]; then + continue + fi + + pcpu_int="${pcpu%.*}" + if [ -z "$pcpu_int" ]; then + pcpu_int=0 + fi + if [ "$pcpu_int" -gt "$max_pcpu" ]; then + fatal "pid ${pid} ('${args}') is using ${pcpu}% of one core" + violation_found=1 + fi + done < <(ps -eo pcpu,pid,args --sort=-pcpu | tail -n +2) + + if [ "$violation_found" -eq 0 ]; then + ok "no process outside the allowlist exceeds ${max_pcpu}% of one core" + fi + return 0 +} + +# (c) cpu0 scaling governor must be "performance". Skipped with a warning +# (not a violation) when the sysfs path doesn't exist on this machine. +check_governor() { + local gov_path="/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" + if [ ! -r "$gov_path" ]; then + echo "WARNING: ${gov_path} not present; skipping governor check" + return 0 + fi + + local governor + governor=$(cat "$gov_path" 2>/dev/null) || governor="" + if [ "$governor" != "performance" ]; then + fatal "cpu0 governor is '${governor}', not 'performance'" + else + ok "cpu0 governor is 'performance'" + fi + return 0 +} + +# (d) free-space floors on / and /tmp (default 10G each). Skip entirely via +# GATE_SKIP_DISK_CHECK=1. +check_disk() { + if [ "${GATE_SKIP_DISK_CHECK:-0}" = "1" ]; then + echo "WARNING: disk free-space check skipped (GATE_SKIP_DISK_CHECK=1)" + return 0 + fi + + local floor_gb=10 + local floor_bytes=$((floor_gb * 1024 * 1024 * 1024)) + local path avail_bytes avail_gb + + for path in / /tmp; do + avail_bytes=$(df --output=avail -B1 "$path" 2>/dev/null | tail -n 1 | tr -d ' ') || avail_bytes="" + if [ -z "$avail_bytes" ]; then + echo "WARNING: could not determine free space on ${path}; skipping" + continue + fi + if [ "$avail_bytes" -lt "$floor_bytes" ]; then + avail_gb=$((avail_bytes / 1024 / 1024 / 1024)) + fatal "${path} has only ${avail_gb}G free, below the ${floor_gb}G floor" + else + ok "${path} has enough free space (floor ${floor_gb}G)" + fi + done + return 0 +} + +echo "Brainy gate preflight" +echo "----------------------" + +build_ancestor_pids +check_load +check_processes +check_governor +check_disk + +echo "----------------------" +if [ "$VIOLATIONS" -gt 0 ]; then + echo "FATAL: gate preflight failed with ${VIOLATIONS} violation(s) — machine is not gate-clean" + exit 1 +fi + +echo "gate preflight passed — machine is gate-clean" +exit 0 diff --git a/scripts/gate/vitest-verdict-check.sh b/scripts/gate/vitest-verdict-check.sh new file mode 100755 index 00000000..36243a1d --- /dev/null +++ b/scripts/gate/vitest-verdict-check.sh @@ -0,0 +1,158 @@ +#!/bin/bash +set -euo pipefail + +# Brainy Vitest Verdict Check +# Confirms a vitest run's own summary line is trustworthy before anything +# downstream treats a green run as green. See scripts/gate/README.md for why +# (the 2026-08-13 lost-day ledger). +# +# Usage: +# vitest-verdict-check.sh +# vitest-verdict-check.sh --count-tests +# +# The first form checks the "Test Files" summary line's total against an +# exact expected count. The second checks the "Tests" summary line's total +# against a minimum. Both also fail on any sign the worker pool died +# mid-run, whether or not a summary line still made it into the log. +# +# Exit 0 and print one OK line per passing check when the log is clean. +# Exit 1 and print one FATAL line per violation, quoting the exact line or +# string that tripped it, when it is not. +# +# Known trap (shared with gate-preflight.sh): every helper below ends on an +# explicit `return 0` as its own statement, never on a loop or test, so a +# helper's last command can never hand its own exit status back as the +# function's under `set -e`. The same applies to `var=$(cmd)` assignments +# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that +# legitimately finds nothing (exit 1) would otherwise kill the script +# instead of just leaving the variable empty — every such assignment below +# is paired with an explicit `|| true` inside the substitution. + +usage() { + echo "Usage: $0 " + echo " $0 --count-tests " + exit 1 +} + +MODE="files" +if [ "${1:-}" = "--count-tests" ]; then + MODE="tests" + shift +fi + +LOG_FILE="${1:-}" +THRESHOLD="${2:-}" + +if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then + usage +fi + +if [ ! -f "$LOG_FILE" ]; then + echo "FATAL: log file '${LOG_FILE}' does not exist" + exit 1 +fi + +if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then + echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer" + exit 1 +fi + +VIOLATIONS=0 + +fatal() { + echo "FATAL: $1" + VIOLATIONS=$((VIOLATIONS + 1)) +} + +ok() { + echo "OK: $1" +} + +# Vitest colorizes its summary with ANSI escapes; strip them before parsing +# anything, or the color codes end up embedded in the fields we grep for. +CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")" + +# Worker-pool death: if either string appears, the run's own summary line — +# even if present and even if its numbers look fine — cannot be trusted, +# because the process died mid-suite and vitest's own accounting is what +# died with it. +check_worker_death() { + if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then + fatal "log contains 'Unhandled Error' — worker pool died mid-run" + fi + if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then + fatal "log contains 'Timeout calling' — worker pool died mid-run" + fi + return 0 +} + +# Shared shape between the "Test Files" and "Tests" summary lines: +#