diff --git a/.claude/skills/architecture.md b/.claude/skills/architecture.md index 4de3c287..de046b17 100644 --- a/.claude/skills/architecture.md +++ b/.claude/skills/architecture.md @@ -2,7 +2,7 @@ ## What Is Brainy -@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. +@soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. ## Core Architecture diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml deleted file mode 100644 index da5887f6..00000000 --- a/.forgejo/workflows/ci.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: CI - -# Branch pushes only — a release TAG deliberately does not re-run CI: the -# tagged commit's CI already ran on its branch push, and the runner is -# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the -# tag's publish-source run and starve every release (observed on 8.10.3 and -# 9.0.0: the publish sat behind the tag's own redundant CI). -concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: ['**'] - pull_request: - -jobs: - node: - name: Node ${{ matrix.node-version }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - node-version: ['22', '24'] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: npm - - run: npm ci - - run: npm run test:unit - - # The correctness plant's full gate: integration + conformance run here on - # dedicated iron, on every push, so a release never depends on any other - # machine being up. Verdicts live in this run's log (never inferred). - integration: - name: Integration + conformance (Node 22) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - run: npm ci - - run: npm run test:ci-integration - - run: npx vitest run tests/conformance - - bun: - name: Bun (latest) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - run: npm ci - # test:bun imports the built dist/, so build first. - - run: npm run build - # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). - - run: npm run test:bun diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml deleted file mode 100644 index 58cb1d30..00000000 --- a/.forgejo/workflows/publish-source.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Publish (The Source) - -# Datacenter-side publish to The Source (source.soulcraft.com — our -# self-hosted Forgejo; never call it "the forge", Forge is a different -# product), moved off the laptop: an 87MB tarball PUT over the laptop's WAN -# times out; The Source's own runner does it in seconds. -# scripts/release.sh tags + pushes, then polls this workflow's result (npm -# view against The Source's registry) before it ever touches the npmjs leg — -# see the "delegation contract" in scripts/release.sh's home-publish step. - -on: - push: - tags: - - 'v*' - -jobs: - publish: - name: Publish to The Source registry - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - run: npm ci - - run: npm run build - - name: Publish + readback-verify on The Source registry - env: - # The stored repo-settings secret keeps its historical name. - FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }} - run: | - set -eo pipefail - - SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" - VERSION="$(node -p "require('./package.json').version")" - # The dist-tag follows the version: a prerelease (any hyphen — - # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — - # every consumer resolving 'latest' from this registry would otherwise - # be handed a release candidate. Same rule scripts/release.sh applies - # to the storefront leg. - NPM_TAG="latest" - case "$VERSION" in - *-*) NPM_TAG="rc" ;; - esac - echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." - - TMPRC="$(mktemp)" - chmod 600 "$TMPRC" - { - echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}" - echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}" - } > "$TMPRC" - - # The release script bumps package.json's version before it tags, so - # this tag's checkout already carries the version being published — - # nothing here re-derives it from the tag name. - PUBLISH_OK=true - if ! npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then - PUBLISH_OK=false - fi - - # Readback verify is the source of truth, run regardless of the publish - # exit code: a benign duplicate publish (a prior run, or a mirror, already - # landed this exact version) reports failure even though the registry - # already holds the right content. - LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" - rm -f "$TMPRC" - - if [ "$LANDED_VERSION" != "$VERSION" ]; then - echo "::error::Readback verify FAILED — The Source registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." - exit 1 - fi - - if [ "$PUBLISH_OK" = true ]; then - echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry." - else - echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." - fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..cdb2ab14 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + pull_request: + +jobs: + node: + name: Node ${{ matrix.node-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ['22', '24'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + - run: npm ci + - run: npm run test:unit + + bun: + name: Bun (latest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + - run: npm ci + # test:bun imports the built dist/, so build first. + - run: npm run build + # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). + - run: npm run test:bun diff --git a/CHANGELOG.md b/CHANGELOG.md index a54d609e..fcfeadb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,314 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) - -- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) -- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25) -- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40) -- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2) -- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927) -- docs: measurements in public history carry numbers, not provenance (02c61636) -- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010) -- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950) -- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9) -- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5) -- perf(open): answer "are there any entities?" with one directory read (417ddb51) -- perf(generations): discover generations by directory name, not by walking the log (9dd39921) -- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc) -- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca) -- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f) -- chore: keep the generated neural stamps at main's values (c1f09723) -- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3) -- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02) -- feat(open): open never waits for a provider that is rebuilding itself (131daa08) -- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f) -- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6) -- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b) -- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f) -- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c) -- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b) - - -### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) - -- Merge branch 'next/open-brainy-rename' (a58372f0) -- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) -- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) - - -### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) - -- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) - - -### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - -- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) -- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) -- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1) -- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4) -- Merge branch 'next/enumeration-identity-rekey' (204d74c1) -- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16) -- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a) -- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab) -- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9) -- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245) -- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee) -- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb) -- chore(release): 10.4.1-rc.1 (7870dc40) - - -### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - -- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) -- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) - - -### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) - -- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) - - -### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) - -- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) - - -### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) - -- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) -- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) -- fix(add): empty string is real data, not a missing field (258e9042) -- feat(vfs): implement readdir's recursive option — typed since 7.30, never read (fc516da6) -- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) - - -### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) - -- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) -- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) -- fix(storage): an unknown nested storage config can never silently land on the shared default root (ddd5e719) -- docs(release): the 10.4.0 entry, the index-health concept doc, and the API surfaces — written from the tree, not the plan (8cced871) -- fix(plugins): the silent-degrade doors close — a broken accelerator install can never read as absent (b9ba50fb) -- feat(recovery): the catchup verdict is consumed; verb rows go live; the metadata rebuild goes online (18f172e0) -- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) - - -### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) - -- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) -- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) -- test(fold-checkpoint): the ARM-AT-FLIP pin arms its crash instead of racing the pending-flush timer (4176439b) -- fix(health): one contract for a throwing probe — heal is none, serving is not withheld; repair report gains missing/rebuilt/reason (116550eb) -- feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete (7c8c8be3) -- fix(delete): the null-metadata skip closes — index legs run id-keyed or narrate, never silently strand postings (607e9f54) -- feat(repair): repairIndex returns the per-family receipt and narrates its summary (8d45f964) -- fix(reads): the readiness gate guards every index read surface — serving empty from a not-ready provider is unrepresentable (40e7119b) -- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) - - -### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) - -- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) -- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) - - -### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) - -- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) -- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) -- test(budgets): iron-honest wall-clock budgets — 3x the worst honest-iron measurement (314e0e6c) -- fix(locks): live writers are never auto-evicted; evicted writers are fenced at every commit barrier (292e7c04) -- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) - - -### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) - -- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) -- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) -- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) - - -### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) - -- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) -- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) -- feat(recovery): the fold-checkpoint bound — crash folds (checkpoint, head], never the whole log twice (ff43de1a) -- fix(log): pad-frame construction is total; the at-ack sync-failure compensation splits by phase — a production adoption's two write-path defects, cured at their roots (cbe34d11) -- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) - - -### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) - -- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) -- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) -- fix(recovery): walks are healers — the typed/tolerant boundary redrawn where block-layer fault injection proved it belonged (0e3facf4) -- feat(log): log authority is the fleet default — adopt-at-open, oracle-gated; plus the power-cut throw-site cures and the loud torn-record contract (214c98b4) -- fix(durability): three block-layer power-loss findings from the first fault-injection box run — all cured, matrix 15/15 (67c606be) -- docs: RELEASES.md frames the release as 10.0.0 — honest major (log format v2 forward-only); comment wording cleanup (d1698fa5) -- fix(persistence): the idle flush trigger debounces under load — deferred to the floor, never dropped, never a flush-per-gap amplifier (a50726e6) -- feat(reprojection): the one doors-open machinery — budget-capped, yielding, foreground-preempted, atomic-swap; poison records quarantine typed (d1651f98) -- feat(embedding): deferred-embed markers become log records — the sidecar recovery path is deleted (b47787bb) -- feat(conformance): the golden-log fold oracle — encoder bytes and fold semantics pinned by content hash (c95bea88) -- feat(engine): the wiring wave — stamps ride every flush, provider generations, waitForIndexed, adopt-backfill, match-all serves (b53e6e89) -- feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data (b35d87a7) -- feat(log): v2 is the LIVE write format — envelope records with minted ints, genesis, sector seals; v1 readable forever (26c60251) -- docs: RELEASES.md — the unreleased write-path and lifecycle entry (consumer-facing draft; version set at cut) (73eb88d4) -- feat(temporal): as-of semantic recall joins the release contract — past vectors byte-exact, pinned (f7ca0d26) -- fix(log): acked writes survive power loss; rejected writes never silently commit — the kill-matrix goes 11/11 with zero .fails debt (13022c51) -- feat(plugin): every provider write surface carries the real committed generation (2d532684) -- feat(log): fact-log format v2 codec — record envelope, type registry, genesis, sector seals; fault-injection shim (34841074) -- feat(log): the guarded log-authority core — group-commit durable-at-ack, the per-brain switch, the verification oracle (65953097) -- docs: Path Registry rows DP6/DP8/MT5 flip to contracted+pinned — the deferred-embedding and atomic-update train landed with cited tests (9fda6d95) -- feat(embedding): MT5 — deferred embedding with durable markers; write acks never wait on a neural net (287384cf) -- fix(index): the flicker window dies — atomic in-place vector update; lazy open honors every provider's not-ready report; the Path Registry twin table (ebe06cdf) -- feat(persistence): the engine owns its flush cadence — callers never call flush() in hot paths again (3236a01b) -- fix(aggregation): the lifecycle cluster — flush stamps, behind-stamp catches up incrementally, the native rebuild finally gets invoked, deletes are never silently skipped (1dc861d2) -- perf(sort): ordered reads never do per-row storage round-trips — the 199-317s production scan class dies structurally (607b6b56) -- chore: the home registry is The Source, never 'the forge' — sweep the misnomer out of the release rail, workflows, and release notes (Forge is a different product; the stored CI secret keeps its historical name) (09352c2b) -- ci: tags stop triggering the CI matrix (redundant re-run of already-tested commits starved every release's publish run on the sequential runner) + release.sh forge poll window 20→50 min (c6c6ea6b) -- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) - - -### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) - -- docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) -- fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) -- docs: v9.0.0 release notes — the field-addressing law migration ledger; retitle the shipped 8.11.0 canonical-enumeration entry (header went stale at its cut) (55a7512c) -- feat(namespace): merge the field-addressing law train — no special names, system.* scalars, nested-bag storage, epoch-3 index keys (19b477ae) -- feat(namespace): NO SPECIAL NAMES + storage fidelity — the ruled completion of the field-addressing law (24bf6cdb) -- feat(namespace): write-door forgery refusal (user metadata keys may never start 'system.') + refusal messages name both spellings in every branch (the non-colliding case marks system. honestly as NOT valid) — cross-engine message pin alignment (48a6130a) -- feat(namespace): conformance green 19/19 — data-aware did-you-mean on unindexed bare addresses, ordering contract on the column top-K path (never drop, nulls last, ties by id), shape-complete addressed reads (entity views AND raw storage shapes, shadow-proof both scopes), per-key source matching for dotted addresses; refusal classes unified under UnresolvableFieldError (8e962dab) -- feat(namespace): aggregation reads under the law + epoch 3 (the key-split rebuild) + THE ARMING COMMIT — the capability constant, the law module, and the typed refusals export from the package root; both engines' conformance suites light on this signal (7492b6cb) -- feat(namespace): egress guard + validation speak the law — whereMatcher's resolver reads system.* from the record and bare names from the metadata bag only (the bare-system switch is dead); validateFindParams refuses cursor/includeRelations/writeOnly typed (accepted-and-ignored dies as a class), validates order, and parses every orderBy address (c2fb28a2) -- fix(namespace): noun-record updates preserve legacy inline HNSW adjacency — the placeholder-adjacency write stamped out pre-codec records' stored connections (crash-window unreachability); codec-era records were never at risk (empty field is the blob marker); pin covers the legacy shape (4679c894) -- feat(namespace): find's own filter builders speak the frozen keys — params.type/subtype/service become system.* index keys at every construction site (three pipelines + the canonical buildMetadataFilter); the where.type→noun alias is dead (bare 'type' belongs to the user now) (7a28a946) -- feat(namespace): the index speaks the frozen keys — record-frame scalars index under literal 'system.' (legacy 'noun' spelling folds into system.type; plumbing never indexed from a record frame), user fields stay bare in every shape; filter + sorted paths route every address through parseFieldAddress; storage fallbacks read the addressed side of the record (11c724bc) -- docs(namespace): the d.ts JSDoc wave — the sealed field-addressing law on the full find + aggregation surface, present-tense, with the refusal semantics and migration note inline (comment-only; verified zero code lines changed) (fcb24ab6) -- test(namespace): unit pins for the pure law — the ruled maps verbatim (incl. the relation mirror, unpinnable via public API), plumbing refusals both kinds, did-you-mean text (5502abcd) -- fix(namespace): the JS sorted fallback honors the ruled ordering contract — nulls last in BOTH directions (was nulls-first on desc) + deterministic id-ascending tie-break (56deb2e8) -- test(namespace)+docs: the cross-engine conformance suite (self-arming — skips until the resolver exports land) + the public field-addressing docs page; sidebar order deconflicted to 7 (d8d0b55f) -- feat(namespace): the one field-addressing law as a single source of truth — parseFieldAddress + the ruled ten-scalar system maps + plumbing invisibility + refusal builders (module only; query surfaces wire in next) (8f9a9989) -- docs: port the 8.10.3 backport-release changelog entry to main (f6b14d21) -- docs: port the 8.10.2 backport-release changelog entry to main — release branches carry the version bump, main carries the durable record (0b059ac5) -- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (1a09be06) -- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (cb717be2) -- fix(release): double the forge-publish poll budget — the runner executes jobs sequentially and the publish run queues behind the ci matrix (64049631) -- Merge branch 'release/8.11.0' (1865f60a) -- Merge branch 'release/8.10.1' (fc9f0d72) -- chore: the forge is the address — retire the archived mirror from every live surface (415e824a) -- Merge remote-tracking branch 'origin/main' (069a8894) -- Merge branch 'release/8.10.0' (d918c060) -- ci: run the pipeline on the forge (9a5a9ccc) -- feat: two-tier history reads + the repacker + generationDigest — D1+D3 wired end-to-end (1201e255) -- feat: generation-segment store — the D1+D3 packed-tier file format (d8acb377) -- feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) - - -### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) - -- docs: the last two archived-host links point home (91ef1c8b) -- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) -- feat(release): the forge publish leg moves to CI on the tag push; the laptop verifies by readback and keeps the abort-before-storefront guard (3e4a17dc) -- feat: canonical enumeration mode for export — storage-walked, canon-complete, with an index-drift report (4d196af4) -- ci: run the pipeline on the forge (999d0ebb) - - -### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) - -- docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) -- fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) - - -### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - -- docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) -- fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) - - -### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - -- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) -- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) -- fix: transaction timeouts are a typed no-hot-retry contract; engine-side non-retry pinned; dead transaction path removed (003e2a74) -- chore: the forge is the address — retire the archived mirror from every live surface (22702b81) - - -### [8.10.0](https://github.com/soulcraftlabs/brainy/compare/v8.9.0...v8.10.0) (2026-07-23) - -- docs: adoption storefront — contributing guide, security policy, README support + cor section (9a99a7b) -- fix(release): push the public mirror explicitly and verify the tag lands at the right commit before publishing (6ba94c8) -- docs: project guide version line points at npm instead of a hardcoded stale number (3a1efc9) -- feat: vector provider identity is a required name field (hnsw-js), rendered [vector-index:] (3be4ba9) -- feat: warm contract (warm/warmOnOpen/provider warm hook), configurable transact budget floor, backend-neutral vector index op names (55b867c) - - -### [8.9.0](https://github.com/soulcraftlabs/brainy/compare/v8.8.2...v8.9.0) (2026-07-19) - -- docs: measured performance envelopes v1 (per-op p50/p95 at 1k and 10k, pure-JS floor) (5cabd78) -- fix: release drains in-flight writer-lock heartbeat — no phantom lock after unlink (70e4bc8) -- feat: flush() never compacts — history maintenance moves to close() with bounded passes (300d9f2) - - -### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19) - -- fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d) -- chore: push public docs to the soulcraft.com ingest door on release (42037d0) - - -### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18) - -- fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48) -- fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass (4fcef7b) - - -### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17) - -- feat: OS-limit detection for pool-scale deployments (16a73b8) - - -### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17) - -- fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46) - - -### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17) - -- feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb) - - -### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17) - -- feat: brain.auditGraph() — read-only graph-truth audit (2a03fae) - - -### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17) - -- fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064) - - -### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17) - -- fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7) -- docs: external-backups/sparse-storage guide + generation fact log concept (593bb8b) - - -### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15) - -- test: tolerant timing assertion in the execution-time measure test (4dc0a92) -- feat: committedGeneration capability + pinned durability/stability contracts (d1ecee1) -- docs: RELEASES.md entry for 8.5.0 (provider fact-log access + shared verifier) (e4f37cd) -- feat: provider access to the fact log + shared stamp verifier via internals (352e356) - - -### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15) - -- docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43) -- feat: entity-tree family stamp — sourceGeneration + rollup coherence at open (2888ae6) -- feat: generation fact log — after-image commit records, dual-written at every commit point (38b0041) - - ### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15) - docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd) diff --git a/CLAUDE.md b/CLAUDE.md index 56df0b72..568c10db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md` **Brainy's current open actions:** None. MIT open-source — no platform-specific actions. -**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` +**Current version:** `@soulcraft/brainy@7.31.5` (latest published; 8.0.0 release candidate on `feat/8.0-u64-ids`) --- ## Project Overview -Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license. +Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license. ## Getting Started @@ -91,7 +91,7 @@ test: add/update tests (patch version bump) ## Docs Pipeline — soulcraft.com/docs -Docs in `docs/**/*.md` are published with the npm package (included in `files`) and go live on soulcraft.com/docs via the docs ingest API: the release script's `scripts/push-docs.js` step POSTs every public doc to `https://soulcraft.com/api/docs/ingest` (auth: `DOCS_INGEST_SECRET` in the environment). No separate deploy step is involved (the old deploy-to-publish flow was retired in a platform change, 2026-08). Frontmatter controls what appears publicly. +Docs in `docs/**/*.md` are published with the npm package (included in `files`) and synced to soulcraft.com/docs on every portal deploy. Frontmatter controls what appears publicly. ### Docs check triggers @@ -161,9 +161,9 @@ npm run release:major # Breaking changes (rare, manual decision) The script: verifies clean git state, builds, tests, bumps version, updates CHANGELOG.md, commits, tags, pushes, publishes to npm, and creates a GitHub release. After a successful release, remind the user: -> "Published. Docs are live on soulcraft.com/docs (pushed via the ingest API during the release) — spot-check a changed page with curl." +> "Published. Deploy portal to pick up the new docs → go to the portal project and deploy." -There is no separate deploy step anymore. If the docs push failed (the script warns loudly), re-run `node scripts/push-docs.js` with `DOCS_INGEST_SECRET` set. +Do NOT deploy portal from here. Portal is always deployed separately from within the portal project. ## Closed-Source Product Names — HARD RULE diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54d4f784..ab8a0246 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,77 +1,298 @@ # Contributing to Brainy -Brainy is MIT-licensed and genuinely open to outside contributions. This page -is the honest, current path — please don't rely on older instructions you -may find elsewhere in the repo's history. +Thank you for your interest in contributing to Brainy! This document provides guidelines and instructions for contributing to the project. -## Where the project lives +## Code of Conduct -The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. -It's anonymously readable and cloneable — no account needed to browse, clone, -or build. +By participating in this project, you agree to abide by our Code of Conduct: +- Be respectful and inclusive +- Welcome newcomers and help them get started +- Focus on constructive criticism +- Respect differing viewpoints and experiences -## How to contribute +## How to Contribute -**Found a bug, or have an idea?** Email **brainy@soulcraft.com**. No account, -no ceremony — you'll get a receipt, and it goes to a human. +### Reporting Issues -**Want to send a patch?** Two ways, both first-class: +Before creating an issue, please check existing issues to avoid duplicates. -- **Email a patch.** Run `git format-patch` against your change and email the - output to **brainy@soulcraft.com**. This is a genuinely supported path, not - a fallback — plenty of good contributions arrive this way. -- **Open a pull request on the forge.** Request an account at - **source.soulcraft.com** (registration is request-with-approval, so allow - a little lag), clone, push a branch, and open a PR there. Maintainers - review and land it. +When creating an issue, include: +- Clear, descriptive title +- Detailed description of the problem +- Steps to reproduce +- Expected vs actual behavior +- System information (OS, Node version, Brainy version) +- Code examples if applicable -Either way, for anything beyond a small fix, opening an issue first (email is -fine) to talk through the approach saves everyone rework. +### Suggesting Features -## Development setup +Feature requests are welcome! Please provide: +- Clear use case +- Proposed API/interface +- Examples of how it would work +- Any potential challenges or considerations +### Pull Requests + +#### Before Starting + +1. Check existing issues and PRs +2. Open an issue to discuss significant changes +3. Fork the repository +4. Create a feature branch from `main` + +#### Development Setup + +**Quick Setup (Recommended):** ```bash -git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git +# Clone your fork +git clone https://github.com/your-username/brainy.git cd brainy + +# Run setup script (installs all dependencies including Rust) +./scripts/setup-dev.sh +``` + +**Manual Setup:** +```bash +# Clone your fork +git clone https://github.com/your-username/brainy.git +cd brainy + +# Install system dependencies (Ubuntu/Debian) +sudo apt-get install -y build-essential pkg-config libssl-dev + +# Install Rust (for WASM embedding engine) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source ~/.cargo/env +rustup target add wasm32-unknown-unknown +cargo install wasm-pack + +# Install Node.js dependencies npm install + +# Build Candle WASM embedding engine +npm run build:candle + +# Build TypeScript npm run build + +# Run tests npm test ``` -Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite; -see `package.json` for `test:integration`, `test:coverage`, and friends. +#### Making Changes -## Standards +1. **Follow the code style** + - TypeScript for all source code + - Clear variable and function names + - Comments for complex logic + - JSDoc for public APIs -- **Strict TypeScript.** No `any` escape hatches to dodge the type checker. -- **Tests exercise real behavior.** No mocking away the thing you're supposed - to be testing. -- **No stubs, no TODO-code.** If something can't be finished, say so and - leave it out — don't merge a placeholder. -- **JSDoc on every exported function, class, and type.** -- **[Conventional Commits](https://www.conventionalcommits.org/).** `feat:`, - `fix:`, `docs:`, `perf:`, `refactor:`, `test:`, `chore:`. Never - `BREAKING CHANGE` in a commit message — major version bumps are a separate, - deliberate decision. -- **Performance claims are measured or labeled projected.** If a PR or its - description states a number, cite the benchmark that produced it (see - [docs/performance-envelopes.md](docs/performance-envelopes.md) for the - pattern). Don't state an estimate as if it were measured. -- **Measurements carry numbers, not provenance.** Public commit messages and - docs give the SHAPE a number was taken at and never where it was taken: no - hostnames, no store or deployment identities, no operational anecdotes about - someone's running system. "A 14,056-noun / 72,679-verb production-shaped - store, measured solo under an exclusive lock" tells a reader everything the - number depends on; the machine it ran on and whose data it was tell them - nothing except where somebody's infrastructure lives. -- **Documents that answer or reference a confidential specification never enter - this repository, even summarized.** The public docs describe THIS engine and - the published contract, and nothing else — a summary of a private document is - still that document's contents. +2. **Write tests** + - Add tests for new features + - Update tests for changes + - Ensure all tests pass -## License +3. **Update documentation** + - Update README if needed + - Add/update API documentation + - Include examples -Brainy is [MIT licensed](LICENSE). Contributions are accepted under the same -license — there's no CLA to sign. +#### Commit Guidelines -Thank you for considering a contribution. +Follow conventional commits format: + +``` +type(scope): description + +[optional body] + +[optional footer] +``` + +Types: +- `feat`: New feature +- `fix`: Bug fix +- `docs`: Documentation changes +- `style`: Code style changes +- `refactor`: Code refactoring +- `perf`: Performance improvements +- `test`: Test changes +- `chore`: Build/tooling changes + +Examples: +```bash +feat(triple): add graph traversal depth limit +fix(storage): handle concurrent write conflicts +docs(api): update search method documentation +``` + +#### Submitting PR + +1. Push to your fork +2. Create PR against `main` branch +3. Fill out PR template +4. Ensure CI checks pass +5. Wait for review + +### Testing + +#### Running Tests + +```bash +# Run all tests +npm test + +# Run specific test file +npm test tests/core.test.ts + +# Run with coverage +npm run test:coverage + +# Watch mode +npm run test:watch +``` + +#### Writing Tests + +```typescript +import { describe, it, expect } from 'vitest' +import { Brainy } from '../src' + +describe('Feature Name', () => { + it('should do something specific', async () => { + const brain = new Brainy() + await brain.init() + + // Test implementation + const result = await brain.search("test") + + expect(result).toBeDefined() + expect(result.length).toBeGreaterThan(0) + }) +}) +``` + +## Architecture Guidelines + +### Adding New Features + +1. **Check existing functionality** + - Review `ARCHITECTURE.md` + - Check if similar features exist + - Consider if it should be an augmentation + +2. **Design considerations** + - Maintain backward compatibility + - Consider performance impact + - Think about all storage adapters + - Plan for extensibility + +3. **Implementation checklist** + - [ ] Core functionality + - [ ] Tests (unit and integration) + - [ ] Documentation + - [ ] TypeScript types + - [ ] Examples + - [ ] Performance benchmarks (if applicable) + +### Creating Augmentations + +Augmentations extend Brainy's functionality: + +```typescript +import { BrainyAugmentation } from '../types' + +export class MyAugmentation extends BrainyAugmentation { + name = 'MyAugmentation' + + async onInit(brain: Brainy): Promise { + // Initialize augmentation + } + + async onAdd(item: any, brain: Brainy): Promise { + // Process before adding + return item + } + + async onSearch(query: any, results: any[], brain: Brainy): Promise { + // Process search results + return results + } +} +``` + +### Performance Considerations + +- Use batch operations where possible +- Implement caching strategically +- Consider memory usage +- Profile performance impacts +- Add benchmarks for critical paths + +## Documentation + +### API Documentation + +Use JSDoc for all public APIs: + +```typescript +/** + * Searches for similar items using vector similarity + * @param query - Search query (text or vector) + * @param options - Search options + * @returns Array of search results with scores + * @example + * ```typescript + * const results = await brain.search("machine learning", { limit: 10 }) + * ``` + */ +async search(query: string | Vector, options?: SearchOptions): Promise { + // Implementation +} +``` + +### Examples + +Add examples for new features: + +```typescript +// examples/feature-name.ts +import { Brainy } from 'brainy' + +async function exampleUsage() { + const brain = new Brainy() + await brain.init() + + // Show feature usage + // Include comments explaining what's happening + // Handle errors appropriately +} + +exampleUsage().catch(console.error) +``` + +## Release Process + +1. **Version bump**: Follow semantic versioning +2. **Update CHANGELOG**: Document all changes +3. **Run tests**: Ensure all tests pass +4. **Build**: Generate distribution files +5. **Tag**: Create git tag for version +6. **Publish**: Release to npm + +## Getting Help + +- **Discord**: Join our community +- **Issues**: Ask questions on GitHub +- **Discussions**: Share ideas and get feedback + +## Recognition + +Contributors will be recognized in: +- CHANGELOG.md for their contributions +- README.md contributors section +- GitHub contributors page + +Thank you for contributing to Brainy! 🧠 \ No newline at end of file diff --git a/README.md b/README.md index 762c9ec3..d1342f52 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- Brainy + Brainy

Brainy

@@ -11,9 +11,9 @@

- Package on The Source - Repository - CI + npm version + npm downloads + CI Documentation MIT License TypeScript @@ -23,15 +23,12 @@ Quick start · One query · Features · - Scale with Cor · - Docs · - Support + Scale with Cor · + Docs

--- -**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license. - Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: | You write | Brainy indexes it as | You query it with | @@ -47,14 +44,12 @@ It runs **inside your process** — no server, no Docker, nothing to operate — ## Quick start ```bash -bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended -npm install @soulcraftlabs/brainy # Node.js ≥ 22 +bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended +npm install @soulcraft/brainy # Node.js ≥ 22 ``` -> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read). - ```javascript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() // in-memory; one line swaps to disk await brain.init() @@ -177,11 +172,9 @@ await brain.vfs.search('React components with hooks') // semantic file **[Multi-process model](docs/concepts/multi-process.md)** · **[Inspection guide](docs/guides/inspection.md)** -## When you outgrow Brainy +## From laptop to hundreds of millions -Brainy's pure-TypeScript engines carry real workloads a long way on their own — see the measured, per-operation numbers (not marketing figures) in **[docs/performance-envelopes.md](docs/performance-envelopes.md)** for what to expect, unaccelerated, on plain filesystem storage. - -When a deployment needs native-scale vector/graph performance — memory-mapped indexes that don't need your dataset in RAM, billion-scale ambitions — add the native engine. **The API doesn't change:** +Brainy's TypeScript engines take you a long way. When you outgrow them, add the native engine — **the API doesn't change**: ```bash npm install @soulcraft/cor @@ -194,14 +187,13 @@ await brain.init() // @soulcraft/cor detected — same code, native engines un Installing the package is the opt-in: if `@soulcraft/cor` is present, it loads and announces itself in the init log; if it's present but broken, `init()` **throws** — an installed accelerator never silently vanishes behind the JS engines. Opt out with `plugins: []`, or pin exactly what loads with `plugins: ['@soulcraft/cor']`. [`@soulcraft/cor`](https://www.npmjs.com/package/@soulcraft/cor) (Brainy 8.x ↔ Cor 3.x, version-matched) registers Rust implementations behind every provider seam: SIMD distance kernels, memory-mapped storage, a disk-native vector index that doesn't need your dataset in RAM, durable LSM field/graph indexes that serve cold opens instantly, and native aggregation. Recall@10 measured **0.99 / 0.96 / 0.96 at 1M / 10M / 100M vectors** in Cor's release gate. -Open core, commercial accelerator: Brainy is MIT and complete on its own — Cor is more headroom for when you need it, not capability held back to sell you later. Licensing and support: **cor@soulcraft.com**. +Open core, commercial accelerator: Brainy is MIT and complete on its own; Cor is licensed and funds both. ## Performance -- Per-operation p50/p95 at 1k and 10k entities, pure-JS floor, measured and re-run every release that touches a measured path: **[docs/performance-envelopes.md](docs/performance-envelopes.md)**. - JS distance kernels: **~6× faster cosine, ~1.4× euclidean** than 7.x (measured: [`tests/benchmarks/distance-microbench.mjs`](tests/benchmarks/distance-microbench.mjs), 384-dim, median of 41). - Whole-graph reads are single **O(N + E)** cursor walks — a consumer-measured 19k-edge export dropped from ~27 s of per-node calls to one scan. -- Capacity planning and architecture: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)** +- Full numbers and capacity planning: **[docs/PERFORMANCE.md](docs/PERFORMANCE.md)** · **[docs/SCALING.md](docs/SCALING.md)** ## Use cases @@ -220,10 +212,6 @@ Open core, commercial accelerator: Brainy is MIT and complete on its own — Cor **Bun ≥ 1.1** (recommended) or **Node.js ≥ 22**. Brainy 8.x is server-only; the 7.x line remains on npm for browser use. -## Support & community +## Contributing & license -- **Bugs and ideas** → **brainy@soulcraft.com** — no account needed, you'll get a receipt. -- **Security reports** → **security@soulcraft.com** — see **[SECURITY.md](SECURITY.md)**. -- **Contributing** → see **[CONTRIBUTING.md](CONTRIBUTING.md)**. - -MIT © Brainy Contributors. +Contributions welcome — see **[CONTRIBUTING.md](CONTRIBUTING.md)**. MIT © Brainy Contributors. diff --git a/RELEASES.md b/RELEASES.md index e8833b80..69159560 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,1179 +1,15 @@ # @soulcraft/brainy — Release Notes for Consumers This file is the **quick reference for downstream sessions** tracking Brainy changes. -Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases +Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/soulcraftlabs/brainy/releases **How to use:** Brainy is the underlying data engine for downstream applications. Read this when: - Upgrading `@soulcraft/brainy` in your application - Debugging data, query, or storage behaviour - A new Brainy feature is available that you want to adopt -## Removed APIs — 7.x → 8.x (the complete ledger) - -Every public API removed at the 8.0 major, with its sanctioned replacement. If your code -still calls a left-column name on 8.x it throws (or the config key is rejected) — the -replacement is always a one-line change. (Standing contract from 8.9.0 forward: removals -happen only at majors, after ≥1 minor of loud runtime deprecation naming the replacement.) - -| Removed (7.x) | Replacement (8.x) | -|---|---| -| `brain.search(query, k)` | `find({ query })` — semantic; `find({ query, searchMode })` for hybrid | -| `brain.getRelations({...})` | `related(id, opts)` for adjacency; `find({ connected: {...} })` for scoped traversal | -| `brain.neural()` clustering | `find({ vector })` + aggregation `GROUP BY` | -| `Db.search()` | `db.find({ vector })` | -| Pre-8.0 storage path aliases (`directory`, `basePath`, …) | one `storage.path` key (old aliases throw) | -| Reserved keys inside `metadata` bags (silently remapped in 7.x) | top-level params (`subtype`, `visibility`, `confidence`, `weight`, …) — reserved-in-bag throws | -| 7.x COW branches layout (`branches/main/`) | generational MVCC (`asOf()`, `now()`, `db.persist(path)`) — on-disk migration is automatic at first 8.x open | - -The fork/snapshot family (`brain.snapshot()`, `createSnapshot()`, `restoreSnapshot()`) -is sometimes cited as a 7.x removal — those methods never existed on 7.x; the 8.0 Db API -(`asOf`/`persist`/`restore({confirm})`) is their first real implementation. - --- -## v10.4.4 — 2026-08-28 - -**A correctness and observability release.** The headline is not speed: it is that a -restart now tells you the truth about itself, a store stops lying about how much it -holds, and the engine stops doing work nobody asked for. There is a performance -improvement and it is modest; it is stated exactly below rather than rounded up. - -### The dark restart — fixed at the root - -A service could stop cleanly, exit 0, having awaited `close()` on every store it held, -and its next boot would announce `Overwriting stale writer lock … appears dead` for -every one of them. Nothing had crashed. Two deployments hit this; the same defect also -made those boots pay a crash-recovery fold they did not owe. - -The cause was not the lock. `close()` released it correctly — when it got there. A -failure part-way through close skipped both the release AND the clean-shutdown marker, -and "the recorded pid is gone" reads identically for an orderly restart and a crash. - -- `close()` is now two parts and the second is unconditional: the flush-request watcher, - the **writer lock**, the VFS timers and the terminal `closed` flag are released whether - the durable steps succeeded or not. The original failure is narrated with what it costs - the next open, then rethrown. -- Releasing the lock writes a **clean-close record** naming the lock generation it gave - up. The next open reads that record instead of guessing: recorded → nothing to recover; - absent → it says so, and names the recovery it is about to run. This also ends two - long-standing false alarms — a recycled pid locking a store out of its own reopen, and - `Re-acquiring writer lock … this is a bug` after a perfectly clean close. -- The signal path stopped failing in a batch. One store's failing flush used to strand - every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the - generation store's close (the marker) is part of shutdown, the lock goes in a `finally`, - and the handler no longer calls `process.exit()` when the host application has its own - signal handler, a race that truncated the host's own shutdown mid-flight. - -### The count ledger stops lying, and `counts.json` is written atomically - -The all-tier scalars are the denominator a coverage check subtracts against. A ledger -derived under the old rule — one entity per id DIRECTORY — counted ghost and scar -containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for -the life of the store. Two copies of one archive could disagree, and a downstream index -heal reported remaining work that did not exist. - -- Such a ledger now derives itself honestly **in the background** after the open, counting - identity records, and persists the correction stamped. Nothing waits for it, because no - read is served from a denominator. -- A derivation that raced a write refuses to stamp its number: one retry on a quiet store, - then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under - a barrier. -- `counts.json` is written temp+rename. A truncating write left a window in which a - concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open - down the full-rescan path, so the cheapest file in the store was buying the most - expensive recovery. - -### An open and a repair narrate themselves — on a channel a log level cannot silence - -A store could open for three minutes and print nothing at all. The phase timings existed; -they were written to a channel that every production-looking environment clamps away. - -- Narration moved to an always-visible channel. An open now heartbeats the phase it is in, - names each phase as it ends with what it was paying for, and names the expensive STEP - inside a phase. `repairIndex()` does the same and its receipt carries a per-family - `durationMs` — a repair that ran for half an hour with no output could only be watched - through `top`. -- A brain nobody has written to now does nothing: a flush over a clean store is a no-op - and says nothing, the graph index's auto-flush asks before it acts, and the - cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a - directory every 500 ms per store forever, with a slow safety sweep behind it and a - narrated fall back to polling where a filesystem cannot be watched. -- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()` - does not wait for it, every other family serves, and that family's doors refuse **by - name, carrying the provider's own progress**, saying plainly that they open by - themselves and no action is needed. Health narration dedupes by content, so an unchanged - verdict is silent however a provider's generation counter moves. - -### For operators — one behaviour change - -**Four `where` operators that previously returned an empty page now raise -`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range -posting index cannot evaluate a substring, a pattern or an array length without reading -every row, and it now refuses by name instead of answering with an empty result that -looks like an answer. - -**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and -`excludes`. All 25 accepted operator tokens now agree between this engine and its -accelerated counterpart. - -### Performance — stated exactly - -Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under -an exclusive lock: - -- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is - one fix — generation discovery reads directory NAMES instead of recursively walking the - entire generation log (−9.2 s, and it scales with history rather than row count). The - VFS phase is **unchanged**. -- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving - off the critical path accounts for storage-init dropping 5,941 ms → 25 ms. -- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own - init is under 2 s of that phase. It is the log-authority adoption and/or the - pending-embed log recovery, both now instrumented so the next measurement names the - culprit outright. - -Continuing work, named so nobody has to rediscover it: that ~38 s term; making the -generation store's committed-range set lazy; the hydration path that substitutes -`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix -filter built with a `$startsWith` spelling no operator set accepts, so -`searchFiles({ path })` throws today. - ---- - -## v10.4.3 — 2026-08-27 (Open Brainy's first release) - -**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for -byte — only the name, the registry, and the pointers changed.** Install: - -```bash -npm install @soulcraftlabs/brainy -``` - -with the registry line in your `.npmrc` (anonymous read): - -``` -@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/ -``` - -- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the - npmjs republish step is retired from the release rail. Existing npmjs versions of - `@soulcraft/brainy` stay as they are and receive no new versions. -- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects. -- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one - install-line change (`@soulcraft/brainy` → `@soulcraftlabs/brainy`), which downstream - applications make together with their native-engine bump. - -## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector) - -**This is the last release of the MIT engine under the `@soulcraft/brainy` name.** -The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API, -client library, types and protocol, an openly specified canonical format, and the TypeScript -reference engine, scoped honestly as a single-node engine for stores up to roughly one -million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a -major version bump; that engine implements the same API over the same open format at -production scale, requires a license, and refuses loudly without one. Nothing changes -for existing installs until that major ships; the move is announced with it. - -Six fixes, one law: a vector with no magnitude carries no information, so it must -never reach a vector index — in any engine — and the canonical store must say so. - -- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item - shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted - with an empty vector leg, never embedded, never indexed, counted as unvectored in - the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing — - no longer need a placeholder vector and never enter the vector leg. `vector: []` - together with `deferEmbedding: true` is refused with a typed error (a supplied - vector has nothing to defer). Previously `vector: []` threw a dimension error. -- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is - the sanctioned way to strip a vector from an existing row: canonical vector → `[]`, - removal from the vector index, the vectored ledger decremented exactly once — and - idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and - it clears a pending deferred-embed marker durably so the background worker cannot - re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the - index from canonical rows); shedding historical vectors needs this door. -- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on - any write path is persisted as unvectored (`[]`) with one warning naming the row; - the vector-index operations keep their own refusal as a second line. The engine's - own VFS root, which used to persist a deliberate all-zero placeholder (harmless - under cosine distance, a false attractor under a downstream engine's - squared-euclidean serving — a production incident this week), is now created - unvectored, and an existing store's legacy root is migrated on open by a single - fixed-path read before the health gate runs — never a walk. -- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the - cursor walks behind them enumerate by the metadata record, the same key the - canonical ledger counts by — previously the walk keyed on the vector file, so a - row holding metadata but no vector was counted yet never yielded (a permanent - "missing" phantom in coverage math), while an orphaned vector-only directory - could be yielded as a phantom id. The recovery fold also never deletes an existing - vector when it replays a metadata-only after-image (preserve-if-absent). One - documented gap remains: a verb's endpoints live only in its vector leg, so a - metadata-only verb is counted and loudly skipped, never fabricated — the fix is a - canonical-format change and lands with the open format. -- **The ledger's one-time derivation counts identity records.** Stores upgraded from - pre-ledger versions derived their ALL-visibility scalars once by counting id - directories, which included ghost and scar containers left by an old partial-delete - defect — an inflated denominator whose coverage row could never reach exact. The - derivation now counts only directories holding a metadata record, `counts.json` - carries a derivation-rule stamp, and a ledger derived under the old rule is marked - `suspect` at open (one O(1) field read, one warning) so the online `repairIndex()` - path clears it with a real recount. -- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and - zero-norm rows (one summary line), re-pins the vector dimension from the first real - vector after a restart (previously a restart left the pin unset, so a wrong-length - insert became the new pin instead of being rejected), and `addItem` / `updateItem` - throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing - a vector-less node. -- **Smaller:** a failing plugin activation now rethrows with the original error as - `cause` (the originating file and line survive to the caller's log); build - generators stamp from the repository history of their inputs instead of wall clock, - so two builds of the same tree are byte-identical. - -Adoption: one restart, paired with its native-engine release. The first open of an -existing store runs the legacy-root migration (one narrated line) and, on stores that -upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned -recount — no rebuild in either case. - -## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds) - -Two production defects from the same week, fixed together as a patch to 10.4.0. - -- **The read gate is per family.** A read now refuses only when the index family it - actually consults is unhealthy: a metadata filter is served while the vector leg is - rebuilding; a semantic query is refused only by the vector family; a graph - traversal only by the graph family. Previously any unhealthy family refused every - read on the brain — under a long vector rebuild, a production deployment's - metadata-only reads were refused for the duration, and the retries became a write - pump of their own. -- **Unchanged data never re-embeds.** `update()` compares the incoming `data` - structurally with the stored record; an update carrying identical data (a common - shape for periodic upserts) no longer embeds again and no longer churns the vector - leg. Previously every such update re-embedded and re-inserted, which under load - saturated the vector index with near-identical vectors. - -Adoption: one restart, paired with its native-engine release. - -## v10.4.0 — 2026-08-25 (the health report has a name) - -Three related cures, one root cause: an index deciding whether it could be trusted -by sampling itself instead of by exact accounting. This release replaces every -sampled self-probe with ledger-derived truth, and a read against an unhealthy index -now refuses loudly instead of guessing. - -- **The canonical count ledger.** Storage now tracks two scalars per family - (nouns/verbs) on the write path: the user-facing `counted` total — unchanged, - still what `getNounCount()` / `getVerbCount()` return — and a new ALL-visibility - `all` total covering every tier, the real denominator a derived index's own - coverage math needs. The unfiltered storage-level `totalCount` returned by - `getNouns()` / `getVerbs()` is now this unclamped ALL scalar; previously it could - only ever move up (`Math.max(scalar, scanned)`), so an inflated counter could - never self-correct. A delete that cannot prove the record it removed actually - existed (no canonical read, no prior image available) no longer decrements on - faith — it marks the ledger `suspect` (narrated once per session) instead of - silently drifting, and the next `repairIndex()` clears the flag with a real - recount. -- **One contract for a throwing health probe.** A provider's `validateInvariants()` - is documented to never throw — but if one does anyway (a bug, a transient fault), - it is now read the same way everywhere: `heal: 'none'`, the error named in the - report, never synthesized into a rebuild trigger and never swallowed into "looks - fine." A flaky check can no longer buy itself a rebuild. `repairIndex()`'s - per-family receipt also gains `missing` (an exact count plus a capped id sample), - `rebuilt` (a full rebuild ran, vs. an incremental heal), and `reason`. -- **The named health report; reads refuse instead of rebuilding.** Any index - provider may now expose a synchronous, O(1) `healthReport()` — composed from the - provider's own exact ledgers, never a sample — and this is the one signal - Brainy's read gate trusts. The first-query lazy-build path is gone: `brain.init()` - now runs every needed rebuild to completion before it returns, always, regardless - of dataset size. A read that lands on a provider whose health report says it - isn't serving throws a typed error instead of triggering a rebuild mid-query — - `GraphIndexNotReadyError`, `MetadataIndexNotReadyError`, or - `VectorIndexNotReadyError` (all exported from `@soulcraft/brainy`), naming the - reasons. `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] | 'all' })` is - the new explicit operator door: it rebuilds the named family unconditionally, no - health check consulted — reach for it when you have independent reason to - distrust a family regardless of what it self-reports. Bare `repairIndex()` is - unchanged in spirit: report-driven, heals only what its own checks say needs it. -- New concept doc: [Index Health](docs/concepts/index-health.md) walks the whole - story from a consumer's side — degraded-but-serving vs. not-ready, what - `repairIndex()` checks and heals per family, what `suspect` counts mean. - -**Nothing to change to adopt this.** No API removed, no signature narrowed — -`repairIndex()` gains an optional options bag and its return value gains fields, -both additive. The honest notes: if your code ever relied on a `find()` against a -cold/not-yet-built index quietly triggering a rebuild and returning results a beat -later, that behavior is gone — it now throws one of the three typed -`*NotReadyError` classes instead (catch them if you need to distinguish "not ready -yet" from "no results"). And `disableAutoRebuild: true` no longer defers index -construction to the first query — a needed rebuild always runs at `open()` now; -the flag has no effect on timing. Full manual control still lives in -`repairIndex({ rebuild: [...] })`. - -- **Crash-reopen catchup.** After an unclean shutdown, the metadata index now - folds the exact fact window it missed — `find()` serves every acked write on - reopen, closing the gap where canonical reads and counts recovered a - crash-window write but the index kept serving its pre-crash state until the - next full rebuild. Related root-cause fixed alongside: `close()` never - stamped the index watermarks (only `flush()` did), so a close without a - prior flush caused a needless full rescan verdict on the next open. -- **Relation rows are live in the metadata index.** Previously verb rows - entered the metadata index only during a rebuild — so a rebuilt store's - relation postings went stale from the first `relate()` after it. Relations - are now posted and retracted on the live write path (relate / unrelate / - updateRelation / remove's cascade, and their `transact()` forms), in the - same commit as the graph leg. -- **The metadata rebuild is online.** `rebuild()` for the metadata family no - longer clears and rebuilds in place (reads went empty for the duration): it - builds a complete replacement beside the serving index, mirrors concurrent - writes to both, swaps atomically, and persists once after the swap. Reads - never observe a partial index. `repairIndex({ rebuild: ['metadata'] })` uses - it automatically. -- **Incremental heal is routed.** A provider invariant that asks for the - incremental heal (`heal: 'repair'`) now routes to the provider's own - `repair()` when it exposes one — re-posting exactly what its ledger names, - never a store-sized rebuild — and the post-heal re-read of the report decides - success; a repair that doesn't converge is recorded with the escalation named. -- **The vector family joins the count ledger.** `getCanonicalCounts()` gains - `vectors: { all }` — the count of canonical entities holding a real vector - (deferred-embed entities count when their vector lands). And the open gate - closes the vector leg: a store whose canonical rows hold vectors but whose - derived vector index is empty now builds at `open()` (or refuses with the - typed error) instead of silently serving empty vector-search results. -- **An unknown storage config shape fails loudly.** A nested `config` object - carrying a path-shaped key (a shape that was never supported) used to fall - through silently to the default shared directory — every instance writing one - store while callers believed each had its own. It now throws, naming the - canonical `path` key. -- **Relation index rows are JSON-safe.** Internal endpoint identifiers can no - longer ride the metadata-index crossing (a native provider serializes it); - they stay on the graph operations where they belong. -- **A broken accelerator install can never read as "not installed."** The - auto-detection free pass now requires the resolution error to name the - accelerator package itself, exactly — a missing platform-binary sibling - package, an inner file path, or a dependency failure is a broken install and - `init()` throws loudly. And a plugin that declines activation is narrated on - the always-on log channel, so `silent: true` can no longer hide a fallback - to the default engines. - ---- - -## v10.3.1 — 2026-08-18 (the fold that behaves) - -Three recovery cures from one production first-boot incident (a brain's first -process restart after a live storage-authority flip looked hung and was -restarted three times mid-recovery). **Adopt this version before flipping -brains with existing history** — it is the intended adoption target for -fleets moving to the crash-safe authority. - -- **Recovery streams.** The boot-time log fold now consumes the generation - log one segment-batch at a time — memory stays bounded at one segment for - any log size. Previously it materialized every fact into one array, which - on a ~7k-fact log produced multi-GB allocation pressure and a process that - looked wedged while it worked. -- **Recovery narrates.** The fold announces itself before the work begins - ("recovery fold beginning — do not restart, the fold is finite") and prints - progress every thousand facts. A visible fold gets to finish; a silent one - gets killed by a well-meaning operator, and each kill makes the next boot - pay the whole fold again. -- **Bounded recovery from the flip itself.** Adopting the log authority now - founds the recovery checkpoint at the moment of the flip (one paged - canonical sync, bounded memory, then the stamp) — so even the FIRST unclean - shutdown after a flip replays only the log's tail. Previously the bound - could only establish itself at a completed crash recovery, which is exactly - the recovery the incident kept interrupting. - ---- - -## v10.3.0 — 2026-08-18 (the trust-and-provenance release) - -Four consumer-driven cures. Pairs with the same native accelerator line -(>=4.1.0); adopt alongside the accelerator's 4.2.0 for its paired fixes. - -- **Writer-lock fencing.** A live writer is never auto-evicted (staleness now - requires the holding process to be dead — a >60s stall is a slow writer, not - a dead one); the lock claim is atomic (no empty-file window a racer can - misread as torn); and every flush commit and transact barrier verifies lock - ownership first, so a forced-out or lock-deleted writer fails typed - (`BRAINY_WRITER_FENCED`) instead of writing on unaware — the split-brain - class a shared dev store hit is dead at all three roots. The documented - same-process re-open ("warn and take over") stays benign: ownership is - per-process. Consumers that raised stop-timeouts as mitigation can retire - them. -- **Transaction-log provenance.** `TxLogEntry` gains an optional `origin` - field — absent means a user write (existing consumers unchanged); - engine-originated commits stamp themselves (`system:embed-landing`, - `system:adoption-backfill`, `system:reconcile`), and the same stamp rides - the commit fact's meta. Activity feeds filter on fact instead of guessing; - a reported "double tick" (the deferred vector landing indistinguishable from - a user save) is cured without collapsing genuine rapid saves. -- **The attested reconcile door.** `reconcileLogDivergence(id, {attest})` - resolves the one adoption-refusing divergence class - (`log-live-canonical-absent`) with a human's word: `'deleted'` mints the - tombstone the log always lacked; `'restore'` folds the log's only copy back - into canonical; wrong-class calls refuse typed with nothing written. Loud, - narrated, single-row. -- **Iron-honest test budgets.** The wall-clock micro-budgets are recalibrated - as order-of-magnitude guards (3x the worst measurement across three machine - classes) so honest hardware differences can never again read as failures; - real performance enforcement lives in the dedicated perf lanes. - ---- - -## v10.2.0 — 2026-08-17 (adoption completes in one call) - -One fix, headline-sized for large stores. Pairs with the same native accelerator -version as 10.1.0 — no accelerator bump needed. - -- **The adoption backfill runs to completion.** Adopting the crash-safe storage - authority first re-commits every row the log never saw (a one-time baseline - backfill). That backfill had a fixed ceiling of 800 rows per - `adoptLogAuthority()` call — sized for small drift, not for a large pre-existing - store — so a store with a 12,700-row baseline advanced 800 rows per call and - stayed on the prior authority across restarts (a production deployment's - report). Now one call adopts a baseline of any size: the backfill sees the - entire curable set at once, cures all of it, and loops only until green — the - no-progress guard is the sole stop. Pace rides the write path (~100 rows/s - measured end to end, versus ~1.7 rows/s under the old page-per-scan shape), - and progress is narrated so an operator watching a live service sees motion. - Stores that already adopted are unaffected; stores still on the prior authority - flip in a single call on their next open or on an explicit - `adoptLogAuthority()`. -- Verification report unchanged on the wire (still lists at most 200 mismatches; - counts remain complete) — only the adoption path reads the full set. - ---- - -## v10.1.0 — 2026-08-13 (the bounded-recovery and write-path-cure release) - -The theme: **crash recovery is bounded, restores are durably founded, and two -production-reported write-path defects are cured at their roots.** Ships together -with the matching native accelerator version; adopt as a pair. - -- **Bounded crash recovery (the fold-checkpoint bound).** Recovery after an unclean - shutdown now replays only the log segment above a durably-stamped checkpoint - instead of the whole log. The checkpoint advances only after a canonical-sync - barrier makes every touched record durable (deletes included), so the bound can - lag but can never overstate durability. Existing stores converge automatically at - their first recovery — zero operator steps; recovery cost stops scaling with - store age. -- **Restores are unclean events, by construction.** `restore()` now runs its swap - fully quiesced (no background flush can race the directory replacement — a - consumer-reported `ENOTEMPTY` crash class is dead), and a snapshot's durability - stamps never survive the restore: the reopen folds the restored log, re-syncs - what it re-applied, and stamps fresh. Restored state is durably founded at - restore time instead of inheriting assertions about bytes the disk never synced. -- **Write-path cures from a production report.** (1) Log pad-frame construction is - total — a size-class boundary hole could previously kill a sync with "pad frame - not constructible". (2) The at-ack sync-failure compensation now splits by phase: - the generation counter can never re-mint a number the log may already carry, so - the non-monotonic append refusal loop reported by a downstream deployment cannot - recur. Both pinned with the reporter's exact shapes. -- **Operator-truthful sparse queries.** `where` on a field no store row has ever - carried now serves the honest answer (`eq`/`in`/range → empty; `ne`/`exists:false` - → all rows; `exists:true` → empty) with a throttled did-you-mean warning, instead - of refusing. `orderBy` on unknown fields and ambiguous spellings keep their typed - refusals. -- **Cross-package error identity.** `UnresolvableFieldError` thrown across package - boundaries is re-normalized so `instanceof` checks in consuming applications - match regardless of duplicated dependency trees. -- Release tooling: publishes now push the tag before the branch (the publish - workflow can no longer queue behind a redundant CI run) and verify registry - byte-identity with a propagation-tolerant raw-registry probe. - ---- - -## v10.0.0 — 2026-08-10 (the write-path and lifecycle release) - -The theme: **writes ack fast and honestly, startup adopts instead of rebuilding, and -every query path serves, announces, or refuses — never silently degrades.** Ships as -one release together with the matching native accelerator version. - -**The storage-authority posture (the release's headline):** a NEW brain's default -is **durable-at-ack log authority** — the generation log is the source of truth, -every write acknowledgment is covered by a group-committed fsync, and crash -recovery is a replay of the log (an acked write survives power loss, proven by -fault-injection tests). An EXISTING brain adopts at its first open under 10.0.0, -gated by a verification oracle: the log is replayed and diffed against stored -truth record-by-record; curable gaps are backfilled; the brain flips only on a -green verdict and a brain that cannot verify stays on the previous posture and -says so loudly. The explicit opt-out is `logAuthority: 'defer'` in the config -(no automatic adoption; flip later with `adoptLogAuthority()`). - -**Why a major:** the generation log gains write format v2 — new segments carry typed, -versioned records with integrity seals. A 9.x build refuses a v2 segment with a clear -version-naming error (never a misread), which means **a brain written by 10.x cannot -be opened by 9.x**. Existing v1 history stays readable forever; upgrading requires no -migration and no data touch — the format moves forward only as you write. - -### New capabilities - -- **`deferEmbedding: true`** on `add()`/`update()`: the write acks at durability; the - embedding runs on a crash-safe background worker and the vector swaps in atomically. - The row is id/metadata-findable immediately; semantic recall converges when the embed - lands. Barriers and gauges: `awaitPendingEmbeds()`, `waitForIndexed('semantic')`, - `getIndexStatus().pendingEmbeds`. VFS file writes adopt this end to end — file-write - ack no longer waits on a neural net (measured ~50× faster serial writes on a - production-shaped corpus). -- **`waitForIndexed(path?, { generation?, timeoutMs? })`** — the one honest read - barrier for write-then-recall flows. Typed timeout error naming what was still - pending; never a silent partial wait. -- **Engine-owned persistence cadence** (`persistence.policy: 'auto'`, now the default): - the engine flushes on write-count/interval/idle triggers in the background, - single-flight. **Delete `flush()` calls from hot paths** — `flush()` remains as an - awaitable durability barrier. A hung flush can never block a write ack. -- **Time-travel recall contract**: `asOf(G).find()` serves vectors exactly as they - stood at G — a later update never leaks into an earlier pin; deleted rows mask; - beyond-head pins refuse typed. -- **Log-authority storage (opt-in, per brain)**: `verifyLogAuthority()` audits the - generation log against stored truth record-by-record and names every divergence; - `adoptLogAuthority()` flips a brain to log-authoritative storage only on a green - audit (self-healing curable divergences first), enabling durable-at-ack writes: - concurrent writers share one fsync and an acked write survives power loss, by - construction (crash-recovery replay is pinned by fault-injection tests). - -### Behaviour changes - -- **`find({ where: {} })` now serves match-all** (previously returned an empty result - silently — warm and cold). Same fix applies to count, streaming, and graph-scoped - seeding paths. -- **`removeMany({ where: {} })` now refuses with a typed error** — a match-all bulk - delete must be explicit, never inherited from an empty filter object. -- **Aggregations always answer**: state persists at every `flush()` (not only close), - an unclean exit reconciles incrementally instead of rescanning the store, and - deletes without a before-image flag a loud rescan instead of silently skipping. -- **Vector updates are atomic in place** — a row is never transiently absent from - search during an update (the "flicker" class is gone); type-only re-index of an - unchanged vector is a no-op. - -### Format note - -- The generation log gains **format v2** (typed, versioned records with integrity - seals). v1 segments remain readable forever; new segments write v2. Older brainy - builds refuse v2 segments with a clear version-naming error rather than misreading - them. Records reserve encryption fields for a future release — zero behaviour today. - -## v8.11.0 — 2026-07-27 (canonical enumeration mode for export — storage-walked, canon-complete) - -From a fleet data-migration program's requirement for whole-brain exports that are -provably canon-complete: `export()`'s default enumeration for a whole-brain/predicate -selector is a generation-correct paginated `find()` walk — a projection query riding -the metadata index as an acceleration structure. Production has documented both of the -index's failure classes: a lost/stale posting can silently OMIT a canonical record from -an export, and a stale posting can silently INCLUDE a phantom row. Neither is visible -to the caller today. - -- **New: `export(selector, { enumeration: 'canonical' })`** (default remains `'index'` — - unchanged behavior on this release). Canonical mode walks every live noun/verb - directly off the storage adapter's canonical shard layout (`storage.getNouns()` / - `getVerbs()` — the same primitive `repairIndex()`'s recount and every index-heal - walk use) instead of the metadata/graph indexes, then applies the selector as a - plain predicate over the walked records. This guarantees canon-completeness — index - corruption cannot hide a live record from the export — at the cost of an O(N) walk - regardless of selector selectivity. Relations are also walked canonically in this - mode, for every selector, not just the whole-brain case. Requires the LIVE current - generation: called on a historical `asOf()` view or a speculative `with()` overlay it - throws `CanonicalEnumerationUnavailableError` rather than silently mixing generations - or missing an overlay's own entities — `enumeration: 'index'` (the default) is - unaffected and still composes with `asOf()`/`with()` as before. -- **New: `export(selector, { enumeration: 'canonical', reportIndexDrift: true })`** — - also runs the index-based enumeration and diffs it against canonical ground truth, - attaching `PortableGraph.drift: { canonicalOnly: string[], indexOnly: string[] }` - (canon-present ids the index missed; index-visible ids canon-absent — phantoms). - Migration-audit evidence, not a repair: nonzero drift is reported loudly - (`console.warn` with the counts) and nothing is auto-healed — run `brain.repairIndex()` - to reconcile the metadata index once drift is confirmed. -- **New: `export(selector, { includeHidden: true })`** (default: false — unchanged - behavior). Without it, a whole-brain/predicate export could never carry a - `visibility:'internal'` or `'system'` row, in EITHER `enumeration` mode — a real gap - for a bulk-migration fold auditing per-visibility-tier, where a hidden tier is real - user data, not noise to drop. `includeHidden` admits both tiers into candidacy in - both modes (and implies `includeSystem`; `includeSystem` alone keeps its narrower, - pre-existing meaning). **Migration-grade exports set `includeHidden: true`** — a - complete-canon export must carry every visibility tier; consumer-facing exports - leave it off. -- **Ops note (consumer-invisible): the release pipeline's home-registry publish (The - Source, source.soulcraft.com) now runs on CI**, triggered by the release tag, instead of PUTting the tarball from the laptop - over WAN — no change to what gets published or how a consumer installs it. - -## v9.0.0 — 2026-08-04 (the field-addressing law: your names and system.*, nothing in between) - -**Major.** One law now governs every field name, on every surface: - -> **Data is either in main space — where you can use ANY name — or it is in -> `system.*`.** - -Read `docs/concepts/field-addressing.md` (published on the docs site) for the -full contract; this entry is the migration ledger. - -### Breaking — query surfaces (`where` / `orderBy` / `groupBy` / aggregation) - -- **A bare field name ALWAYS addresses your metadata.** `orderBy: 'createdAt'` - no longer silently means the engine timestamp — it now refuses with a typed - `UnresolvableFieldError` naming both candidates unless you actually have a - user field of that name. Engine scalars are addressed explicitly: - `system.id`, `system.type`, `system.subtype`, `system.createdAt`, - `system.updatedAt`, `system.confidence`, `system.weight`, - `system.visibility`, `system.service`, `system.createdBy` (relations mirror - with `system.verb`/`system.sourceId`/`system.targetId`). - **Sweep list:** `where: { subtype: … }` → `where: { 'system.subtype': … }` · - `orderBy: 'createdAt'` → `'system.createdAt'` · `groupBy: ['noun']` → - `['system.type']` · any bare `visibility`/`service`/`confidence` filter that - meant the engine value → its `system.*` spelling. Every missed site fails - LOUDLY with the correction in the error message — nothing silently changes - meaning without telling you. -- **Unimplemented `find()` options refuse** (`cursor`, `includeRelations`, - `writeOnly` → `UnsupportedFindOptionError`); `order` is validated; - accepted-and-ignored is dead as a class. -- **The ordering contract is pinned cross-engine:** missing/null `orderBy` - values sort LAST in both directions, ties break by id ascending, and rows - are never dropped from an ordered read. - -### Breaking — write surfaces - -- **There are no reserved metadata names anymore.** `metadata: { confidence, - type, id, level, data, content, … }` are ordinary user fields — stored - verbatim, indexed, filterable, sortable, aggregatable, faithful across - restarts, index rebuilds, and `asOf()` time travel. The 8.x - reserved-key-in-bag throw is GONE; code that relied on it (or on the - `'warn'`/`'remap'` lift) must set engine scalars via their dedicated params - (`confidence`, `weight`, `subtype`, `visibility`, …) — the bag never touches - them now. -- **`reservedFieldPolicy` is removed.** Passing it throws at construction with - the migration note. `RESERVED_ENTITY_FIELDS`/`RESERVED_RELATION_FIELDS` - remain exported but now describe the stored record's engine half, not a ban - list; the `NoReservedEntityKeys`/`NoReservedRelationKeys` types are no-op - (deprecated). -- **The one refused spelling:** a metadata key literally starting `system.` - (namespace forgery) — typed error on `add`/`update`/`relate`/`updateRelation`. -- **Name-based index exclusions are gone.** Fields named `content`, `data`, - `id`, `vector`, … in your bag now INDEX like everything else (they were - silently un-indexed before — `where` on them returned `[]` with no error). - Value-shape rules stay, uniform across all names: arrays >10 never become - posting scalars; long values index hashed. -- **Migration transforms receive one normalized view** (engine fields - top-level, your bag nested under `metadata`) regardless of how old the - stored record is, and must return the same shape — a stray non-engine - top-level key refuses with the fix in the message. - -### Storage format (automatic, no action) - -- New/updated records persist as **nested-bag records** (engine fields - top-level, your bag verbatim under `metadata`, sealed by a format stamp) — - the shape that makes collider names lossless. Old flat records stay - readable forever; nothing rewrites your data in place. -- **Index epoch 3:** derived-index keys split the namespaces (bare user keys · - literal `system.` keys; the legacy `noun` column is gone). Every - brain rebuilds its derived indexes from canonical once, at first open — - observable via `getIndexStatus()`, no manual step. Pair this release with - the same-day native-accelerator release (its peer floor rises to `>=9`). -- Raw-record consumers (fact-log scanners, export tooling): read bags through - the exported shape-aware splitters (`splitNounMetadataRecord` / - `splitVerbMetadataRecord`) — they handle both record eras. - -### Fixed in the same train - -- Default visibility exclusion was a silent no-op under the new addressing on - pre-release builds (internal/system-tier rows could leak into default - reads) — now pinned by conformance tests at every lifecycle boundary. -- Per-type count surfaces (`getStats()`, count-by-type) read the new type - column, with a legacy fallback for pre-rebuild reads. -- Aggregation `source.where` evaluated dotted keys as nested paths — dotted - addresses now match per-key, and the internal per-type counts aggregate - rebuilds itself onto the new keys automatically. - -### Conformance - -Both engines ship a shared self-arming conformance suite (the law cases, the -ordering contract, and the reopen-collider fidelity case: every collider name -written as user data, verified verbatim through live reads, reopen, a forced -epoch rebuild, and time travel). Capability signal: -`FIELD_ADDRESSING_CAPABILITY = 'field-addressing/v1'` plus the typed error -classes, exported from the package root. - -## v8.10.3 — 2026-08-03, 8.10-line backport (natural field names stop colliding with engine internals) - -From a production report: sorting by a user metadata field named `level` silently -returned insertion order — the engine's internal HNSW node layer (also called -`level`) shadowed the user's field in every by-name read, and the indexing path -stamped a hardcoded `0` into the same index column (multi-valued poison). `level` -is a perfectly natural field name (game characters, priorities, floors); the -engine was wrong, not the caller. - -- **`level` is user data now, everywhere.** Engine plumbing no longer resolves by - name, never shadows metadata, and never enters the indexed views. `orderBy: - 'level'`, `where: { level: 9 }`, `groupBy: ['level']` all read YOUR field. - Regression pins: `tests/integration/level-field-shadow.test.ts` (the reporting - consumer's exact repro rows). -- **Index epoch 2.** The derived posting set changed, so every existing brain - rebuilds its metadata index from canonical at first open — poisoned columns - heal automatically; no manual step. First open after upgrade pays one rebuild - (observable via `getIndexStatus()`); pair this release with the same-day - native-accelerator release, which makes `level` indexable on the native path. -- **`transact()` metadata-only updates stop rewriting the vector record** — the - v8.10.2 write-granularity law now covers the batch/plan path too (it was - fixed for `update()` but the transact plan builder still staged the - unconditional save). If you batch stat touches through `transact()`, this is - your write-amplification fix. -- (The "coming next" note this entry carried shipped as v9.0.0 — the - field-addressing law above.) - ---- - -## v8.10.2 — 2026-07-29 (metadata-only updates stop rewriting the vector record) - -From a production incident on a large deployment: a read-heavy sweep that bumped -per-entity stats (metadata-only `update()` calls) saturated the disk — 5.8GB written -in 40 minutes — because every `update()` unconditionally re-persisted the WHOLE noun -record, unchanged vector included, fsynced. - -- **`update()` write granularity fixed at the core.** A metadata-only update (no new - `data`, `vector`, or `type`) now writes the metadata leg and index deltas ONLY — - the vector-bearing noun record is never rewritten. Vector-side writes and HNSW - reindexing still happen exactly when the vector side actually changed. Regression - pins: `tests/integration/update-write-granularity.test.ts`. -- **Consumer guidance:** per-entity stat touches are now cheap, but batch them anyway - (one `transact()` instead of N `update()` calls) — granularity fixes the cost per - touch; batching fixes the count. -- Idle VFS `PathResolver` no longer logs `NaN% hit rate` once a minute (stats log - only on new traffic, at debug level). -- Native graph providers' `graph-lsm-*` storage keys are recognized as system - resources — the per-boot `Unknown key format` warning for them is gone. - -Pairs with the native accelerator's same-day patch release; adopt as one bump. - ---- - -## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) - -From a production incident: a native-provider op ground 38-40s inside a transaction, -blew the ~32s apply-phase budget, was rolled back (zero loss, by design), and a -downstream pipeline hot-retried the identical operation into a 6-minute, 100%-CPU -storm. Investigation confirmed Brainy itself never auto-retries a timed-out -transaction — the storm was entirely the consumer's own retry loop, driven by a -"retryable" doc-prose claim with no machine-readable contract to branch on. This -release closes that contract gap and, separately, fixes a real `warm()` reporting gap -surfaced by the same investigation. - -- **`TransactionTimeoutError` is now a machine-readable no-hot-retry contract.** Two - new typed, always-`true` fields replace prose-only guidance: - - `retryable: true` — the operation MAY succeed on a later attempt, once the - underlying slowness resolves or the budget is deliberately raised - (`transactionBudgetFloorMs`, or a batch's own `timeoutMs` override). - - `hotRetryUnsafe: true` — an immediate, identical retry re-pays the FULL cost of - the work that just timed out (it does not resume partway) and can cascade into - exactly the CPU storm above. **Never loop on this error.** The documented pattern - is a latch, not a retry loop: - ``` - on TransactionTimeoutError: - record { at: Date.now(), error } - rethrow loudly to your own caller - hold a cooldown window before any re-attempt - clear the latch only on a subsequent success - ``` - - `context` (unchanged, now fully documented) carries the backoff inputs: - `timeoutMs`, `operationIndex`, `elapsedMs`, `totalOperations`, `operationName`. - - Every "retryable" doc-prose site referencing this error (`transact()`'s - `timeoutMs` option, `transactionBudgetFloorMs`, `Transaction.execute()`) now - points at these fields instead of bare prose. - - Regression-pinned: the engine never internally re-drives a timed-out operation - (verified via an execution counter through both the single-op write path and - `add()`'s upsert-race retry loop), so this has always been true — it is now - provable and typed. -- **Dead code removed**: `TransactionManager.executeTransactionWithResult()` had zero - callers in this codebase and is deleted. -- **`brain.warm()`'s metadata surface now routes through the ACTIVE provider.** A - production deployment's warm report showed `metadata: 'unavailable'` under a native - metadata provider — the previous logic only duck-typed the built-in JS manager's - `hydrateAll()` method, which a native provider has no reason to implement. The - metadata provider contract (`MetadataIndexProvider`, `src/plugin.ts`) gains an - optional `warm?(): Promise` hook, mirroring the existing vector and graph - provider hooks. `brain.warm()` now checks the active provider's own `warm()` FIRST, - falls back to the JS manager's `hydrateAll()` when absent, and only reports - `'unavailable'` when neither exists — never `init()` as a stand-in, since a native - provider's `init()` may be a cheap verify rather than a real warm. A native - provider lights this surface up the same way `@soulcraft/cor` already lights the - vector and graph surfaces: implement `warm()` on its metadata provider. -- **New: `brain.maintenanceDebt()`** — the observability seam so an operator sees a - provider's outstanding background maintenance work (pending bytes/items, last pass - outcome, whether it's converging) BEFORE it grinds into the kind of budget-busting - op this release's timeout contract exists for, instead of discovering it as a CPU - storm. It is a pure passthrough: brainy applies no thresholds, no polling, and no - estimation — it calls each active provider's own optional `maintenanceDebt?()` hook - (vector, metadata, graph — the same three contracts `warm?()` lives on) and reports - the payload verbatim, or `'unavailable'` when a surface's provider doesn't track - debt. Useful as a pre-warm/post-warm check or a boot gate. `@soulcraft/cor` does not - yet implement the hook as of this release — expect it on cor's next release; until - then all three surfaces honestly report `'unavailable'`. - -## Unreleased (the warm contract: cold-restart writes stop paying demand-load latency) - -From a production deployment's cold-restart incident: the FIRST writes after every -restart on a large brain measured 33–35s each (page cache cold) against the transact -apply budget — Brainy's own op-count-scaled budget, `max(30s, opCount × 2s)`, e.g. -32,000ms for a 16-op batch. There is no external deadline in this story, and no -post-completion veto either: every write is itself a multi-operation transaction (a -single `add()` applies several operations — canonical writes, the vector-index insert, -the metadata-index update), so ONE cold operation that legitimately runs ~33s consumes -the whole budget, the gate before the NEXT operation trips, and the write rolls back -atomically (zero loss, by design) — refused, retried, refused again, until the page -cache warms passively (~30 minutes). The cure is not weaker atomicity; it is the two -new knobs below — a budget floor sized for cold stores, and a warm contract that pays -demand-load cost OFF the transaction path. - -- **The budget's start-gating contract is now explicit, documented, and pinned by - regression tests.** The budget gates STARTING the next operation — completed work is - never rolled back for elapsed time — and a transaction's first operation now - unconditionally starts by code, not merely because elapsed time happens to be ~0 when - it is checked. This has been the shipped schedule since 8.7.0 (no behavior change for - existing integrations); it is now stated in `Transaction.execute()`'s contract JSDoc - and enforced by tests so it cannot silently regress. Mid-batch atomicity is unchanged: - a trip before operation `i+1` still rolls back `0..i` and throws a retryable - `TransactionTimeoutError`. -- **The budget's 30s floor is now configurable**: `new Brainy({ transactionBudgetFloorMs })` - raises (or lowers) the floor of `max(transactionBudgetFloorMs, opCount × 2000)` for every - internal transact batch. Useful for a store whose cold writes legitimately run past 30s - per operation, so a bulk batch gets a proportionally larger runway instead of tripping - mid-batch on cold-cache latency. -- **New: `brain.warm()`** — eagerly loads/faults-in the vector index, metadata index, and - graph adjacency so the first real operation after a cold restart runs at steady-state - cost instead of paying demand-load latency on the critical path. Returns a `WarmReport` - with one honest outcome per surface — never conflate the first two: - - `'warmed'` — the surface's own provider `warm()` hook ran, or a full-hydration seam - loaded every shard/field/segment from storage. Steady-state cost is paid. - - `'probed'` — no `warm()` hook was available, so a best-effort read (one `search()` call - for the vector index) faulted in *some* backing storage as a side effect — real work, - but never reported as `'warmed'`. - - `'unavailable'` — nothing ran (no hook, no hydration seam, or nothing to probe). -- **New config: `warmOnOpen: true`** makes `init()` await `brain.warm()` before it resolves - — a deliberate blocking trade-off: startup takes longer, the first request doesn't. - Default `false` (unchanged lazy behavior). -- **New optional provider hook: `warm?(): Promise`** on the vector and graph - acceleration provider contracts (`src/plugin.ts`) — a native provider can implement it to - eagerly pretouch its own backing storage (e.g. mmap pretouch); absence means brainy falls - back to the probe/hydration behavior above. -- **Transaction op-name strings changed in journals/timings**: the vector-index - transaction operations were renamed from `AddToHNSW`/`RemoveFromHNSW` to backend-neutral - `AddToVectorIndex(...)`/`RemoveFromVectorIndex(...)` — the old names hard-coded an - algorithm that may not be the one actually running (a non-HNSW native vector provider - emitting `"RemoveFromHNSW"` has sent an operator hunting an index that doesn't exist). - The parenthesized suffix names the ACTIVE backend: `hnsw-js` for the built-in engine, or - the native provider's own identity when it self-identifies. **If you parse these op-name - strings** (log processors, journal tooling), update your matcher from - `AddToHNSW`/`RemoveFromHNSW` to `AddToVectorIndex(`/`RemoveFromVectorIndex(`. -- **Provider identity is now a REQUIRED `name` field** on the vector provider contract - (`VectorIndexProvider.name`, `src/plugin.ts`) — every implementation self-reports its own - identity truthfully (its algorithm/engine), never inheriting a default. It renders as the - op-name suffix above and, wherever the vector index identifies itself in prose log lines, - as the tag `[vector-index:]`. **Native provider adoption is a one-line change**: - declare `readonly name = ''`. A provider instance that still lacks - `name` at runtime (an older native build compiled against the previous, optional field) is - never crashed on and never silently mislabeled: it stamps `unknown-provider` and emits one - loud warning naming the missing field, so the gap is discoverable instead of a permanent - fossil label in every journal line. - -## v8.9.0 — 2026-07-19 (flush is durability-only: history maintenance moves to close()) - -The write path stops paying maintenance costs — the last structural piece of the -flush-storm class (a production deployment measured single writes blocked 25–191s behind -history reclaim running inline on flush under memory pressure): - -- **`flush()` never compacts history.** It persists the current window's deltas and - nothing else — its cost no longer depends on history backlog or retention mode, in any - configuration. **`close()` is the auto-compaction site** (time-bounded per pass, ~5s; - an early stop is a consistent prefix and the next pass resumes). -- **`compactHistory()` gains `timeBudgetMs`** — bound your own maintenance windows; the - same resumable-prefix guarantee applies. -- **The documented trade**: a long-lived writer that never closes accumulates history - until its next explicit `compactHistory()`. Predictable writes, explicit maintenance. - If you run bounded retention on an always-on service, schedule a periodic - `compactHistory({ ...caps, timeBudgetMs })` in your maintenance window. -- **New public doc: `docs/performance-envelopes.md`** — measured per-op envelopes - (p50/p95 at stated scales, hardware, and backend, with the measuring script cited). - Refresh rule going forward: any release touching a measured path re-runs that op's - benchmark and updates the envelope in the same release. -- **New in this file: the Removed APIs 7.x→8.x table** (top of this document) — every - removal with its sanctioned replacement, one place, per the engine-currency contract. - Standing from here: removals only at majors, after ≥1 minor of loud runtime deprecation. - -## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting) - -Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution -regimes where there must be one: - -- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on - delete.** The delete/update hooks fed the aggregation engine a partial entity view (type, - service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group - on the way DOWN — counts drifted upward forever after any delete, and updates that moved an - entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity - entity view (every reserved field top-level, the same shape the add path uses). If your - deployment derives stats from reserved-field aggregates, re-define those aggregates once - after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted - persisted counts do not self-heal retroactively. -- **Aggregation `source.where` on reserved fields now filters** instead of silently matching - nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level - standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says. -- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally - (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or - `ids: []` used to resolve successfully having deleted nothing. All three now throw. -- **`find()` accepts both where-key spellings.** Metadata is flattened at index time - (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now - falls back to its flattened spelling when the prefixed one isn't indexed — the - "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A - literal nested custom key named `metadata` still wins when indexed as spelled.) - -## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest) - -### The flush-storm fix (production incident, reported by a long-running deployment) - -Under the default adaptive retention, **every `flush()` re-walked the entire committed -generation history** to compute total history bytes for the budget check — O(all -generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with -70,000+ accumulated generations that turned every write into a full-tail scan (60-100s -writes), even though the budget (free-RAM-based) never tripped and nothing was ever -reclaimed. Fixed: - -- `historyBytes()` now maintains a **running total**: seeded by one walk on first use, - then updated incrementally at every commit and reclaim — the adaptive retention check - on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through - both commit paths and compaction). -- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count, - total on-disk bytes, generation/timestamp range, compaction horizon, retention mode, - and the effective adaptive budget — the one-call fleet-audit for sizing retention - exposure per brain. -- Interim guidance for keep-everything deployments already affected: `retention: 'all'` - skips the adaptive accounting entirely (and is the correct policy if you never want - history reclaimed). The accumulated files are harmless at rest; this release removes - the per-write cost of their existence. - -### The import dedup off-switch (lifecycle honesty) - -The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes -after an import, merging entities judged duplicates by id / name / vector similarity) had -three lifecycle defects, all fixed: - -- **`enableDeduplication: false` now actually disables it.** The background pass was - scheduled unconditionally — an import that explicitly opted out could still have - entities auto-removed 5 minutes later. The flag now gates BOTH the inline merge and - the background pass (regression-pinned). -- **One deduplicator per brain, owned by the brain.** Each `import()` call constructed its - own coordinator + deduplicator, so the "debounced" timer never actually debounced across - imports (N imports = N delete timers). The brain now owns a single instance — the - debounce genuinely spans imports — and `close()` cancels pending work, so a delete pass - can never fire against a closed brain. -- **The 5-minute timer is unref'd** — a pending pass no longer holds the process open - (the exit-hang class; this timer had escaped the earlier sweep). - -Retention note for keep-everything deployments: with `enableDeduplication: false` on -import calls and `retention: 'all'` in config, no engine path removes records -automatically. - -## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments) - -Small minor: brains now detect the two OS limits that bite at pool scale and warn **before** -the incident instead of during it. - -- At open (once per process, Linux-only, measurement-only), Brainy reads `RLIMIT_NOFILE` - (soft/hard, from `/proc/self/limits`) and `vm.max_map_count`, and warns loudly when either - sits below the pool-scale floors (soft NOFILE < 65 536; max_map_count < 262 144) — with the - exact raise commands (`ulimit -n` / `LimitNOFILE=` / `sysctl vm.max_map_count`). On stock - defaults the failure otherwise arrives as `EMFILE` or a failed mmap deep inside an index - open, long after the real cause stopped being visible. An unreadable limit produces **no** - warning — no measurement, no claim (non-Linux platforms stay silent). -- Exported for ops doors: `checkOsLimits()` returns the full `OsLimitsReport` - (values + warnings) programmatically, with the floors exported as constants. - -## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init) - -Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a -second writer on the same brain directory fail loudly): - -- **Lock acquisition claims atomically.** The acquire path used read-then-write, leaving a - window where two processes racing an *absent* lock could both "succeed" — and the loser - kept running unlocked, silently. The claim is now an atomic create-exclusive write - (`O_EXCL`): exactly one racer wins; the loser re-evaluates and either fails loudly with - the winner's details or performs a verified stale-takeover. Bounded retries; contention - beyond them fails loudly rather than degrading into a lockless open. -- **`BRAINY_WRITER_LOCKED` survives `init()`.** The conflict error documents a - machine-readable contract (`err.code`, `err.lockInfo` with the holder's pid/host/ - heartbeat), but init's error wrapping silently stripped both, leaving consumers a message - to regex against. The error now passes through unwrapped. - -Measured while verifying (for operators sizing audits): `brain.auditGraph()` at a -production-consumer scale of ~2,600 relationships / 800 entities costs ~0.1 s warm and -~0.5 s cold, with exact scar counting across reopen. - -## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry) - -The bulk-import ergonomics release, from a consumer's measured production incident (a serial -import on network-attached storage at ~2 s/op met a flat 30 s transaction budget): - -- **The transact apply budget now scales with the batch** — `max(30 s, opCount × 2 s)` — or - is exactly what you pass as the new `TransactOptions.timeoutMs`. A flat 30 s cap silently - limited honest bulk work to ~15 operations on slow disks while looking generous for small - batches. Internal batch paths (e.g. `removeMany` chunks) get the same scaling. -- **`TransactionTimeoutError` is a diagnosis, not just a failure**: it now reports the - operation it stopped at as `i/N` with the operation's name, elapsed vs budgeted time, and - states the batch rolled back atomically and is retryable. Its `context` carries the same - fields programmatically. -- **The transact envelope is documented** — batch sizing, budget math, chunking with - `ifAbsent` idempotency, and the precompute pattern (`embedBatch` + per-op `vector`) that - keeps model inference out of the commit path. Guide: `docs/guides/optimistic-concurrency.md`. -- Note: `brain.embed()` / `brain.embedBatch()` (the precompute APIs) already ship — public, - with native-provider passthrough, verified end-to-end (batch and single paths produce - bit-identical vectors; a vector-supplied `add` is fully searchable). Honest measurement: - on the default WASM engine, batch throughput ≈ sequential (~160 ms/text) — the precompute - win is keeping inference out of the budgeted commit path, not raw embedding speed. - -## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument) - -A minor release adding one new public API, from the fleet's graph-trust program: a read-only -audit that **proves whether relationship reads return stored truth** on a given brain. - -- **`brain.auditGraph(options?)`** walks every canonical relationship record, queries the same - read path applications use (`related()` / VFS `readdir`) with all visibility tiers included, - and classifies every discrepancy into its failure family: `missingFromReads` (records the - read path omits — a stale adjacency index), `danglingEndpoints` (relationships whose endpoint - entity no longer exists — the historical partial-delete scar class), and `readOnlyVerbIds` - (read-path edges with no stored record — ghosts). Design-hidden internal/system edges are - counted separately so intentional hiding is never misclassified as loss. Counts are exact; - example lists cap at `maxExamples` with an explicit `truncatedExamples` flag; the result is - narrated loudly on incoherence. Mutates nothing — safe on a live brain. -- The operational pairing: audit → if incoherent, `repairIndex()` → audit again. A `coherent` - report after the repair is the verified all-clear. Run it after any engine upgrade, restore, - or migration. Guide: `docs/guides/inspection.md`. -- Also: `getNounIds` pagination now refuses an undecodable resume cursor loudly (the same - contract `getNouns`/`getVerbs` gained in 8.5.2 — the third and final walk brought under it). -- Types exported: `GraphAuditReport`, `GraphAuditDiscrepancy`. - -## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud) - -Hardening patch from a migration incident (a byte-copied store on new hardware; the service -entered a silent full-CPU loop at boot). Four changes, all in the aggregation engine's -backfill/adoption path: - -- **Backfill walks are exception-safe and non-destructive.** A rescan now builds into a - staging map and swaps in atomically on completion; a mid-walk failure drops the staging map, - keeps the previous live state serving, and surfaces the storage error to the failing query. - Previously the walk wiped live state *before* a scan that could throw, never cleared the - pending flag on failure, and re-ran a full walk on every subsequent query — a silent - wipe/walk/throw loop at the caller's retry rate. -- **Failed walks are latched.** After a walk fails, retries within a 30-second cooldown rethrow - the recorded error instantly instead of re-walking — a tight caller-side retry loop now costs - one loud error per query, never a full store walk per query. -- **Adoption is generation-verified.** Persisted aggregation state is stamped with the store's - committed generation at flush; reopen adoption requires the stamp to equal the current - watermark. Stale state (unclean shutdown) or over-counting state (a fact-log truncation on a - copied store pulled the watermark back) triggers exactly one loud rescan — never a silent - adopt. Pre-8.5.2 state on generation-aware stores rescans once after upgrade, then is stamped. -- **The path narrates.** Adoption decisions, no-adoptable-state outcomes, walk start/finish - (entity count + duration), and walk failures all log by default; a non-advancing storage - pagination cursor aborts the walk loudly instead of looping forever. - -Plus three guards from a full audit of every loop in the open/init path: - -- **Invalid pagination cursors fail loudly.** A supplied-but-undecodable resume token to - `getNouns`/`getVerbs` used to silently restart the walk at offset 0 — to a `while(hasMore)` - caller that re-serves page 1 forever (an unbounded silent CPU loop). It now throws with a - clear message instead. -- **The graph cold-load verb walk has a stall guard.** `hasMore=true` with a missing or - non-advancing cursor aborts loudly instead of re-reading the same page forever. -- **A derived index AHEAD of the store is named at open.** Brainy already surfaced a provider - generation *behind* the committed watermark; the *ahead* direction (the signature of a - byte-copy of a live service, or a log truncation during crash recovery) now logs a loud - warning explaining what happened and that `brain.repairIndex()` forces a heal — instead of - passing unnamed into whatever the derived index does next. - -## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed) - -Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box -while vector search stayed fast, and every `find({ limit: 5000 })` suddenly failing against an -"auto-configured query limit of 1000"). Three fixes, one cosmetic: - -- **Aggregation state is actually adopted on reopen.** The boot pattern `defineAggregate()` → - query raced the engine's async state load: the synchronous define always won, flagged a - backfill, and the first query then wiped the just-loaded persisted state and re-walked the - entire store — every restart, forever. Reopening with an unchanged definition now adopts the - persisted state directly (zero scans); a backfill runs only on a real definition change, a - missing/failed state load, or a write that landed before adoption (exactness wins). Apps that - rely on persisted definitions without re-defining at boot also no longer race a spurious - "Aggregate not defined". -- **Backfills are single-flight and batched.** Concurrent queries on a cold aggregate used to - each wipe the others' partial state and start their own full store walk — under steady query - arrival the store never converged (the 40–90 s loop). Now all concurrent queries share one - walk, and one walk fills every aggregate pending backfill (M aggregates ≠ M scans). -- **The query-cap "learning" ratchet is removed.** `maxLimit` is a memory-protection bound, but - a hidden tuner shrank it 20 % per recorded query while the lifetime-average query time - exceeded 1 s — down to a floor of 1000, below the documented 10 000 auto floor, with no - practical recovery, and the resulting error blamed "available free memory" (stale basis - label). The cap now comes from its construction-time basis (or your explicit - `maxQueryLimit` / `reservedQueryMemory`) alone and never changes at runtime; query timing is - recorded for diagnostics only. -- **Cosmetic:** the engine's own persistence keys (`__aggregation_*`, `brainy:entityIdMapper`) - no longer log `[Storage] Unknown key format` at boot — they were always routed correctly; - they're now recognized before the warning fires. - -Operationally: if a host was bitten, upgrading and restarting is the whole fix — no repair -ritual needed. Setting `maxQueryLimit` explicitly remains the valve that bypasses auto-detection -entirely. - -## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier) - -Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass: - -- **Index providers can now reach the fact log through the storage adapter** — new optional - capability `storage.scanFacts()` / `storage.factLogHeadGeneration()` / `storage.factSegmentPaths()`, - wired by the host brain at init as a closure over its live log. Providers hold only `storage` and - must never construct their own fact-log reader (the log's open path is writer-side); `null` means - "no fact log here — use the enumeration walk." -- **The family-stamp verifier is shared** via `@soulcraft/brainy/internals` - (`readFamilyStamp` / `writeFamilyStamp` / `verifyFamilyStamp` + types) so first-party native - providers run literally the same verification function, never a synchronized copy. -- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a - per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass. -- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider - compares its stamp's `sourceGeneration` against the store's truth without parsing the private - manifest format. -- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for - `transact()` today; the single-op path is pinned as the documented future target — group commit - becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan - snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate - beneath it). - -No behavior change for applications; all additions. - -## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream) - -A minor release, fully backward-compatible (all additions; no behavior change for existing APIs). -This is infrastructure: it changes nothing about how you query today, and lays the substrate that -makes index heals and incremental catch-up sequential-read problems instead of directory walks. - -- **Every committed write now also appends a "fact" — an after-image commit record.** Alongside the - existing before-image history, each committed generation (single-op and `transact()` alike) appends - what each touched entity/relationship *became* — or a body-less tombstone for a removal — to an - append-only, checksummed segment log under `_generations/facts/`. Crash-safe by construction: a - torn tail is detected and ignored; on open the log is reconciled to committed truth, so an absent - generation always means "never committed." `transact()` facts are durable when `transact()` - returns; single-op facts ride the same group-commit flush as their history. - -- **New: `brain.scanFacts()`** — stream committed facts in commit order, in batches, with heal-grade - telemetry (total scope up front; per-batch generation range, byte size, and segment; a summary - cross-check at the end; loud abort on any gap — never a silent skip). **`brain.factSegmentPaths()`** - hands zero-copy consumers the immutable sealed segment files directly. New exported types: - `CommitFact`, `FactOp`, `FactScanBatch`, `FactScanHandle`. Facts accumulate from the first write - after upgrading — pre-existing history is not retroactively converted (enumeration remains the - fallback for old data). - -- **New: the entity-tree family stamp.** At every flush/close, brainy stamps which committed - generation the canonical entity files reflect plus the rollup invariants (entity/relationship - counts) that verify the tree whole. At open, coherence is a comparison — a genuine divergence is - loud and names the failing invariant; `repairIndex()` recounts from canonical and re-stamps. New - exports: `readFamilyStamp`, `verifyFamilyStamp`, `ENTITY_TREE_STAMP_PATH`, `FamilyStamp` types. - -- **Storage adapters** gain optional binary raw-byte primitives (`appendRawBytes`, `readRawBytes`, - `writeRawBytes`, `rawByteSize`) — feature-detected; the filesystem and memory adapters implement - them; an adapter without them simply hosts no fact log. The fact-log namespace is registered as a - protected family: no sweeper or GC can delete under it. - -No API breaks. 24 new tests; the full commit-path regression suite is green. - ## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory) One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index 91d40d49..00000000 --- a/SECURITY.md +++ /dev/null @@ -1,36 +0,0 @@ -# Security Policy - -## Reporting a vulnerability - -Email **security@soulcraft.com**. That's the one door for security reports -across the company, and it works the same way for Brainy: every report is -read by a human, you'll get a private receipt, and we'll work with you on -coordinated disclosure — please don't open a public issue for anything -that isn't already public. - -Include what you'd want if you were on the other end: affected version, -how to reproduce, and what you think the impact is. If you have a patch or -a suggested fix, send it along — it's welcome but not required. - -There is no bounty program today. We're saying that plainly so you know -what to expect going in. - -## Response time - -We respond as fast as truth allows. That means: no fixed SLA, no promise of -a reply within a specific number of hours — but a real report from a real -person gets read promptly and taken seriously. If you haven't heard anything -in a reasonable stretch, a follow-up email is completely fine. - -## Supported versions - -The latest `8.x` minor release line receives security fixes. If you're -running an older major version, please upgrade before reporting — we can't -commit to backporting fixes to unsupported lines. - -## Scope - -This policy covers the `@soulcraftlabs/brainy` package itself — the code in -this repository. If you're evaluating a deployment that also uses -`@soulcraft/cor`, report issues in that package the same way, to the same -address; we'll route internally. diff --git a/bin/brainy-ts.js b/bin/brainy-ts.js index 90a35e98..4e9aedb8 100644 --- a/bin/brainy-ts.js +++ b/bin/brainy-ts.js @@ -3,7 +3,7 @@ /** * Modern TypeScript CLI Runner * - * This is the entry point after npm install @soulcraftlabs/brainy + * This is the entry point after npm install @soulcraft/brainy * It runs the compiled TypeScript CLI code */ diff --git a/bun.lock b/bun.lock index 1e3e66e2..c31b3865 100644 --- a/bun.lock +++ b/bun.lock @@ -3,7 +3,7 @@ "configVersion": 0, "workspaces": { "": { - "name": "@soulcraftlabs/brainy", + "name": "@soulcraft/brainy", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", "@azure/identity": "^4.0.0", diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md index b2d22fb1..4134ae63 100644 --- a/docs/DEVELOPER_LEARNING_PATH.md +++ b/docs/DEVELOPER_LEARNING_PATH.md @@ -25,13 +25,13 @@ ### Prerequisites ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Your First Neural Database ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' // Step 1: Create and initialize Brainy const brain = new Brainy({ @@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou ### Building a Knowledge Graph ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**. ### Triple Intelligence in Action ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in ### Files as Intelligent Entities ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**. ### Production-Ready Deployment ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' // 1. PRODUCTION STORAGE - Filesystem with off-site snapshots console.log('Initializing production storage...\n') diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 77fbbd79..1cc38ce9 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -1217,7 +1217,7 @@ where: { await brain.find({ type: 'Document' }) // ✅ Correct: Use NounType enum -import { NounType } from '@soulcraftlabs/brainy' +import { NounType } from '@soulcraft/brainy' await brain.find({ type: NounType.Document }) // ❌ Error: Operator not recognized diff --git a/docs/MIGRATION-V3-TO-V4.md b/docs/MIGRATION-V3-TO-V4.md index 680b6928..29c409ac 100644 --- a/docs/MIGRATION-V3-TO-V4.md +++ b/docs/MIGRATION-V3-TO-V4.md @@ -153,13 +153,13 @@ brainy-data/ ### Step 1: Update Brainy Package ```bash -npm install @soulcraftlabs/brainy@latest +npm install @soulcraft/brainy@latest ``` **Check your version:** ```bash -npm list @soulcraftlabs/brainy -# Should show: @soulcraftlabs/brainy@4.0.0 +npm list @soulcraft/brainy +# Should show: @soulcraft/brainy@4.0.0 ``` ### Step 2: No Code Changes Required! ✅ @@ -374,7 +374,7 @@ If you encounter issues, you can rollback: ```bash # Reinstall v3 -npm install @soulcraftlabs/brainy@^3.50.0 +npm install @soulcraft/brainy@^3.50.0 # Restart application ``` @@ -389,7 +389,7 @@ rm -rf ./data cp -r ./data-backup ./data # Reinstall v3 -npm install @soulcraftlabs/brainy@^3.50.0 +npm install @soulcraft/brainy@^3.50.0 ``` ## Common Migration Scenarios @@ -539,7 +539,7 @@ console.log('Storage type:', status.type) **Migration Checklist:** - ✅ Backup data -- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`) +- ✅ Update npm package (`npm install @soulcraft/brainy@latest`) - ✅ Restart application (automatic migration) - ✅ Verify data integrity - ✅ Enable lifecycle policies diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index b543e84a..248a2c70 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -323,24 +323,58 @@ Only the graph adjacency index carries a committed scale assertion: - ✅ **Single-Node by Design**: One process owns one `path`; scale out at the service layer - ✅ **Zero Stubs**: Every line of code is production-ready -## Index Build at Open (10.4+) +## Lazy Loading Performance -As of 10.4, `brain.init()` runs every needed index rebuild to completion before -it returns — always, regardless of dataset size. There is no lazy, -first-query rebuild path: a brain either finishes opening healthy, or `init()` -fails loudly. `disableAutoRebuild` no longer defers index construction to a -first query; it has no effect on *when* a rebuild runs. Manual control over -rebuilds is `repairIndex({ rebuild: [...] })`. See -[Index Health](concepts/index-health.md) for the full read-gate contract -(providers self-report readiness via `healthReport()`; a read against a -not-serving provider throws a typed `*NotReadyError` rather than rebuilding -mid-query). +Brainy supports two initialization modes for optimal performance across different use cases: - +### Mode 1: Auto-Rebuild (Default) + +```javascript +const brain = new Brainy() +await brain.init() // Rebuilds indexes during init (~500ms-3s for 10K entities) +``` + +**Performance:** +- Init time: 500ms-3s (depends on dataset size) +- First query: Instant (indexes already loaded) +- Use case: Traditional applications, long-running servers + +### Mode 2: Lazy Loading + +```javascript +const brain = new Brainy({ disableAutoRebuild: true }) +await brain.init() // Returns instantly (0-10ms) + +const results = await brain.find({ limit: 10 }) // First query triggers rebuild (~50-200ms) +const more = await brain.find({ limit: 100 }) // Subsequent queries instant (0ms check) +``` + +**Performance:** +- Init time: 0-10ms (instant) +- First query: 50-200ms (includes index rebuild for 1K-10K entities) +- Subsequent queries: 0ms check (instant) +- Concurrent queries: Wait for same rebuild (mutex prevents duplicates) + +**Concurrency Safety:** +```javascript +// 100 concurrent queries immediately after init +await brain.init() + +const promises = Array.from({ length: 100 }, () => + brain.find({ limit: 10 }) +) + +const results = await Promise.all(promises) +// ✅ Only 1 rebuild triggered (mutex) +// ✅ All 100 queries return correct results +// ✅ Total time: ~60ms (not 6000ms!) +``` + +**Use Cases for Lazy Loading:** +- **Serverless/Edge**: Minimize cold start time (0-10ms init) +- **Development**: Faster restarts during development +- **Large datasets**: Defer index loading until needed +- **Read-heavy workloads**: Writes don't wait for index rebuild ## Zero Configuration Required @@ -350,6 +384,10 @@ Brainy is designed to be **smart enough to tune itself dynamically**. No configu // That's it. Brainy handles everything. const brain = new Brainy() await brain.init() + +// Or with lazy loading for serverless +const brain = new Brainy({ disableAutoRebuild: true }) +await brain.init() // Instant (0-10ms) ``` ### Automatic Self-Tuning @@ -357,6 +395,7 @@ await brain.init() - **Metadata Index**: Auto-builds sorted indices for range queries on first use - **Graph Index**: Auto-flushes every 30 seconds - **Default Tuning**: Research-based vector index defaults +- **Lazy Loading**: Indices built only when needed - **Cache Management**: LRU caches with TTL ### Intelligent Defaults diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index 238d2252..d9a4d3e7 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -10,7 +10,7 @@ next: - guides/storage-adapters --- -# Plugin System +# Plugin Development Guide Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cor` provides optional native acceleration, and it's the same system available to any developer. @@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement ### 1. Implement the `BrainyPlugin` interface ```typescript -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' const myPlugin: BrainyPlugin = { name: 'my-brainy-plugin', // Must be unique (typically your npm package name) @@ -90,7 +90,7 @@ await brain.init() **Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`: ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import myPlugin from './my-plugin.js' const brain = new Brainy() @@ -200,30 +200,15 @@ members so a warm reopen never pays a redundant rebuild-from-canonical: - **`init?(): Promise`** — eager cold-load. Brainy awaits it once during `brain.init()`, after the metadata provider's `init()` (the id-mapper hydrates first) and **before the rebuild gate**. -- **`healthReport?(): HealthReport`** — the PREFERRED signal (10.4+). A named, - synchronous, O(1) verdict derived from the provider's own exact ledgers — never a - sample, never I/O, must never throw for a well-formed provider. Brainy's read gate - (`assessProviderHealth()`) reads this INSTEAD of `isReady()` / size heuristics when - present: `serving: false` refuses the read with a typed `*NotReadyError` rather than - triggering a rebuild — a read never starts a store walk. `healthy` marks every - *verified* invariant holding; a family named in `unledgered` counts as neither - healthy nor broken. See `HealthReport` / `LedgerInvariantResult` / - `InvariantSource` in `src/plugin.ts`, and - [Index Health](concepts/index-health.md) for the consumer-facing story. -- **`isReady?(): boolean`** — honest durability signal, the fallback when - `healthReport()` is absent. `true` ⇔ the persisted index is loaded (or cheaply - demand-loadable) and consistent with what was last persisted. When exposed, the - gate defers to this signal **instead of** the `size() === 0` / `totalEntries === 0` - heuristics — a disk-native index may report 0 resident entries while fully durable. - Never return `true` if the durable state failed to load: the signal is honest in - both directions, and a not-ready provider gets its rebuild even when `size() > 0`. +- **`isReady?(): boolean`** — honest durability signal. `true` ⇔ the persisted index is + loaded (or cheaply demand-loadable) and consistent with what was last persisted. When + exposed, the rebuild gate defers to this signal **instead of** the `size() === 0` / + `totalEntries === 0` heuristics — a disk-native index may report 0 resident entries + while fully durable. Never return `true` if the durable state failed to load: the + signal is honest in both directions, and a not-ready provider gets its rebuild even + when `size() > 0`. - **`isMigrating?(): boolean`** — while `true`, the provider owns its index (background migration); brainy skips its rebuild entirely. -- **`validateInvariants?(): Promise`** — the async DEEP - diagnostic (full scans allowed), distinct from the bounded, sync `healthReport()`. - Must never throw — a failure is `healthy: false` data, not an exception; a provider - that throws anyway is read as a loud, unverified failure (never as "healthy") by - every caller, never silently retried into a rebuild. Providers that implement none of these keep the size/count heuristics — correct for engines whose `rebuild()` *is* their load path (like brainy's built-in JS vector index). @@ -272,10 +257,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor #### `cache` **Type:** `UnifiedCache` -Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`). +Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`). ```typescript -import type { UnifiedCache } from '@soulcraftlabs/brainy/internals' +import type { UnifiedCache } from '@soulcraft/brainy/internals' context.registerProvider('cache', myNativeCache) ``` @@ -325,8 +310,8 @@ Plugins can register custom storage backends that users reference by name. ### Implementing a Storage Adapter ```typescript -import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin' -import type { StorageAdapter } from '@soulcraftlabs/brainy' +import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin' +import type { StorageAdapter } from '@soulcraft/brainy' class MyStorageAdapter implements StorageAdapter { async init(): Promise { /* ... */ } @@ -360,9 +345,9 @@ Brainy provides three entry points for plugin developers: | Import Path | Contents | Stability | |-------------|----------|-----------| -| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) | -| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | -| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | +| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) | +| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | +| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | ## Diagnostics @@ -440,7 +425,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations ```typescript // simd-distance-plugin/src/plugin.ts -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' // Hypothetical native module import { simdCosineDistance } from './native.js' @@ -470,7 +455,7 @@ export default simdDistancePlugin "main": "./dist/plugin.js", "types": "./dist/plugin.d.ts", "peerDependencies": { - "@soulcraftlabs/brainy": ">=7.0.0" + "@soulcraft/brainy": ">=7.0.0" } } ``` @@ -478,7 +463,7 @@ export default simdDistancePlugin Usage: ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) await brain.init() diff --git a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md index ad4a4a40..4568cd31 100644 --- a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md +++ b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md @@ -54,7 +54,7 @@ After 40 API calls: ```typescript // server.ts -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // SINGLETON INSTANCE let brainInstance: Brainy | null = null @@ -174,7 +174,7 @@ process.on('SIGTERM', async () => { ```typescript // server.ts - Clean Bun implementation -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain: Brainy | null = null diff --git a/docs/README.md b/docs/README.md index ddb37d20..3290001f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,7 +5,7 @@ ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md index 4b120bd8..94c6a6cb 100644 --- a/docs/RELEASE-GUIDE.md +++ b/docs/RELEASE-GUIDE.md @@ -99,7 +99,7 @@ Examples: ```bash # 1. Deprecate wrong version on npm -npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y" +npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y" # 2. Fix version in package.json # 3. Republish correct version diff --git a/docs/SCALING.md b/docs/SCALING.md index 054d2096..e9ae1136 100644 --- a/docs/SCALING.md +++ b/docs/SCALING.md @@ -13,7 +13,7 @@ ### In-Memory ```typescript -import Brainy from '@soulcraftlabs/brainy' +import Brainy from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) ``` @@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the -open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native +open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`. `find()` query latency, p50 / p95 (200 queries each): diff --git a/docs/api-contract.json b/docs/api-contract.json deleted file mode 100644 index aafd838a..00000000 --- a/docs/api-contract.json +++ /dev/null @@ -1,1544 +0,0 @@ -{ - "contractVersion": 1, - "engine": "@soulcraftlabs/brainy", - "compatibility": { - "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", - "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" - }, - "doors": [ - { - "name": "adaptiveHistoryBudgetBytes", - "kind": "method", - "arity": 1 - }, - { - "name": "add", - "kind": "method", - "arity": 1 - }, - { - "name": "addMany", - "kind": "method", - "arity": 1 - }, - { - "name": "adoptLogAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "adoptLogAuthorityInner", - "kind": "method", - "arity": 0 - }, - { - "name": "aggViewFromEntity", - "kind": "method", - "arity": 1 - }, - { - "name": "anyProviderMigrating", - "kind": "method", - "arity": 0 - }, - { - "name": "applyFusionScoring", - "kind": "method", - "arity": 2 - }, - { - "name": "applyGraphConstraints", - "kind": "method", - "arity": 2 - }, - { - "name": "armIdleFlushTimer", - "kind": "method", - "arity": 2 - }, - { - "name": "asOf", - "kind": "method", - "arity": 2 - }, - { - "name": "assertGenerationStoreReady", - "kind": "method", - "arity": 1 - }, - { - "name": "assertWritable", - "kind": "method", - "arity": 1 - }, - { - "name": "audit", - "kind": "method", - "arity": 0 - }, - { - "name": "auditGraph", - "kind": "method", - "arity": 0 - }, - { - "name": "autoAdoptLegacyVfsBlobsIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "autoAlpha", - "kind": "method", - "arity": 1 - }, - { - "name": "autoCompactHistory", - "kind": "method", - "arity": 0 - }, - { - "name": "awaitMigrationLock", - "kind": "method", - "arity": 1 - }, - { - "name": "awaitPendingEmbeds", - "kind": "method", - "arity": 0 - }, - { - "name": "backfillAggregateIfNeeded", - "kind": "method", - "arity": 1 - }, - { - "name": "batchGet", - "kind": "method", - "arity": 2 - }, - { - "name": "brainWideStrictRequiresSubtype", - "kind": "method", - "arity": 1 - }, - { - "name": "bridgeLegacyPendingEmbedSidecars", - "kind": "method", - "arity": 0 - }, - { - "name": "buildAtGenerationVectors", - "kind": "method", - "arity": 2 - }, - { - "name": "buildGraphView", - "kind": "method", - "arity": 4 - }, - { - "name": "buildMetadataFilter", - "kind": "method", - "arity": 1 - }, - { - "name": "buildMigrationUpdate", - "kind": "method", - "arity": 5 - }, - { - "name": "buildRelationMigrationUpdate", - "kind": "method", - "arity": 5 - }, - { - "name": "cacheVerbInt", - "kind": "method", - "arity": 2 - }, - { - "name": "canServeVectorAtGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "checkHealth", - "kind": "method", - "arity": 0 - }, - { - "name": "checkMigrations", - "kind": "method", - "arity": 0 - }, - { - "name": "clear", - "kind": "method", - "arity": 0 - }, - { - "name": "clearPendingEmbed", - "kind": "method", - "arity": 1 - }, - { - "name": "close", - "kind": "method", - "arity": 0 - }, - { - "name": "closeDurableSteps", - "kind": "method", - "arity": 0 - }, - { - "name": "cluster", - "kind": "method", - "arity": 1 - }, - { - "name": "collectProviderInvariants", - "kind": "method", - "arity": 0 - }, - { - "name": "compactHistory", - "kind": "method", - "arity": 1 - }, - { - "name": "consumeMetadataWatermarkVerdict", - "kind": "method", - "arity": 1 - }, - { - "name": "convertMetadataToEntity", - "kind": "method", - "arity": 2 - }, - { - "name": "convertNounToEntity", - "kind": "method", - "arity": 1 - }, - { - "name": "counts", - "kind": "accessor" - }, - { - "name": "createIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "createMigrationBackupIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "createPinnedDb", - "kind": "method", - "arity": 1 - }, - { - "name": "createResult", - "kind": "method", - "arity": 4 - }, - { - "name": "dbFinalizationRegistry", - "kind": "accessor" - }, - { - "name": "dbHost", - "kind": "accessor" - }, - { - "name": "defineAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "detectIdKind", - "kind": "method", - "arity": 3 - }, - { - "name": "diagnostics", - "kind": "method", - "arity": 0 - }, - { - "name": "diff", - "kind": "method", - "arity": 2 - }, - { - "name": "embed", - "kind": "method", - "arity": 1 - }, - { - "name": "embedBatch", - "kind": "method", - "arity": 2 - }, - { - "name": "emitCommitted", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceSubtypeOnAdd", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceSubtypeOnRelate", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceTrackedFieldValues", - "kind": "method", - "arity": 2 - }, - { - "name": "enhanceNLPResult", - "kind": "method", - "arity": 2 - }, - { - "name": "enqueuePendingEmbed", - "kind": "method", - "arity": 1 - }, - { - "name": "ensureAggregationIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "ensureIndexesLoaded", - "kind": "method", - "arity": 0 - }, - { - "name": "ensureInitialized", - "kind": "method", - "arity": 1 - }, - { - "name": "entityForAggFromRawRecord", - "kind": "method", - "arity": 1 - }, - { - "name": "entityFromGenerationRecord", - "kind": "method", - "arity": 3 - }, - { - "name": "entityIntsToUuids", - "kind": "method", - "arity": 1 - }, - { - "name": "entityViewFromRawRecord", - "kind": "method", - "arity": 2 - }, - { - "name": "excludedVisibilityTiers", - "kind": "method", - "arity": 1 - }, - { - "name": "executeGraphSearch", - "kind": "method", - "arity": 2 - }, - { - "name": "executeProximitySearch", - "kind": "method", - "arity": 1 - }, - { - "name": "executeTextSearch", - "kind": "method", - "arity": 2 - }, - { - "name": "executeVectorSearch", - "kind": "method", - "arity": 3 - }, - { - "name": "explain", - "kind": "method", - "arity": 1 - }, - { - "name": "export", - "kind": "method", - "arity": 0 - }, - { - "name": "extract", - "kind": "method", - "arity": 2 - }, - { - "name": "extractConcepts", - "kind": "method", - "arity": 2 - }, - { - "name": "extractEntities", - "kind": "method", - "arity": 2 - }, - { - "name": "factSegmentPaths", - "kind": "method", - "arity": 1 - }, - { - "name": "fieldCountsAggregateName", - "kind": "method", - "arity": 1 - }, - { - "name": "fillSubtypes", - "kind": "method", - "arity": 1 - }, - { - "name": "filterIdsBelted", - "kind": "method", - "arity": 2 - }, - { - "name": "find", - "kind": "method", - "arity": 1 - }, - { - "name": "findAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "findDuplicates", - "kind": "method", - "arity": 1 - }, - { - "name": "findMatchingWords", - "kind": "method", - "arity": 3 - }, - { - "name": "flush", - "kind": "method", - "arity": 0 - }, - { - "name": "formatInfo", - "kind": "method", - "arity": 0 - }, - { - "name": "formatSubtypeError", - "kind": "method", - "arity": 1 - }, - { - "name": "generation", - "kind": "method", - "arity": 0 - }, - { - "name": "generationDigest", - "kind": "method", - "arity": 1 - }, - { - "name": "get", - "kind": "method", - "arity": 2 - }, - { - "name": "getActivePlugins", - "kind": "method", - "arity": 0 - }, - { - "name": "getAvailableFields", - "kind": "method", - "arity": 0 - }, - { - "name": "getBackgroundDeduplicator", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldsForType", - "kind": "method", - "arity": 1 - }, - { - "name": "getFieldStatistics", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldsWithCardinality", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldValues", - "kind": "method", - "arity": 1 - }, - { - "name": "getIndexStats", - "kind": "method", - "arity": 0 - }, - { - "name": "getIndexStatus", - "kind": "method", - "arity": 0 - }, - { - "name": "getMemoryStats", - "kind": "method", - "arity": 0 - }, - { - "name": "getNeighborUuids", - "kind": "method", - "arity": 2 - }, - { - "name": "getNounCount", - "kind": "method", - "arity": 0 - }, - { - "name": "getOptimalQueryPlan", - "kind": "method", - "arity": 1 - }, - { - "name": "getStats", - "kind": "method", - "arity": 1 - }, - { - "name": "getStorageType", - "kind": "method", - "arity": 0 - }, - { - "name": "getSubtypeRule", - "kind": "method", - "arity": 1 - }, - { - "name": "getTripleIntelligence", - "kind": "method", - "arity": 0 - }, - { - "name": "getTypedNeighbors", - "kind": "method", - "arity": 4 - }, - { - "name": "getVerbCount", - "kind": "method", - "arity": 0 - }, - { - "name": "graph", - "kind": "accessor" - }, - { - "name": "graphAccelerationProvider", - "kind": "method", - "arity": 0 - }, - { - "name": "graphCommunities", - "kind": "method", - "arity": 1 - }, - { - "name": "graphCommunitiesFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphCommunitiesNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphEntityInt", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExport", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExportFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExportNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphPath", - "kind": "method", - "arity": 3 - }, - { - "name": "graphPathFallback", - "kind": "method", - "arity": 3 - }, - { - "name": "graphPathNative", - "kind": "method", - "arity": 4 - }, - { - "name": "graphRank", - "kind": "method", - "arity": 1 - }, - { - "name": "graphRankFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphRankNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphSubgraph", - "kind": "method", - "arity": 2 - }, - { - "name": "graphSubgraphFallback", - "kind": "method", - "arity": 4 - }, - { - "name": "graphSubgraphFromQuery", - "kind": "method", - "arity": 5 - }, - { - "name": "graphSubgraphNative", - "kind": "method", - "arity": 5 - }, - { - "name": "groupByLabel", - "kind": "method", - "arity": 2 - }, - { - "name": "hasStorageMethod", - "kind": "method", - "arity": 1 - }, - { - "name": "hasVectorOrTextCriteria", - "kind": "method", - "arity": 1 - }, - { - "name": "health", - "kind": "method", - "arity": 0 - }, - { - "name": "highlight", - "kind": "method", - "arity": 1 - }, - { - "name": "highlightSemanticPhase", - "kind": "method", - "arity": 5 - }, - { - "name": "history", - "kind": "method", - "arity": 2 - }, - { - "name": "historyStats", - "kind": "method", - "arity": 0 - }, - { - "name": "hub", - "kind": "accessor" - }, - { - "name": "hydrateIdMapperForGraphRebuild", - "kind": "method", - "arity": 0 - }, - { - "name": "hydrateNativeSubgraph", - "kind": "method", - "arity": 2 - }, - { - "name": "import", - "kind": "method", - "arity": 2 - }, - { - "name": "importPluginPackage", - "kind": "method", - "arity": 1 - }, - { - "name": "incidentEdges", - "kind": "method", - "arity": 3 - }, - { - "name": "indexStats", - "kind": "method", - "arity": 0 - }, - { - "name": "init", - "kind": "method", - "arity": 1 - }, - { - "name": "insights", - "kind": "method", - "arity": 0 - }, - { - "name": "isEmbeddingReady", - "kind": "method", - "arity": 0 - }, - { - "name": "isInfrastructureWrite", - "kind": "method", - "arity": 1 - }, - { - "name": "isInitialized", - "kind": "accessor" - }, - { - "name": "isReadOnly", - "kind": "accessor" - }, - { - "name": "kickBackgroundFlush", - "kind": "method", - "arity": 1 - }, - { - "name": "kickEmbedWorker", - "kind": "method", - "arity": 0 - }, - { - "name": "legacyLayoutMigrationPhase", - "kind": "method", - "arity": 0 - }, - { - "name": "loadAnalyticsGraph", - "kind": "method", - "arity": 1 - }, - { - "name": "loadPlugins", - "kind": "method", - "arity": 0 - }, - { - "name": "logAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "maintenanceDebt", - "kind": "method", - "arity": 0 - }, - { - "name": "materializeAtGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "metadataIndexRetractionOp", - "kind": "method", - "arity": 3 - }, - { - "name": "migrate", - "kind": "method", - "arity": 1 - }, - { - "name": "migrateField", - "kind": "method", - "arity": 1 - }, - { - "name": "migrateInternal", - "kind": "method", - "arity": 2 - }, - { - "name": "migrateLegacyZeroNormVfsRootIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "migrationSnapshot", - "kind": "method", - "arity": 0 - }, - { - "name": "neededFamiliesMigrating", - "kind": "method", - "arity": 1 - }, - { - "name": "neighbors", - "kind": "method", - "arity": 2 - }, - { - "name": "newId", - "kind": "method", - "arity": 0 - }, - { - "name": "nlp", - "kind": "method", - "arity": 0 - }, - { - "name": "normalizeConfig", - "kind": "method", - "arity": 1 - }, - { - "name": "noteWriteForPersistence", - "kind": "method", - "arity": 0 - }, - { - "name": "now", - "kind": "method", - "arity": 0 - }, - { - "name": "onChange", - "kind": "method", - "arity": 1 - }, - { - "name": "pagination", - "kind": "accessor" - }, - { - "name": "parseMigrationPath", - "kind": "method", - "arity": 1 - }, - { - "name": "parseNaturalQuery", - "kind": "method", - "arity": 1 - }, - { - "name": "pathExists", - "kind": "method", - "arity": 2 - }, - { - "name": "pendingEmbedCount", - "kind": "method", - "arity": 0 - }, - { - "name": "performInit", - "kind": "method", - "arity": 1 - }, - { - "name": "persistPinnedGeneration", - "kind": "method", - "arity": 2 - }, - { - "name": "persistSingleOp", - "kind": "method", - "arity": 6 - }, - { - "name": "pickMetadataProbe", - "kind": "method", - "arity": 1 - }, - { - "name": "pickVectorProbe", - "kind": "method", - "arity": 0 - }, - { - "name": "pinGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "planGetEntity", - "kind": "method", - "arity": 3 - }, - { - "name": "planTransact", - "kind": "method", - "arity": 1 - }, - { - "name": "planTxAdd", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxRelate", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxRemove", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxUnrelate", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxUpdate", - "kind": "method", - "arity": 3 - }, - { - "name": "projectionGauges", - "kind": "method", - "arity": 0 - }, - { - "name": "providerForFamily", - "kind": "method", - "arity": 1 - }, - { - "name": "providerIsMigrating", - "kind": "method", - "arity": 1 - }, - { - "name": "providerMigrationStatus", - "kind": "method", - "arity": 0 - }, - { - "name": "queryAggregate", - "kind": "method", - "arity": 2 - }, - { - "name": "queryIndexFamilies", - "kind": "method", - "arity": 1 - }, - { - "name": "readPath", - "kind": "method", - "arity": 2 - }, - { - "name": "ready", - "kind": "accessor" - }, - { - "name": "rebuildIndexesIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "rebuildMetadataIndexOnline", - "kind": "method", - "arity": 0 - }, - { - "name": "reconcileLogDivergence", - "kind": "method", - "arity": 2 - }, - { - "name": "reconstructPath", - "kind": "method", - "arity": 4 - }, - { - "name": "recordStateAt", - "kind": "method", - "arity": 3 - }, - { - "name": "recoverPendingEmbedsFromLog", - "kind": "method", - "arity": 0 - }, - { - "name": "registerShutdownHooks", - "kind": "method", - "arity": 0 - }, - { - "name": "relate", - "kind": "method", - "arity": 1 - }, - { - "name": "related", - "kind": "method", - "arity": 1 - }, - { - "name": "relateMany", - "kind": "method", - "arity": 1 - }, - { - "name": "relationFromGenerationRecord", - "kind": "method", - "arity": 2 - }, - { - "name": "relationshipSubtypesOf", - "kind": "method", - "arity": 1 - }, - { - "name": "releaseGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "remove", - "kind": "method", - "arity": 1 - }, - { - "name": "removeAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "removeMany", - "kind": "method", - "arity": 1 - }, - { - "name": "removeMigrationBackupSafe", - "kind": "method", - "arity": 0 - }, - { - "name": "repackHistory", - "kind": "method", - "arity": 1 - }, - { - "name": "repairIndex", - "kind": "method", - "arity": 1 - }, - { - "name": "requestFlush", - "kind": "method", - "arity": 1 - }, - { - "name": "requireProviders", - "kind": "method", - "arity": 1 - }, - { - "name": "requireSubtype", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveAsOfGeneration", - "kind": "method", - "arity": 2 - }, - { - "name": "resolveDiffEndpoint", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveHiddenIds", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveHNSWPersistMode", - "kind": "method", - "arity": 0 - }, - { - "name": "resolveRawGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveRetentionPolicy", - "kind": "method", - "arity": 0 - }, - { - "name": "resolveVerbEndpointInts", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveVerbIntsToIds", - "kind": "method", - "arity": 1 - }, - { - "name": "restore", - "kind": "method", - "arity": 2 - }, - { - "name": "rrfFusion", - "kind": "method", - "arity": 4 - }, - { - "name": "runAggregationBackfillWalk", - "kind": "method", - "arity": 0 - }, - { - "name": "runAggregationCatchUp", - "kind": "method", - "arity": 0 - }, - { - "name": "runEmbedWorker", - "kind": "method", - "arity": 0 - }, - { - "name": "runOracle", - "kind": "method", - "arity": 1 - }, - { - "name": "runRepairIndexPhases", - "kind": "method", - "arity": 5 - }, - { - "name": "scanFacts", - "kind": "method", - "arity": 1 - }, - { - "name": "seedIdsToInts", - "kind": "method", - "arity": 1 - }, - { - "name": "selectorToSeedIds", - "kind": "method", - "arity": 1 - }, - { - "name": "setRetentionBudget", - "kind": "method", - "arity": 1 - }, - { - "name": "setupEmbedder", - "kind": "method", - "arity": 0 - }, - { - "name": "setupIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "setupStorage", - "kind": "method", - "arity": 0 - }, - { - "name": "similar", - "kind": "method", - "arity": 1 - }, - { - "name": "similarity", - "kind": "method", - "arity": 2 - }, - { - "name": "splitForHighlighting", - "kind": "method", - "arity": 2 - }, - { - "name": "stampBrainFormat", - "kind": "method", - "arity": 0 - }, - { - "name": "stampBrainFormatIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "stampEntityTree", - "kind": "method", - "arity": 0 - }, - { - "name": "stampProjectionWatermarks", - "kind": "method", - "arity": 0 - }, - { - "name": "stats", - "kind": "method", - "arity": 0 - }, - { - "name": "storageAdapter", - "kind": "accessor" - }, - { - "name": "stream", - "kind": "method", - "arity": 0 - }, - { - "name": "streaming", - "kind": "accessor" - }, - { - "name": "subtypesOf", - "kind": "method", - "arity": 1 - }, - { - "name": "trackField", - "kind": "method", - "arity": 1 - }, - { - "name": "transact", - "kind": "method", - "arity": 2 - }, - { - "name": "transactionLog", - "kind": "method", - "arity": 1 - }, - { - "name": "unrelate", - "kind": "method", - "arity": 1 - }, - { - "name": "unvectorNounForRootMigration", - "kind": "method", - "arity": 1 - }, - { - "name": "update", - "kind": "method", - "arity": 1 - }, - { - "name": "updateMany", - "kind": "method", - "arity": 1 - }, - { - "name": "updateRelation", - "kind": "method", - "arity": 1 - }, - { - "name": "upsertMergeParams", - "kind": "method", - "arity": 2 - }, - { - "name": "use", - "kind": "method", - "arity": 1 - }, - { - "name": "usesDefaultWasmEmbedder", - "kind": "method", - "arity": 0 - }, - { - "name": "validateIndexConsistency", - "kind": "method", - "arity": 0 - }, - { - "name": "vectorSearchAtGeneration", - "kind": "method", - "arity": 4 - }, - { - "name": "verbsToRelations", - "kind": "method", - "arity": 1 - }, - { - "name": "verbToRelationLike", - "kind": "method", - "arity": 1 - }, - { - "name": "verifyEntityTreeStamp", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyGraphAdjacencyLive", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyLogAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyMetadataLive", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyVectorLive", - "kind": "method", - "arity": 0 - }, - { - "name": "versionedIndexProviders", - "kind": "method", - "arity": 0 - }, - { - "name": "vfs", - "kind": "accessor" - }, - { - "name": "waitForIndexed", - "kind": "method", - "arity": 2 - }, - { - "name": "warm", - "kind": "method", - "arity": 0 - }, - { - "name": "warmupEmbeddings", - "kind": "method", - "arity": 0 - }, - { - "name": "warnIfReadsDegraded", - "kind": "method", - "arity": 1 - }, - { - "name": "wireConnectionsCodec", - "kind": "method", - "arity": 0 - }, - { - "name": "wireGraphIdResolver", - "kind": "method", - "arity": 0 - } - ], - "errors": [ - "BrainyError", - "DerivedArtifactMissingError", - "GraphIndexNotReadyError", - "MetadataIndexNotReadyError", - "MigrationInProgressError", - "ProtectedArtifactError", - "VectorIndexNotReadyError" - ], - "operators": { - "accepted": [ - "between", - "contains", - "endsWith", - "eq", - "equals", - "excludes", - "exists", - "greaterThan", - "greaterThanOrEqual", - "gt", - "gte", - "hasAll", - "in", - "length", - "lessThan", - "lessThanOrEqual", - "lt", - "lte", - "matches", - "missing", - "ne", - "noneOf", - "notEquals", - "oneOf", - "startsWith" - ], - "servedOnIndexPath": [ - "between", - "contains", - "eq", - "equals", - "excludes", - "exists", - "greaterThan", - "greaterThanOrEqual", - "gt", - "gte", - "hasAll", - "in", - "lessThan", - "lessThanOrEqual", - "lt", - "lte", - "missing", - "ne", - "noneOf", - "notEquals", - "oneOf" - ], - "refusedByIndexPath": [ - "endsWith", - "length", - "matches", - "startsWith" - ], - "combinators": [ - "allOf", - "anyOf", - "not" - ] - }, - "fieldAddressing": { - "systemKeyPrefix": "system.", - "systemEntityScalars": [ - "confidence", - "createdAt", - "createdBy", - "id", - "service", - "subtype", - "type", - "updatedAt", - "visibility", - "weight" - ], - "systemRelationScalars": [ - "confidence", - "createdAt", - "createdBy", - "service", - "sourceId", - "subtype", - "targetId", - "updatedAt", - "verb", - "visibility", - "weight" - ], - "plumbingFields": [ - "_rev", - "connections", - "data", - "level", - "vector" - ] - }, - "health": { - "verdicts": [ - "pass", - "warn", - "fail" - ], - "healKinds": [ - "none", - "repair", - "rebuild" - ], - "servingWithholdingInvariants": [ - "index-initialized", - "durable-state-present", - "manifest-residency", - "replay-clean", - "strand-latch" - ] - } -} diff --git a/docs/api/README.md b/docs/api/README.md index ba49ff48..4ca84364 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -24,7 +24,7 @@ next: ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() // Zero config! await brain.init() // VFS auto-initialized! @@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization ### Db API errors -All exported from `@soulcraftlabs/brainy`: +All exported from `@soulcraft/brainy`: | Error | Thrown by | Meaning | |---|---|---| @@ -1451,34 +1451,6 @@ const count = await brain.getVerbCount() --- -### The canonical count ledger (`StorageAdapter.getCanonicalCounts()`) - -An OPTIONAL method on the `StorageAdapter` interface (implemented by both -built-in adapters), not a method on `Brainy` itself — relevant if you're -writing a custom storage adapter or composing a provider's own -`healthReport()`. O(1), no I/O. Per family (`nouns`/`verbs`): - -```typescript -interface CanonicalCounts { - nouns: { counted: number; all: number } - verbs: { counted: number; all: number } - suspect: boolean -} -``` - -- `counted` mirrors `getNounCount()` / `getVerbCount()` (public + internal tiers). -- `all` is the ALL-visibility scalar — every tier, including system/internal - records — the denominator a derived index's own coverage math is measured - against. -- `suspect` is `true` when an unprovable delete has left `all` unverified since - the last recount; `brain.repairIndex()` clears it with a real canonical walk. - -Adapters without the ledger omit the method; treat absence as "no -denominator," never as zero. See -**[Index Health](../concepts/index-health.md)** for the full story. - ---- - ### Subtype & facet APIs Full guide: **[Subtypes & Facets](../guides/subtypes-and-facets.md)**. @@ -1859,104 +1831,6 @@ const semanticOnly = await brain.getStats({ excludeVFS: true }) --- -### `repairIndex(options?)` → `Promise` - -The ceremony door for index repair. Bare `repairIndex()` is report-driven: it -prunes orphaned containers, recomputes count rollups, reconciles VFS -containment, and rebuilds only a derived-index family whose own health check -asks for it. Pass `options.rebuild` to force one or more families to rebuild -UNCONDITIONALLY — no health check is consulted — when an operator has -independent reason to reconcile a family regardless of what it self-reports. - -```typescript -// Report-driven: only heals what actually needs it -const report = await brain.repairIndex() -console.log(report.healedTotal, report.families) - -// Explicit: force the graph adjacency to rebuild from canonical, unconditionally -await brain.repairIndex({ rebuild: ['graph'] }) - -// Explicit: force all three derived indexes to rebuild -await brain.repairIndex({ rebuild: 'all' }) -``` - -**`RepairReport`:** -- `families: RepairFamilyReport[]` — one row per family checked -- `healedTotal: number` — items healed across every family -- `durationMs: number` - -**`RepairFamilyReport`** (one row): -- `family: string` — e.g. `'orphaned-containers'`, `'count-rollups'`, - `'vfs-containment'`, `'metadata-corruption'`, `'provider:metadata'`, - `'provider:graph'`, `'provider:vector'` -- `checked: boolean` — was this family actually examined (`false` ⇒ see `skipped`) -- `healed: number` — items re-posted/corrected in place (the incremental heal count) -- `missing?: { count: number; sample: string[] }` — exact count plus a capped id - sample when the check can name what diverged (never the full list) -- `rebuilt?: boolean` — a full generational rebuild ran (vs. an incremental heal) -- `detail?: string` / `reason?: string` — narration -- `skipped?: string` — why the family wasn't checked - -Full walkthrough — what each family checks, degraded-but-serving vs. not-ready, -and what `suspect` counts mean — in -**[Index Health](../concepts/index-health.md)**. - ---- - -### Index readiness: typed errors, `healthReport()`, `disableAutoRebuild` - -Every derived-index provider (vector, graph, metadata) may expose a named, -synchronous, O(1) `healthReport()` composed from its own exact ledgers — the -signal Brainy's read gate trusts over sampling or size heuristics. `init()` -brings every provider to serving before it returns; there is no first-query -lazy-rebuild path. A read that reaches a provider whose health report says it -isn't serving throws instead of rebuilding mid-query: - -| Error | Thrown by | Meaning | -|---|---|---| -| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | Graph adjacency isn't serving | -| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | -| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | - -All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish -"index not ready" from a genuine empty result: - -```typescript -import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy' - -try { - const rows = await brain.find({ where: { status: 'active' } }) -} catch (err) { - if (err instanceof MetadataIndexNotReadyError) { - // reconcile: await brain.repairIndex(), then retry - } else { - throw err - } -} -``` - -**`disableAutoRebuild`** no longer defers index construction to the first -query. A needed rebuild always runs at `open()`, regardless of this flag or -dataset size; the flag has no effect on *when* a rebuild runs. Full manual -control lives in `repairIndex({ rebuild: [...] })`, above. - -### `validateIndexConsistency()` → `Promise<...>` - -The deep, async diagnostic counterpart to `healthReport()` — safe to run on a -live brain, but does more work (a provider's `validateInvariants()` may run a -full scan, not just read a ledger). Aggregates the JS metadata index's own -consistency check with every derived-index provider's invariant report. - -```typescript -const validation = await brain.validateIndexConsistency() -if (!validation.healthy) { - console.log(validation.recommendation) // what to run, e.g. repairIndex() - console.log(validation.providers) // each provider's own invariant report, when exposed -} -``` - ---- - ## Lifecycle ### Initialization @@ -2208,7 +2082,7 @@ For the full taxonomy with all 169 types and their descriptions, see: - **📖 Documentation:** [Full Documentation](../) - **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues) - **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions) -- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy) +- **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy) - **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy) --- diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 83b9e23a..48064757 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with .ack | **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) | A pluggable index provider (the 8.0 plugin contract in -`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the +`@soulcraft/brainy/plugin`) may replace any of the JS implementations; the persisted formats above are contract-bound so JS and native implementations can interleave on the same directory. diff --git a/docs/architecture/finite-type-system.md b/docs/architecture/finite-type-system.md index 48a8b1fe..76492ee5 100644 --- a/docs/architecture/finite-type-system.md +++ b/docs/architecture/finite-type-system.md @@ -126,7 +126,7 @@ class TypeAwareMetadataIndex { **The Design**: Specify types clearly in your API calls: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' // Add entity with explicit type await brain.add({ @@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation { **Brainy's Approach**: Extract **typed** concepts: ```typescript -import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy' +import { NaturalLanguageProcessor } from '@soulcraft/brainy' const nlp = new NaturalLanguageProcessor() const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco") @@ -382,7 +382,7 @@ import { getVerbTypes, BrainyTypes, suggestType -} from '@soulcraftlabs/brainy' +} from '@soulcraft/brainy' // Get all available noun types const nounTypes = getNounTypes() diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md index 6a754b56..8b3dc540 100644 --- a/docs/architecture/index-architecture.md +++ b/docs/architecture/index-architecture.md @@ -723,14 +723,6 @@ async stats(): Promise { ### 5. Index Rebuilding (Lazy Loading Support) -> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is -> RETIRED.** `disableAutoRebuild` no longer defers index construction to a -> first query; `brain.init()` now runs every needed rebuild to completion -> before it returns, unconditionally, and a read against a not-serving -> provider throws a typed `*NotReadyError` instead of rebuilding mid-query. -> See `docs/concepts/index-health.md` for the current contract. Left below -> as historical background on the rebuild mechanics. - **Two modes of index loading:** #### Mode 1: Auto-Rebuild on init() (default) diff --git a/docs/architecture/initialization-and-rebuild.md b/docs/architecture/initialization-and-rebuild.md index e19bdd9f..a1645744 100644 --- a/docs/architecture/initialization-and-rebuild.md +++ b/docs/architecture/initialization-and-rebuild.md @@ -1,15 +1,5 @@ # Initialization and Rebuild Processes -> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is RETIRED.** -> `disableAutoRebuild` no longer defers index construction to a first query; -> `brain.init()` now runs every needed rebuild to completion before it -> returns, unconditionally. A read against a not-serving provider throws a -> typed `*NotReadyError` instead of rebuilding mid-query. See -> `docs/concepts/index-health.md` for the current contract; this document's -> line-number references to `src/brainy.ts` also predate the file's current -> size and are unreliable. Left as historical background on the rebuild -> mechanics, not as a current API description. - This document explains how Brainy's four indexes (MetadataIndex, vector index, GraphAdjacencyIndex, DeletedItemsIndex) initialize and rebuild from persisted storage. ## Core Principle: All Indexes Are Disk-Based diff --git a/docs/architecture/multiprocess-storage-mixin.md b/docs/architecture/multiprocess-storage-mixin.md index 1593bf8f..46f98398 100644 --- a/docs/architecture/multiprocess-storage-mixin.md +++ b/docs/architecture/multiprocess-storage-mixin.md @@ -127,7 +127,7 @@ For reference, a clean migration path: `isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for build/install artifact protection. 5. Document the new contract in `concepts/storage-adapters.md`. -6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by +6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by plugins. Estimated work: ~half a day of code, ~2 hours of doc/example updates, diff --git a/docs/architecture/noun-verb-taxonomy.md b/docs/architecture/noun-verb-taxonomy.md index 3dac6892..286464be 100644 --- a/docs/architecture/noun-verb-taxonomy.md +++ b/docs/architecture/noun-verb-taxonomy.md @@ -20,7 +20,7 @@ next: Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu - **Multi-hop Graph Traversals = Relationship Complexity** - **Result: Model data across virtually any industry** -Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. +Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. ## The Power of Standardization: Universal Interoperability diff --git a/docs/architecture/zero-config.md b/docs/architecture/zero-config.md index a35e6416..d42d6784 100644 --- a/docs/architecture/zero-config.md +++ b/docs/architecture/zero-config.md @@ -35,7 +35,7 @@ constructor and `init()`. ## Instant Start ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // That's it. No config needed. const brain = new Brainy() diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md deleted file mode 100644 index d24dd66b..00000000 --- a/docs/concepts/field-addressing.md +++ /dev/null @@ -1,233 +0,0 @@ ---- -title: Field addressing: your fields and system fields -slug: concepts/field-addressing -public: true -category: concepts -template: concept -order: 7 -description: The one rule for every query-surface field name — a bare name always means your metadata, system. reaches the ten engine scalars explicitly, and anything else refuses by name. -next: - - guides/namespace-migration - - concepts/consistency-model ---- - -# Field addressing: your fields and system fields - -Every query surface in Brainy — `find()`'s `where`, `orderBy`, aggregation -`groupBy`, and aggregation `source.where` — resolves field names by one rule, -with no exceptions: - -> **A bare field name always means your metadata. `system.` reaches an -> engine scalar, and only when you spell it explicitly.** - -```typescript -await brain.find({ orderBy: 'level' }) // reads entity.metadata.level — YOUR field -await brain.find({ orderBy: 'system.createdAt' }) // reads the engine's createdAt scalar -await brain.find({ orderBy: 'metadata.level' }) // identical to bare 'level' — explicit scope -``` - -There is no priority list, no "try the system field, fall back to metadata" -behavior, and no name that resolves differently depending on what else -happens to exist on your entities. A field called `level`, `score`, -`createdAt`, or `type` in your own `metadata` is read as *your* field, every -time, by its bare name. - -## Why this rule exists - -An internal report from a production deployment found that a user metadata -field literally named `level` was being silently shadowed by the engine's -own internal index layer field of the same name — every sort by `level` -returned insertion order, with no error raised. This rule makes that class of -bug structurally impossible: bare names belong to you, unconditionally, and -anything that isn't yours has to be spelled out. - -## The system scalars - -`system.` addresses exactly ten scalars on an entity — no more, no -fewer: - -| System field | What it is | -|---|---| -| `system.id` | The entity's id | -| `system.type` | The entity's `NounType` | -| `system.subtype` | The per-app sub-classification passed to `add()` | -| `system.createdAt` | When the entity was created | -| `system.updatedAt` | When the entity was last written | -| `system.confidence` | The `confidence` param (0–1) | -| `system.weight` | The `weight` param | -| `system.visibility` | `'public'` / `'internal'` (see the visibility tiers in [Consistency Model](./consistency-model.md)) | -| `system.service` | The multi-tenancy `service` tag | -| `system.createdBy` | Who/what created the entity | - -Relationships mirror the same eight shared scalars (`subtype`, `createdAt`, -`updatedAt`, `confidence`, `weight`, `visibility`, `service`, `createdBy`) -plus three of their own: - -| System field (relationship) | What it is | -|---|---| -| `system.verb` | The relationship's `VerbType` | -| `system.sourceId` | The id of the entity the relationship starts from | -| `system.targetId` | The id of the entity the relationship points to | - -Anything not on these two lists is not a system scalar — `system.` for -any other name refuses (see "Refusal semantics" below), even if that name -sounds like it should be engine-owned. - -## Invisible plumbing — never addressable, in either spelling - -Five names are pure engine internals. They are not reachable as a bare name, -and not reachable as `system.` either — they simply have no place on -the query surface: - -- **`vector`** — the stored embedding. It participates in similarity search - (`query`, `near`, vector `find()`), never in `where`/`orderBy`/`groupBy`. -- **`connections`** — graph adjacency. Reached through `connected` and - `brain.related()`, not through field addressing. -- **`level`** — the internal index layer number used by the nearest-neighbor - graph. It is pure index plumbing with no query-surface meaning at all — - which is exactly why a user field of the same name must never be shadowed - by it. `level` as a bare name is always yours; there is no engine-owned - spelling of it to compete with. -- **`data`** — your entity's content payload, not a scalar. It can be a - string, a number, or an arbitrary object, so sorting or filtering it as a - single comparable value would lie about its actual shape. Content is - reached through the content/text-search APIs (`query`, `searchMode: - 'text'`), not through `where`/`orderBy`. -- **`_rev`** — the per-entity revision counter used for optimistic - concurrency (`ifRev`). It is a CAS token, not a queryable dimension. - -`system.level`, `system.vector`, and `system.data` all refuse for the same -reason: they are not in the ten-scalar system map, full stop. - -## `metadata.` — the explicit spelling of "mine" - -Prefix any field with `metadata.` to say the same thing a bare name already -says, spelled out. The two are interchangeable everywhere a field name is -accepted, including `orderBy`: - -```typescript -await brain.find({ where: { 'customer.tier': 'gold' } }) -await brain.find({ where: { 'metadata.customer.tier': 'gold' } }) // identical -await brain.find({ orderBy: 'metadata.score', order: 'desc' }) // identical to orderBy: 'score' -``` - -Reach for the explicit spelling when it reads more clearly next to a -`system.` field in the same query — for example, sorting by your own `score` -while filtering on `system.confidence`. - -## No special names — the write side - -The same law governs writes: - -> **Data is either in main space, where developers can use anything, or it -> is in `system.*`.** - -There are **no reserved metadata names**. A field called `confidence`, -`type`, `id`, `data`, `content`, or anything else inside your `metadata` bag -is an ordinary user field: it is stored verbatim, indexed, filterable, -sortable, aggregatable, and it survives restarts, index rebuilds, and -time-travel (`asOf`) reads exactly as written — even when an engine scalar -shares its spelling. The engine's values are written only through their -dedicated params (`confidence`, `weight`, `subtype`, `visibility`, …) and -read at `system.`; your bag can never touch them and they can never -shadow your bag. - -```typescript -const id = await brain.add({ - data: 'Ada Lovelace', - type: NounType.Person, - confidence: 0.9, // the ENGINE scalar - metadata: { confidence: 'self-rated' } // YOUR field, same spelling — both live -}) - -await brain.find({ where: { confidence: 'self-rated' } }) // finds it (yours) -await brain.find({ where: { 'system.confidence': 0.9 } }) // finds it (engine's) -``` - -The one spelling a write refuses is a metadata key that literally starts -with `system.` — the explicit address namespace cannot be forged as a user -field name. That refusal is typed and names the fix. - -Value **shape** rules still apply uniformly to every name (they are not name -carve-outs): arrays longer than 10 elements are not turned into posting-list -scalars, and very long values are indexed by hash. - -## Refusal semantics - -A name that resolves to neither your metadata nor a system scalar is a typed -refusal, not a silent empty result and not a guess. Refusals name **both** -candidates, so the fix is always in the error text: - -```typescript -await brain.find({ orderBy: 'createdAt' }) -// UnresolvableFieldError: no metadata field 'createdAt' — did you mean -// system.createdAt or metadata.createdAt? -``` - -`UnresolvableFieldError` is exported from the package root: - -```typescript -import { UnresolvableFieldError } from '@soulcraftlabs/brainy' - -try { - await brain.find({ orderBy: 'createdAt' }) -} catch (err) { - if (err instanceof UnresolvableFieldError) { - // err.message names both candidates — usually enough to fix the call site. - } -} -``` - -A handful of `find()` options are not implemented yet: `cursor`, -`includeRelations`, and `writeOnly`. Rather than accepting them and quietly -ignoring the option, `find()` refuses with `UnsupportedFindOptionError` — -also exported from the package root — so a call site can never believe an -unimplemented option took effect when it didn't. - -## The ordering contract - -`orderBy` behaves identically regardless of which engine (the pure-TypeScript -path or a native accelerator) is serving the query: - -- An entity missing the `orderBy` field, or holding `null` on it, sorts - **LAST — in both `asc` and `desc`**. It is never treated as "smaller than - everything" in one direction and "larger than everything" in the other; it - is simply last, either way. -- Rows are **never dropped** from an ordered read because they lack the - field — a missing value changes position, never presence. -- Ties on the `orderBy` field break by **id ascending**, regardless of the - primary sort direction. - -```typescript -// employees: [{ score: 9 }, { score: 5 }, { /* no score field */ }] -await brain.find({ orderBy: 'score', order: 'desc' }) // [9, 5, missing] — missing is last -await brain.find({ orderBy: 'score', order: 'asc' }) // [5, 9, missing] — missing is STILL last -``` - -## Migrating existing call sites - -If you have call sites written before this rule shipped that rely on a bare -system name — `orderBy: 'createdAt'`, `where: { confidence: { greaterThan: -0.8 } }`, and similar — they now refuse instead of silently resolving to the -engine field. The fix is always in the error: swap the bare name for -`system.` (or `metadata.` if you actually meant your own field -of that name, and it happens to share a name with a system scalar): - -```typescript -// Before: bare 'createdAt' silently meant the engine's timestamp. -await brain.find({ orderBy: 'createdAt' }) - -// After: say which one you meant. -await brain.find({ orderBy: 'system.createdAt' }) // the engine timestamp -await brain.find({ orderBy: 'metadata.createdAt' }) // your own field named createdAt, if you have one -``` - -There is no silent migration path by design — every ambiguous call site -surfaces as a refusal naming its own fix, once, the first time it runs -against the new rule. - -## Where to go next - -- [Consistency Model](./consistency-model.md) — visibility tiers, revision - counters, and the rest of the read/write contract this page's - read-time addressing rule. diff --git a/docs/concepts/generation-fact-log.md b/docs/concepts/generation-fact-log.md deleted file mode 100644 index f9b3e974..00000000 --- a/docs/concepts/generation-fact-log.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -title: The Generation Fact Log -slug: concepts/generation-fact-log -public: true -category: concepts -template: concept -order: 6 -description: Every committed write also appends a self-verifying "fact" — an after-image commit record — to an append-only log. What facts are, the crash-safety model, the scanFacts() streaming surface, family stamps, and how index providers consume the log for sequential heals. -next: - - concepts/consistency-model - - guides/snapshots-and-time-travel ---- - -# The Generation Fact Log - -Since 8.4.0, every committed generation also appends a **fact** — a compact record of what each -touched entity or relationship *became* — to an append-only, checksummed log under -`_generations/facts/`. Where the generational history answers *"what did things look like -before?"* (before-images, powering `asOf()` and rollback), the fact log answers *"what happened, -in order?"* — one sequential, self-verifying stream of the store's present being written. - -Nothing about querying changes. The fact log exists for three consumers: - -1. **Index heals and rebuilds** — one sequential read in commit order replaces a per-entity - directory walk over millions of files. -2. **Incremental catch-up** — a derived index that knows which generation it reflects reads *just - the gap*, instead of rebuilding from scratch. -3. **Replay and audit tooling** — anything that wants the store's committed timeline as a stream. - -## What a fact is - -One fact per committed generation: - -- **`generation`** and **`timestamp`** — which commit, when. -- **`ops`** — every write in that commit: `{ kind: 'noun' | 'verb', id, record }` where `record` - holds the entity's full after-image (both stored legs), or **`null` for a tombstone** — a - removal carries no body, by design. -- **`meta`** — the transaction metadata `transact()` was submitted with, when present. -- **`blobHashes`** — content-blob references, for exact reclamation accounting. - -Facts accumulate **from the first write after upgrading** — pre-existing history is not -retroactively converted, and consumers fall back to the enumeration walk when no log exists. - -## Crash safety, in one paragraph - -Facts are appended and fsynced **inside the same durability window as the commit itself**, before -the commit point — so after a crash, the log can only ever be *ahead* of committed truth, never -behind it with a hole. On open, the store reconciles the log back to the committed watermark: -torn tails are detected by per-record checksums and cut; whole records beyond the watermark are -truncated. The invariant every reader can rely on: **an absent generation was never committed; a -present fact was.** `transact()` facts are durable the moment `transact()` returns; single-op -facts share the same group-commit flush as the rest of their generation, so a hard kill loses the -fact and the generation *together* — never a torn state. - -## Reading the log - -```typescript -const scan = brain.scanFacts({ fromGeneration: 1 }) -if (scan) { - // Telemetry up front — progress bars get a denominator from second zero. - console.log(scan.headGeneration, scan.segmentCount, scan.approxFactCount) - - for await (const batch of scan.batches()) { - // Each batch: { facts, firstGeneration, lastGeneration, factCount, byteSize, segmentId } - for (const fact of batch.facts) { - for (const op of fact.ops) { - if (op.record === null) { - // a tombstone: op.id was removed in this generation - } - } - } - } - - console.log(scan.summary()) // { factsYielded, segmentsRead } — the cross-check -} -``` - -- `scanFacts()` returns `null` when the store hosts no fact log (older store, or a storage adapter - without binary append support) — fall back to enumerating entities. -- Scans run against a **snapshot**: facts appended after the scan opens never bleed in, each fact - is yielded exactly once, and a detected gap aborts loudly — never a silent skip. -- `brain.factSegmentPaths()` returns the immutable, *sealed* segment files for zero-copy consumers - (the append-mutable tail is excluded — read it through `scanFacts()`). - -## Family stamps: how a projection proves it's current - -Anything derived from the store — an index, the entity file tree itself — carries a **family -stamp**: a small JSON record of *which committed generation the projection reflects* -(`sourceGeneration`) plus the invariants that verify it whole (exact per-file byte sizes for -bounded families; rollup invariants like entity counts for unbounded ones). At open, coherence is -a **comparison**, not a walk: - -- stamp equals the committed watermark and invariants hold → serve; -- stamp is behind → the projection reads just the gap from the fact log; -- invariants fail → loud, named divergence — `brain.repairIndex()` rebuilds from canonical and - re-stamps. - -The verifier is exported (`verifyFamilyStamp`) so every projection — TypeScript or native — runs -literally the same check. - -## For plugin authors: the storage capability - -Index providers receive the storage adapter, not the brain — so the host wires the log onto it. -Feature-detect and prefer the stream; fall back to enumeration: - -```typescript -const scan = storage.scanFacts?.({ fromGeneration: stamp.sourceGeneration + 1 }) -if (scan) { - // sequential catch-up from the log -} else { - // enumeration walk (older store or adapter) -} -const committed = storage.committedGeneration?.() // the watermark stamps compare against -``` - -Providers must never construct their own reader over the log's files — the open path belongs to -the single writer (it reconciles the log at open); the capability is the sanctioned seam. diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md deleted file mode 100644 index 923267df..00000000 --- a/docs/concepts/index-health.md +++ /dev/null @@ -1,217 +0,0 @@ ---- -title: Index Health -slug: concepts/index-health -public: true -category: concepts -template: concept -order: 8 -description: How Brainy knows whether a derived index can be trusted — exact accounting instead of sampling, the named health report, degraded-but-serving vs. not-ready, and what repairIndex() checks, heals, and rebuilds. -next: - - concepts/generation-fact-log - - guides/inspection ---- - -# Index Health - -Brainy keeps one **canonical** copy of every entity and relationship, and three -**derived** indexes built from it — vector, metadata, and graph — so `find()` can -answer semantically, by filter, and by traversal without re-deriving the answer from -scratch on every query. A derived index is a cache with a serving structure: it can -be present but stale, present but only partially loaded, or fully out of sync with -canonical after a crash. This page is about how Brainy decides whether to trust one, -what it does when it can't, and how you reconcile the two. - -## Exact accounting instead of sampling - -Older health checks worked by inference: does `size()` return something greater -than zero, does a spot-check on one known item come back correct. Both are proxies. -A cold index can report a nonzero count while its actual serving structure never -loaded, and a spot-check only proves the one item it happened to ask about. - -Every derived-index provider may now expose a named, synchronous, O(1) -`healthReport()` — composed from the provider's own **exact ledgers** (real counters -it already maintains on the write path), never a sample or a walk. This is the one -signal Brainy's read gate consults. A provider that doesn't yet expose one falls -back to an honest `isReady()` boolean, and finally to a size heuristic for engines -with neither — but wherever a `healthReport()` exists, it wins. - -Underneath, storage itself keeps an analogous **canonical count ledger**: a -`counted` scalar (the user-facing total — what `getNounCount()` / `getVerbCount()` -return) and an `all` scalar (every tier, including internal records a derived -index's own coverage math needs to compare against). This is the real denominator -a provider's `healthReport()` measures itself by, rather than a total that can only -ever ratchet upward. See [What `suspect` counts mean](#what-suspect-counts-mean) -below for the one case that ledger can't stay exact through on its own. - -## The named report - -A `HealthReport` carries, per provider (`'vector'` / `'graph'` / `'metadata'`): - -- **`healthy`** — `true` iff every *verified* invariant holds. An invariant whose - family has no ledger yet is `unledgered`, never counted either way — unknown, - not passing. -- **`serving`** — can this provider answer a query right now. A failing invariant - graded `heal: 'repair'` or `heal: 'none'` still leaves `serving: true` — this is - **degraded-but-serving**: something is off (say, a stale rollup on an - `employee` record's relationship count) but reads keep working. Only a failure - graded `heal: 'rebuild'` flips `serving` to `false` — **not-ready** — because the - provider itself is telling you its serving structure cannot answer correctly. -- **`invariants`** — each checked condition, with its provenance - (`source: 'ledger'` — an exact count; `'deep'` — a full scan, diagnostic-only; - `'unledgered'` — not yet tracked) and, for a failing one, an exact `missing` - count plus a capped sample of the affected ids — a verdict, never a dump. -- **`generation`** — bumps on every ledger mutation and rebuild, so a caller can - cache a verdict per generation instead of re-deriving it. - -The distinction that matters day to day: `healthy: false` can be entirely benign — -a maintenance window, a divergence `repairIndex()` will clean up on its own -schedule. `serving: false` is not benign. It means this provider is refusing to -answer, on its own word, right now. - -**How a failure gets its grade — the serving law.** A provider grades `heal` by -one question only: *could an answer be wrong?* — never *how expensive is the -fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post -exactly what the ledger names, reads serving throughout); it can never withhold -serving just because healing it takes work. `serving` is withheld only by a -small, named set of rebuild-graded conditions — the index not initialized, its -durable state absent, a manifest naming files that are not resident, a replay -that did not complete cleanly — the states in which an answer could genuinely be -wrong. And a read is only ever refused by the family it actually consults: a -metadata filter is answered by the metadata index alone, vector search by the -vector index, traversal by the graph index — one family's refusal never blocks -another family's reads. - -## Reads refuse — they never rebuild - -A query that reaches a not-serving provider does not trigger a rebuild from inside -the read. Brainy retired that path deliberately: a rebuild kicked off by an ordinary -`find({ where: { status: 'active' } })` call is a dark, unpredictable cost hiding -behind a request that looks like a cheap read. Instead, the read throws a typed, -catchable error naming the reason: - -| Error | Thrown when | Meaning | -|---|---|---| -| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | The graph adjacency index isn't serving — traversal would otherwise return `[]` indistinguishable from "no relationships" | -| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | -| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | - -All three are exported from `@soulcraftlabs/brainy`. Catch them where your application -needs to distinguish "this index isn't ready yet" from "there's genuinely nothing -here" — a health dashboard, a retry policy, an operator alert. The fix is always -the same: reconcile the index, either by reopening the brain (which brings every -provider to serving before `init()` returns — see the next section) or by calling -`repairIndex()` explicitly. - -```typescript -try { - const active = await brain.find({ where: { status: 'active' } }) -} catch (err) { - if (err instanceof MetadataIndexNotReadyError) { - // not a "no results" — the index itself refused; alert or retry after repair - } else { - throw err - } -} -``` - -### Rebuilds happen at open, not on first query - -`brain.init()` runs every needed rebuild to completion **before it returns**, -unconditionally, regardless of dataset size. There is no lazy, first-query -rebuild path anymore — a brain either finishes opening healthy, or it fails -open loudly. `disableAutoRebuild: true` no longer defers index construction to -the first query: it has no effect on *when* a needed rebuild runs. Full manual -control over rebuilds is `repairIndex({ rebuild: [...] })` (below), not this flag. - -## `repairIndex()` — checking and healing - -Bare `repairIndex()` is **report-driven**: it only heals what its own checks say -actually needs it, and it always returns a full per-family receipt. - -```typescript -const report = await brain.repairIndex() -report.healedTotal // total items healed across every family -report.durationMs -report.families // one row per family checked -``` - -Each `RepairFamilyReport` row names what happened: - -- **`checked`** — was this family actually examined (`false` means skipped — - see `skipped` for why). -- **`healed`** — items re-posted or corrected in place. -- **`missing`** — when the check can name what diverged: an exact `count` plus a - capped `sample` of ids. -- **`rebuilt`** — a full generational rebuild ran (as opposed to an incremental - heal). -- **`detail`** / **`reason`** / **`skipped`** — the receipt's narration; a row is - always either checked or explains why it wasn't. Nothing is silent. - -On every call, bare `repairIndex()`: - -1. Prunes orphaned canonical containers left by a partial delete. -2. Recomputes the count rollups from one canonical walk (unconditional — this is - also what clears a `suspect` ledger; see below). -3. Reconciles VFS containment edges, if the VFS is initialized. -4. Runs the metadata index's own corruption detection pass. -5. Consults each of the three derived-index providers' own health check and - rebuilds only a family whose failing invariant actually asks for it - (`heal: 'rebuild'`) — never a provider that reports `healthy` or a lesser - grade. - -### The explicit rebuild door - -`options.rebuild` skips the health check and rebuilds one or more families -**unconditionally** — the operator override for when you have independent reason -to distrust a family regardless of what it self-reports (a suspicious deploy, a -storage-layer incident, a support ticket that doesn't match what the health report -says): - -```typescript -// Force the graph adjacency to rebuild from canonical, no invariant consulted -await brain.repairIndex({ rebuild: ['graph'] }) - -// Force all three derived indexes -await brain.repairIndex({ rebuild: 'all' }) -``` - -A family named this way is recorded with `rebuilt: true` and -`reason: 'explicit rebuild requested'`, and is skipped by the normal -health-driven pass in the same call — it was already rebuilt unconditionally. - -Reach for the explicit door when you need certainty regardless of self-report; -reach for bare `repairIndex()` for routine maintenance and after any incident -where you're not sure which family (if any) needs it. - -## What `suspect` counts mean - -Storage's canonical count ledger increments the ALL-visibility total on every new -record and decrements it on every *proven* delete — one where the record was read, -or the caller supplied its prior image. A delete that cannot prove what it removed -existed doesn't guess: it flags the ledger `suspect` (an operator-visible -`console.warn`, narrated once per session, not once per delete) rather than risk -decrementing a total that was never incremented for that record in the first -place. This is intentionally rare — it's a defensive fallback for callers on an -unusual removal path, not a per-delete cost. - -`suspect` is not directly exposed on any `Brainy` method today — it lives on the -`StorageAdapter`'s optional `getCanonicalCounts()`, primarily consulted by -`repairIndex()`'s recount step and by custom storage adapters composing their own -`healthReport()`. What matters for an application: a `suspect` ledger is not -incorrect, just *unverified since the last recount* — and `repairIndex()`'s -unconditional count-rollup step (step 2, above) recomputes the ALL scalars from a -real canonical walk on every call, clearing the flag with proof either way. - -## Practical guidance - -- **On a normal restart**, do nothing — `init()` brings every provider to - serving before it returns, or fails loudly. -- **On a `*NotReadyError`** from a live read, reconcile with `repairIndex()` - (report-driven is almost always sufficient) and retry. -- **After an incident** where you distrust a specific family regardless of what - it reports healthy — a storage-layer fault, a suspicious restore — use the - explicit door: `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] })`. -- **To audit before trusting a report**, `brain.auditGraph()` walks every stored - relationship and proves (or disproves) that reads return canonical truth, - independent of what any provider self-reports — see - [Inspecting a Live Brainy](../guides/inspection.md). diff --git a/docs/concepts/storage-adapters.md b/docs/concepts/storage-adapters.md index 82aa01e8..af6d068f 100644 --- a/docs/concepts/storage-adapters.md +++ b/docs/concepts/storage-adapters.md @@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from to call `acquireWriterLock()` at init. ```typescript -import { FileSystemStorage } from '@soulcraftlabs/brainy' +import { FileSystemStorage } from '@soulcraft/brainy' export class MmapFileSystemStorage extends FileSystemStorage { public supportsMultiProcessLocking(): boolean { @@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom network backend), extend `BaseStorage` directly: ```typescript -import { BaseStorage } from '@soulcraftlabs/brainy' +import { BaseStorage } from '@soulcraft/brainy' export class MyCloudStorage extends BaseStorage { // BaseStorage's default no-op implementations of the multi-process @@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`, `hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM import (verify in your plugin's `dist/`: `import { FileSystemStorage } from -'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype +'@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype chain at runtime resolves to whatever Brainy version your consumer has installed. @@ -109,8 +109,8 @@ installed. the prototype chain at the consumer-app level: - **Stale `node_modules`** — a lingering install from before the consumer - upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but - `node_modules/@soulcraftlabs/brainy` is still 7.20.x. + upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but + `node_modules/@soulcraft/brainy` is still 7.20.x. - **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy version older than the package.json range, and `bun install` honors the lockfile. @@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint: methods on its prototype chain. Writer locking and the flush-request RPC are disabled for this directory. Likely fix: clean install (`rm -rf node_modules bun.lockb && bun install`) or rebuild your container image to refresh -`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. +`@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. ``` ## Authoring a new storage adapter — minimum checklist @@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh install time — fix install, not your plugin. 6. **Pin your peer dep generously.** `"peerDependencies": { - "@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin + "@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin to an exact patch unless you're tracking a known regression. ## Future direction @@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change. heartbeat semantics, what the lock protects. - [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the read-only mode. -- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the +- `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the authoritative type signatures for every method this page references. diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index 616c8fc4..e58eb2ce 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -11,18 +11,12 @@ No batch jobs. No scheduled recalculations. Aggregates stay current with every w **Defining over existing data:** if you define an aggregate on a store that already holds matching entities, Brainy backfills it from those entities on the first query (a one-time scan, then purely incremental). So `defineAggregate()` behaves the same whether you define it before -or after the data exists. - -**Reopening a persisted brain:** aggregate state persists across restarts. Re-defining the -same aggregate at boot (the normal declarative pattern) adopts the persisted state directly — -no rescan. A backfill scan runs only when the definition actually changed, when no persisted -state exists, or when the state failed to load; and however many aggregates need backfilling, -they share a single scan. +or after the data exists — including when a persisted brain reopens already populated. ## Quick Start ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/external-backups-and-sparse-storage.md b/docs/guides/external-backups-and-sparse-storage.md deleted file mode 100644 index f28dcfd7..00000000 --- a/docs/guides/external-backups-and-sparse-storage.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: External Backups & Sparse Storage -slug: guides/external-backups -public: true -category: guides -template: guide -order: 10 -description: How to back up a brain directory with external tools (tar, rsync, cp) without exploding sparse files — why a store can show 100+ GB "apparent" size on a small disk, which files are sparse, and how persist()/restore() handle it for you. -next: - - guides/snapshots-and-time-travel - - concepts/storage-adapters ---- - -# External Backups & Sparse Storage - -The built-in snapshot path — [`db.persist()` and `brain.restore()`](/docs/guides/snapshots-and-time-travel) — -already handles everything on this page for you. Read this when you back up a brain directory with -**external tools**: `tar`, `rsync`, `cp`, `scp`, or a filesystem-level backup agent. - -## The one-sentence rule - -> **Always use the sparse-aware flag**: `tar czSf` (capital `S`), `rsync --sparse`, -> `cp --sparse=always`. A naive copy can turn a 2 GB store into a 100+ GB one — or fail -> the disk entirely. - -## Why: some files are sparse - -When a native accelerator plugin is active, parts of the index live in **memory-mapped files** -created at a large fixed virtual size — the file's *apparent* size — while the filesystem only -allocates blocks that were actually written. A brand-new id-mapper file can report tens of -gigabytes in `ls -l` while occupying a few megabytes on disk. - -Check the difference yourself: - -```bash -ls -lh brain-data/_id_mapper/ # APPARENT size (can be huge) -du -sh brain-data/ # ALLOCATED size (the real footprint) -``` - -The sparse candidates in a brain directory: - -| Path | What it is | -|---|---| -| `_id_mapper/` | The native id-mapper's mmap files (large fixed virtual size) | -| `_blobs/` | Native index files (vector base, segments) — may be mmap-backed | - -Everything else (entities, `_system`, `_generations`, `_cas` content blobs) is ordinary dense data. - -## Doing it right - -**tar** — the `S` flag detects holes and stores only real data: - -```bash -tar czSf brain-backup.tgz /data/brain -# restore preserves the holes: -tar xzSf brain-backup.tgz -C /data/ -``` - -**rsync**: - -```bash -rsync -a --sparse /data/brain/ backup-host:/backups/brain/ -``` - -**cp**: - -```bash -cp -a --sparse=always /data/brain /backups/brain -``` - -**What goes wrong without the flag:** the copy *materializes* every hole as real zero bytes. -A store whose apparent size exceeds the target disk fails with `ENOSPC` partway through — and a -copy that *does* fit silently costs the full apparent size in storage and transfer time. - -## What the built-in paths do (so you don't have to) - -- **`db.persist(path)`** snapshots via **hard links** — instant and space-shared, since every data - file is immutable-by-rename. The handful of append-in-place files (the transaction log, the - commit fact log's tail segment) and mmap-mutated directories (`_id_mapper/`) are **byte-copied** - instead, so a post-snapshot write can never reach through a shared inode into your backup. -- **`brain.restore(path, { confirm: true })`** is **non-destructive and sparse-aware**: the snapshot - is copied into a staging area *before* any live data is touched (all-zero blocks stay holes), and - only after the copy fully succeeds does an atomic swap move it into place. A failed copy — - including `ENOSPC` — leaves the live store exactly as it was. A crash mid-swap completes forward - on the next open. - -## Live-store caveats for external tools - -1. **Prefer snapshotting a `persist()` output, not the live directory.** `persist()` produces a - crash-consistent, immutable snapshot; running `tar` against a live, actively-written directory - can capture a torn mid-write state. If you must archive live, stop writes first (or accept that - the archive is only as consistent as the moment's flush state). -2. **Never prune or "clean up" files inside a brain directory.** Index files that look stale or - redundant are load-bearing; the store protects its declared index families from in-process - deletion, but an external `rm` bypasses that fence. If space is the concern, `du -sh` first — - the allocated size is usually far smaller than it looks. -3. **Verify restores by opening them.** `Brainy.load(path)` opens any snapshot or restored - directory read-only — the store verifies its own coherence at open and reports loudly if - anything is missing or torn. diff --git a/docs/guides/find-limits.md b/docs/guides/find-limits.md index 4c7fd252..66418b80 100644 --- a/docs/guides/find-limits.md +++ b/docs/guides/find-limits.md @@ -40,10 +40,6 @@ Brainy picks `maxLimit` from the first of these that's available: Worked example: a 4 GB Cloud Run container picks priority 3 → `floor(4 GB × 0.25 / 25 KB) = floor(40 960) = 40 000` results. A 900 MB free-memory box on priority 4 gets `floor(900 MB / 25 KB) = ~36 000`. -The cap is fixed at construction and never changes at runtime. Query timing is recorded -for diagnostics only — a burst of slow queries cannot silently shrink the cap, and the -auto-detected tiers (3 and 4) never go below a floor of 10 000. - > **Calibration note.** Pre-7.30.2 used 100 KB per result instead of 25 KB, which produced caps that were 4× too tight for typical workloads (an 8 KB / result reality). 7.30.2 recalibrated to match observed entity sizes; existing `limit: 10_000` safety patterns now pass silently on any reasonably-sized box. ## What happens when you exceed the cap diff --git a/docs/guides/framework-integration.md b/docs/guides/framework-integration.md index 8f85da00..984466c5 100644 --- a/docs/guides/framework-integration.md +++ b/docs/guides/framework-integration.md @@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server: -- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'` +- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'` - **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node - **Cleaner code**: No browser polyfills, no conditional client/server imports - **Better DX**: One instance shared across your server routes @@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers ### Install Brainy ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Basic Integration ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Run on the server (API route, server component, backend service) // new Brainy() auto-detects filesystem persistence on Node @@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod ```javascript // lib/brain.server.js -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests: ```javascript // server/brain.js (server-only module) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun): ```typescript // server: api/search -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() // auto-detects filesystem persistence on Node await brain.init() @@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or ```javascript // lib/brain.server.js (imported only by server code) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul ```javascript // src/lib/server/brain.js (server-only — note the .server suffix) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -432,7 +432,7 @@ import { defineConfig } from 'vite' export default defineConfig({ ssr: { - external: ['@soulcraftlabs/brainy'] + external: ['@soulcraft/brainy'] } }) ``` @@ -440,7 +440,7 @@ export default defineConfig({ ```javascript // rollup.config.js (server bundle) export default { - external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto'] + external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto'] } ``` @@ -466,7 +466,7 @@ export async function load({ url }) { ```javascript // For build-time usage (runs in Node during the build) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' export async function generateStaticProps() { const brain = new Brainy({ @@ -513,7 +513,7 @@ export async function generateStaticProps() { ### Issue: Large client bundle size **Cause**: A client module is pulling in Brainy. -**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle. +**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle. ### Issue: SSR hydration mismatch **Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup. diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index ffabe55c..e0cd94ed 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING: ## The Ultimate Simplicity ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -300,10 +300,7 @@ await brain.import(data, { // Deduplication enableDeduplication: true, // Check for duplicate entities (default: true) deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85) - // Notes: false disables BOTH the inline merge and the background pass that - // runs ~5 min after the last import (merged duplicates are deleted). - // The inline pass auto-disables for imports >100 entities (O(n²) cost); - // the background pass still covers those unless the flag is false. + // Note: Auto-disabled for imports >100 entities // Performance chunkSize: 100, // Batch size for processing (default: varies by operation) diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md index 18c3cb9a..66f50713 100644 --- a/docs/guides/import-progress-examples.md +++ b/docs/guides/import-progress-examples.md @@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats** ### Basic Progress Tracking ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import * as fs from 'fs' const brain = await Brainy.create() diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 3bc26dae..5a850a86 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -7,7 +7,7 @@ ## Basic Import ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -86,22 +86,11 @@ await brain.import(file, { ```typescript await brain.import(file, { - enableDeduplication: true, // Check for duplicates (default: true) + enableDeduplication: true, // Check for duplicates (default: false) deduplicationThreshold: 0.85 // Similarity threshold (default: 0.85) }) ``` -Deduplication merges entities judged duplicates — the non-primary records are -**deleted**. Set `enableDeduplication: false` to disable it entirely: the flag -gates both the inline merge during import and the background pass that runs -about 5 minutes after the last import. - -```typescript -await brain.import(file, { - enableDeduplication: false // No merging, inline or background -}) -``` - ### Import Tracking Track and organize imports by project: @@ -187,7 +176,7 @@ await brain.import(file, { ## Complete Example ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import * as fs from 'fs' async function importCatalog() { diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 8560b543..015c5118 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI. ## Programmatic inspection ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } @@ -166,34 +166,6 @@ brainy inspect diff /data/brain-prod /data/brain-staging Sample-based — for a full diff, dump both with `inspect dump` and compare the JSONL. -## Auditing graph-read truth - -`brain.auditGraph()` (8.6.0+) proves — or disproves — that relationship reads -return canonical truth on a given brain, without mutating anything. It walks -every stored relationship record, asks the same read path your application -uses (`related()`, VFS `readdir`) with every visibility tier included, and -classifies every discrepancy: - -```typescript -const report = await brain.auditGraph() - -report.coherent // true = related()/readdir can be trusted on this brain -report.missingFromReadsCount // records the read path omits — stale index -report.danglingEndpointsCount // relationships whose endpoint entity is gone -report.readOnlyCount // read-path edges with NO stored record — ghosts -report.visibilityHiddenCount // internal/system edges hidden by design (not a fault) -``` - -Counts are always exact; the example lists (`missingFromReads`, -`danglingEndpoints`, `readOnlyVerbIds`) are capped at `maxExamples` -(default 100) and `truncatedExamples` says so when they are. - -Run it after any engine upgrade, restore, or migration. If it reports -discrepancies, run `brain.repairIndex()` and audit again — a `coherent` -report after the repair is the verified statement that the heal worked. -Cost: one relationship-record walk plus one indexed read per distinct -source entity — safe on a live brain. - ## Repairing a corrupted store If invariants fail and you suspect index corruption, `inspect repair` diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 20d40ea2..0a36f632 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -21,21 +21,21 @@ next: ## Install ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` Or with your preferred package manager: ```bash -bun add @soulcraftlabs/brainy -yarn add @soulcraftlabs/brainy -pnpm add @soulcraftlabs/brainy +bun add @soulcraft/brainy +yarn add @soulcraft/brainy +pnpm add @soulcraft/brainy ``` ## Verify ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -52,7 +52,7 @@ npm install @soulcraft/cor ``` ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) await brain.init() // native providers registered during init @@ -71,7 +71,7 @@ remains available on npm if you need it. Brainy ships with full TypeScript types. No `@types/` package needed: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/migration-3.36.0.md b/docs/guides/migration-3.36.0.md index 5f00534a..8b1f239e 100644 --- a/docs/guides/migration-3.36.0.md +++ b/docs/guides/migration-3.36.0.md @@ -66,7 +66,7 @@ const results = await brain.search("query") **New diagnostics for capacity planning and performance tuning.** ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')} ### Step 1: Update Package ```bash -npm install @soulcraftlabs/brainy@latest +npm install @soulcraft/brainy@latest ``` ### Step 2: Restart Your Application @@ -134,7 +134,7 @@ npm run start ### Check Adaptive Sizing is Working ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -218,7 +218,7 @@ For debugging or compatibility testing: If you need to rollback to v3.35.0: ```bash -npm install @soulcraftlabs/brainy@3.35.0 +npm install @soulcraft/brainy@3.35.0 ``` **Note:** We don't anticipate any issues, but rollback is straightforward if needed. @@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) { ## Next Steps -1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest` +1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest` 2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements 3. 🎯 **Tune:** Adjust based on recommendations (if needed) 4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning diff --git a/docs/guides/model-loading.md b/docs/guides/model-loading.md index cc1b2b6a..e5b7b1d6 100644 --- a/docs/guides/model-loading.md +++ b/docs/guides/model-loading.md @@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings. ```bash # Bun as a runtime — supported and recommended -bun add @soulcraftlabs/brainy +bun add @soulcraft/brainy bun run server.ts ``` diff --git a/docs/guides/namespace-migration.md b/docs/guides/namespace-migration.md deleted file mode 100644 index f7d2c7f7..00000000 --- a/docs/guides/namespace-migration.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: Migrating to 9.0 — your fields and system fields -slug: guides/namespace-migration -public: true -category: guides -template: guide -order: 1 -description: The simple story of the 9.0 field-addressing change and the mechanical checklist for updating your call sites — every miss fails loudly with the fix in the error. -next: - - concepts/field-addressing ---- - -# Migrating to 9.0 — your fields and system fields - -The one-sentence version: **your data's field names are now completely -yours, the engine's own fields all live behind one `system.` prefix, and -nothing in between can silently go wrong anymore.** - -## What changed, simply - -**1. Any field name just works.** Before 9.0 the engine quietly owned -certain names. A field called `level` could be shadowed by the engine's -internal index layer of the same name (sorts silently returned insertion -order); names like `confidence` or `subtype` were rejected inside -`metadata`; names like `content` or `id` were silently never indexed, so -filtering on them returned nothing. All of that is gone. Any name — -`level`, `confidence`, `type`, `id`, `content`, anything — is stored -exactly as written and works with every feature: filtering, sorting, -grouping, aggregation, search, and time-travel reads. - -**2. The engine's fields moved behind `system.`.** The engine still keeps -its own per-record bookkeeping — creation time, type, confidence, and so -on. Those are reached one way only now: spelled out, e.g. -`system.createdAt`, `system.type`. They are just as queryable and sortable -as before. `orderBy: 'createdAt'` means *your* field named `createdAt`; -`orderBy: 'system.createdAt'` means the engine's timestamp. No guessing, -no priority rules. - -**3. Storage keeps the two physically separate.** New records store your -metadata in its own nested compartment, so a user field named -`confidence` and the engine's confidence live side by side, both intact, -through restarts, index rebuilds, and `asOf()` history. Old records stay -readable forever; nothing rewrites your data. - -**4. Mistakes are loud.** An ambiguous or unknown field name is a typed -error naming the fix. Unimplemented options refuse instead of being -ignored. The only forbidden name in your metadata is one literally -starting with `system.`. - -## The mechanical checklist - -Every missed site fails **loudly** with the correction in the error -message — nothing silently changes meaning. Sweep these patterns: - -| Before (8.x) | After (9.0) | -|---|---| -| `orderBy: 'createdAt'` (meaning the engine timestamp) | `orderBy: 'system.createdAt'` | -| `where: { subtype: 'invoice' }` (the engine subtype) | `where: { 'system.subtype': 'invoice' }` | -| `where: { confidence: { greaterThan: 0.8 } }` (the engine scalar) | `where: { 'system.confidence': { greaterThan: 0.8 } }` | -| `groupBy: ['noun']` or `groupBy: ['type']` | `groupBy: ['system.type']` | -| `where: { visibility: 'internal' }` / `{ service: … }` (engine values) | `'system.visibility'` / `'system.service'` | -| `metadata: { confidence: 0.9 }` expecting a throw or a lift to the engine scalar | it is YOUR field now — set the engine scalar via the `confidence` param | -| `new Brainy({ reservedFieldPolicy: … })` | remove the option (it throws with this note) | -| `find({ cursor })` / `includeRelations` / `writeOnly` | refuse with `UnsupportedFindOptionError` — they were silently ignored before | - -If a bare name in a query was genuinely *your* field all along (`orderBy: -'score'`, `where: { status: 'active' }`), **change nothing** — bare names -mean your fields, always. - -## What happens at first open - -Each existing database rebuilds its derived indexes once, automatically, -at the first open on 9.0 (index epoch 3 — the index keys split the two -namespaces). One-time cost, observable via `getIndexStatus()`; no manual -step, and your stored data is not modified. - -## For tooling and raw-record readers - -If you read raw stored records (fact-log scanners, export tooling), use -the exported shape-aware splitters — they handle both record eras: - -```typescript -import { splitNounMetadataRecord } from '@soulcraftlabs/brainy' -const { reserved, custom } = splitNounMetadataRecord(rawRecord) -// reserved = engine fields · custom = the user's bag, ANY names -``` - -Feature detection (never version-sniff): - -```typescript -import * as brainy from '@soulcraftlabs/brainy' -const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1' -``` - -## Where to go next - -- [Field addressing](../concepts/field-addressing.md) — the full contract: - the ten system scalars, the relation mirror, refusal semantics, and the - cross-engine ordering guarantees. diff --git a/docs/guides/nextjs-integration.md b/docs/guides/nextjs-integration.md index 25d6062d..ab55e51f 100644 --- a/docs/guides/nextjs-integration.md +++ b/docs/guides/nextjs-integration.md @@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou ```bash npx create-next-app@latest my-brainy-app cd my-brainy-app -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Basic Setup @@ -18,7 +18,7 @@ npm install @soulcraftlabs/brainy // app/components/BrainyProvider.jsx 'use client' import { createContext, useContext, useEffect, useState } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const BrainyContext = createContext() @@ -271,7 +271,7 @@ export default function SearchPage() { ```javascript // app/api/search/route.js (App Router) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -332,7 +332,7 @@ export async function GET() { ```javascript // pages/api/search.js (Pages Router) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -374,7 +374,7 @@ export default async function handler(req, res) { ```javascript // app/api/data/route.js -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -418,7 +418,7 @@ export async function POST(request) { ```jsx // app/actions/brainy.js 'use server' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -630,7 +630,7 @@ CMD ["npm", "start"] /** @type {import('next').NextConfig} */ const nextConfig = { experimental: { - serverComponentsExternalPackages: ['@soulcraftlabs/brainy'] + serverComponentsExternalPackages: ['@soulcraft/brainy'] }, webpack: (config, { isServer }) => { if (!isServer) { @@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) { // app/contexts/BrainyContext.jsx 'use client' import { createContext, useContext, useReducer, useEffect } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const BrainyContext = createContext() @@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider' import { Search } from '../app/components/Search' // Mock Brainy -jest.mock('@soulcraftlabs/brainy', () => ({ +jest.mock('@soulcraft/brainy', () => ({ Brainy: jest.fn().mockImplementation(() => ({ init: jest.fn().mockResolvedValue(undefined), find: jest.fn().mockResolvedValue([ diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index 2984998b..a21a8af0 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin Every distributed-job scheduler eventually wants this exact loop: ```ts -import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy' +import { Brainy, RevisionConflictError } from '@soulcraft/brainy' const LOCK_ID = '...uuid for this job slot...' @@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved: ```ts -import { GenerationConflictError } from '@soulcraftlabs/brainy' +import { GenerationConflictError } from '@soulcraft/brainy' async function addIfMissingByEmail(email: string, data: string) { for (let attempt = 0; attempt < 5; attempt++) { @@ -180,35 +180,3 @@ Brainy 8.0 has exactly two write-coordination counters, at two granularities: They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md). A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from. - -## The transact envelope: batch size, budget, and bulk imports - -`transact()` applies its batch atomically under one commit — which means the whole batch -shares one **apply budget**. Since 8.7.0 the budget scales with the batch: -`max(30 s, opCount × 2 s)`, or exactly what you pass as `timeoutMs`. A tripped budget rolls -the entire batch back (nothing partial survives) and throws a retryable -`TransactionTimeoutError` that names the operation it stopped at, the batch size, and the -elapsed vs budgeted time — a diagnosis, not just a failure: - -``` -Transaction timed out at operation 41/120 ('add') — 246012ms elapsed, budget 240000ms. -The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch. -``` - -Practical envelope guidance for bulk work: - -1. **Precompute embeddings outside the commit path.** Embedding inside `transact()` spends - the budget on model inference. Use `brain.embedBatch(texts)` and pass each vector via - the op's `vector` field — the commit then pays only storage costs, and a retried batch - never re-pays inference. (The win is *where* the inference happens, not raw embedding - throughput: on the default WASM engine, batch and sequential embedding measure - comparably, ~160 ms/text; native embedding providers may batch faster.) -2. **Chunk very large imports** into batches of a few hundred ops with one `transact()` - each. You lose whole-import atomicity but keep per-chunk atomicity, bounded memory, and - resumability — pair with `ifAbsent` upserts so a retried chunk is idempotent. -3. **Slow disks change the math, not the contract.** On network-attached storage a single - op can cost ~2 s (canonical write + fsync + index maintenance). The scaled default - absorbs that; pass an explicit `timeoutMs` only when you know better than the scale. -4. **`addMany`/`relateMany` are the convenience tier** — they chunk and batch-embed for - you, with per-item error reporting instead of batch atomicity. Choose by what you need: - atomic-all-or-nothing → `transact()`; resilient bulk load → `addMany`. diff --git a/docs/guides/quick-start.md b/docs/guides/quick-start.md index d9a4e896..097c55fe 100644 --- a/docs/guides/quick-start.md +++ b/docs/guides/quick-start.md @@ -18,13 +18,13 @@ Get Brainy running in under a minute. ## 1. Install ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ## 2. Initialize ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -67,7 +67,7 @@ await brain.relate({ ## 5. Query with Triple Intelligence ```typescript -import type { Result } from '@soulcraftlabs/brainy' +import type { Result } from '@soulcraft/brainy' // All three search paradigms in one call const results: Result[] = await brain.find({ diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md index 490aecab..6f7b150a 100644 --- a/docs/guides/snapshots-and-time-travel.md +++ b/docs/guides/snapshots-and-time-travel.md @@ -9,7 +9,6 @@ description: Recipes for the Db API — instant backups with persist(), restore, next: - concepts/consistency-model - guides/optimistic-concurrency - - guides/external-backups --- # Snapshots & Time Travel @@ -42,10 +41,6 @@ bytes. Cross-device targets fall back to per-file byte copies, and persisting an in-memory brain serializes it to the same directory layout — a real, durable store. -> Archiving a brain directory with **external tools** (`tar`, `rsync`, `cp`)? -> Some index files are sparse and can explode to their apparent size under a -> naive copy — see [External Backups & Sparse Storage](/docs/guides/external-backups). - Two things to know: - `persist()` requires the view to still be the store's **latest** @@ -344,12 +339,8 @@ For per-entity write coordination (rather than whole-store history), the ## Keeping history bounded Under Model-B every write is a generation, so history can grow quickly — -Brainy auto-compacts at `close()` (time-bounded per pass) under the -**`retention`** knob (configured on the constructor). Since 8.9.0, `flush()` -never compacts: flushing is durability work and costs only what the current -window's writes cost, regardless of history backlog. A long-lived writer that -never closes keeps its history until its next explicit `compactHistory()` — -schedule one in your maintenance window if you run bounded retention: +Brainy auto-compacts on every `flush()`/`close()` under the **`retention`** +knob (configured on the constructor): ```typescript // Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows, @@ -363,13 +354,10 @@ new Brainy({ retention: 'all' }) new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } }) ``` -Reclaim manually at any time (the same caps, plus an optional per-pass time -budget for maintenance windows — an early stop is a consistent prefix and the -next pass resumes): +Reclaim manually at any time (the same caps): ```typescript await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 }) -await brain.compactHistory({ maxBytes: 512 * 1024 ** 2, timeBudgetMs: 10_000 }) ``` Compaction never breaks a pinned read — record-sets are reclaimed only when diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md index 27dabe75..9f2e2e5b 100644 --- a/docs/guides/standard-import-progress.md +++ b/docs/guides/standard-import-progress.md @@ -11,7 +11,7 @@ ### One Interface for Everything ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = await Brainy.create() @@ -78,7 +78,7 @@ interface ImportProgress { ```typescript import { useState } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' function UniversalImportProgress({ file }: { file: File }) { const [progress, setProgress] = useState({ @@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) { ```typescript import ora from 'ora' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' async function importWithProgress(filePath: string) { const spinner = ora('Starting import...').start() diff --git a/docs/guides/storage-adapters.md b/docs/guides/storage-adapters.md index a4224bc8..06ec9f3a 100644 --- a/docs/guides/storage-adapters.md +++ b/docs/guides/storage-adapters.md @@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map). ## Quick start ```ts -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Filesystem (recommended for any persistent workload): const brain = new Brainy({ @@ -134,7 +134,7 @@ config; the `type` is optional. If you want to skip the factory: ```ts -import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy' +import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy' const fsStorage = new FileSystemStorage('./brainy-data') const memStorage = new MemoryStorage() diff --git a/docs/guides/subtypes-and-facets.md b/docs/guides/subtypes-and-facets.md index 74311528..ff5de320 100644 --- a/docs/guides/subtypes-and-facets.md +++ b/docs/guides/subtypes-and-facets.md @@ -34,7 +34,7 @@ Three layers solve this: ### Write ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -240,7 +240,7 @@ await brain.migrateField({ A realistic adoption sequence for a brain that started without these primitives: ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } }) await brain.init() diff --git a/docs/guides/upgrading-7-to-8.md b/docs/guides/upgrading-7-to-8.md index 53aa2a5c..a3c64fb9 100644 --- a/docs/guides/upgrading-7-to-8.md +++ b/docs/guides/upgrading-7-to-8.md @@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you. ## TL;DR -- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.** +- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.** If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, with no operator action. - Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. @@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and open it.** ```ts -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Opening the store is all that is required — recovery runs during init(). const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) @@ -182,5 +182,5 @@ and opening each store is sufficient. The recovery is copy-only, so no rollback of the recovery itself is ever needed. If you need to roll back the **whole** 7→8 upgrade, restore the directory from your pre-upgrade backup (retained automatically while recovery is incomplete, or -your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old +your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old branch layout in place, so a directory-level restore is the rollback path. diff --git a/docs/guides/vue-integration.md b/docs/guides/vue-integration.md index 34d18ebf..7f7c6a06 100644 --- a/docs/guides/vue-integration.md +++ b/docs/guides/vue-integration.md @@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N npm create vue@latest my-brainy-app cd my-brainy-app npm install -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Basic Setup @@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun ```javascript // server/utils/brain.js (server-only — Nitro never bundles this into the client) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue' export default defineConfig({ plugins: [vue()], ssr: { - external: ['@soulcraftlabs/brainy'] + external: ['@soulcraft/brainy'] } }) ``` diff --git a/docs/neural-extraction.md b/docs/neural-extraction.md index cfb6d764..989b1b60 100644 --- a/docs/neural-extraction.md +++ b/docs/neural-extraction.md @@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c ### Method 1: Brain Instance (Recommended) ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', { import { SmartExtractor, SmartRelationshipExtractor -} from '@soulcraftlabs/brainy' +} from '@soulcraft/brainy' // Or use subpath imports: -import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor' +import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor' const brain = new Brainy() await brain.init() @@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, { **Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration. ```typescript -import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy' +import { SmartExtractor, FormatContext } from '@soulcraft/brainy' const extractor = new SmartExtractor(brain, { minConfidence: 0.7, // Threshold @@ -229,7 +229,7 @@ interface ExtractionResult { **Relationship type classifier.** Determines verb/relationship types between entities. ```typescript -import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' +import { SmartRelationshipExtractor } from '@soulcraft/brainy' const relExtractor = new SmartRelationshipExtractor(brain, { minConfidence: 0.6, @@ -286,7 +286,7 @@ const rel = await relExtractor.infer( **Full extraction orchestrator.** Handles candidate detection, classification, and deduplication. ```typescript -import { NeuralEntityExtractor } from '@soulcraftlabs/brainy' +import { NeuralEntityExtractor } from '@soulcraft/brainy' const extractor = new NeuralEntityExtractor(brain) @@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location) ### Example 2: Excel Data Classification ```typescript -import { SmartExtractor } from '@soulcraftlabs/brainy' +import { SmartExtractor } from '@soulcraft/brainy' const extractor = new SmartExtractor(brain) @@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) { ### Example 3: Relationship Extraction ```typescript -import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' +import { SmartRelationshipExtractor } from '@soulcraft/brainy' const relExtractor = new SmartRelationshipExtractor(brain) diff --git a/docs/path-registry.md b/docs/path-registry.md deleted file mode 100644 index 8a55004c..00000000 --- a/docs/path-registry.md +++ /dev/null @@ -1,86 +0,0 @@ -# The Path Registry — brainy's twin table - -The brainy half of the cross-engine Path Registry (the native accelerator -maintains the master list; IDs are shared and stable — `LC3`, `DP7`, … are -citable in commits, board rounds, release notes, and pins). Every row owes -five things: **service class** (INDEX-SERVED | BOUNDED-FALLBACK, announced | -TYPED REFUSAL), **latency budget** at 1k/10k/100k/1M (design bar: billions), -**lifecycle behavior**, **failure narration**, and a **test pin**. A path not -in this registry does not ship; an unregistered path is a red gate in the -scan audit. - -**The availability bar governing every row: user-visible downtime is -seconds, at restart only.** Migration, heal, compaction, embedding, and -retention run behind the doors — yielding, budget-capped, narrated. No path -may hold the doors while it does housekeeping. - -Status legend: ✅ contracted + pinned (test cited) · 🟡 partial (what holds -and what's missing, stated) · 🔴 owed (named, never silent). - -## LC — Lifecycle - -| ID | Brainy row | Status | -|----|-----------|--------| -| LC1 | Same-version reopen adopts everything: brain-format epoch match → zero rebuilds; aggregation state adopts by stamp; persisted indexes load. | ✅ `tests/unit/brainy/brain-format-handshake` + `migration-deference` (no-drift reopen never rebuilds) | -| LC2 | New empty brain: doors immediate. | ✅ exercised by every suite's setup | -| LC3 | Upgrade, same epoch: as LC1 — new code on unchanged formats owes nothing at open. | ✅ same pins as LC1 (epoch equality is the gate) | -| LC4 | Upgrade with epoch migration: TODAY brainy's epoch rebuild runs at open before doors. | 🔴 **owed — the sev's lockout row.** The doors-open-serving-old-structures design (yielding installments + atomic swap) lands measured-and-gated behind the service-class pair, per the lifecycle-sprint choreography. Acceptance case: the 9,184-row hours-lockout. | -| LC5 | Crash recovery: bounded, resumable, narrated. Aggregation leg ✅ (behind-stamp → incremental catch-up off the fact log + time-travel reconciliation, capped at 5,000 affected before an ANNOUNCED rescan). Vector/metadata legs ride epoch machinery (rebuild-from-canonical, narrated). | 🟡 aggregation pinned (`tests/integration/aggregation-lifecycle-catchup`); the rebuild legs are narrated but not yet installment-yielding (couples to LC4) | -| LC6 | Shutdown under load: close() drains the background flush flight, tears down cadence timers, runs ONE time-bounded compaction pass (~5s budget, resumable). | 🟡 pinned for flush/compaction (8.9.0 suites); SIGTERM drain budget not yet declared | -| LC7 | Rollback/downgrade: an N−1 build opening an N brain. | 🔴 owed — no declared read-compat window or typed refusal today (epoch mismatch triggers a rebuild, not a refusal; v2 nested-bag records read as a phantom user field on pre-law builds). Needs the declared-window contract. | -| LC8 | Relocatable brain directory: no absolute paths in artifacts; persist()/load() round-trips. | 🟡 persist/load pinned; byte-for-byte relocation depot cases are the pair gate's (shared corpora) | -| LC9 | Double-open: second writer gets a typed lock refusal (PID-liveness + heartbeat stale detection; `force` escape hatch logs loudly). | ✅ writer-lock suites (8.7.1) | - -## DP — Data plane - -| ID | Brainy row | Status | -|----|-----------|--------| -| DP1 | `get()` by id: direct storage read + hydrate. INDEX-SERVED (id-mapped). Milliseconds at every scale. | ✅ exercised everywhere; budget rides the pair speed table | -| DP2 | `find({query})`: embed + vector search. The embed dominates (native side owns the budget); JS HNSW serves the search leg. | 🟡 300ms-class p95 is the pair speed-table row; brainy-alone budget declared there | -| DP3 | Filtered/sorted list: column top-K when the field is columnized (INDEX-SERVED, zero canonical reads on the sorted page — value pairs come from ONE batched metadata-record pass); no-column fallback is BOUNDED-ANNOUNCED (one batch pass, announces once per field past 500 rows); unknown field → TYPED REFUSAL naming both candidate spellings. | ✅ `tests/unit/utils/metadataIndex-sort-callshape` (zero per-row reads, batch-only — latency-blind) + `metadataIndex-nested-orderby` (dotted keys serve-or-refuse) + `tests/integration/orderby-sort-bug` | -| DP4 | Aggregation/stats: ALWAYS answers. Write-time incremental; behind-stamp reconciles incrementally; genuine rebuilds go through the native parallel door or the paged JS walk; nothing ever latches off; before-image-less deletes flag a LOUD rescan, never a silent skip. | ✅ `tests/integration/aggregation-lifecycle-catchup` + `tests/unit/aggregation/aggregation-provider-rebuild` | -| DP5 | Graph traversal: `related()` paged via adjacency; whole-graph analytics carry declared cost. | 🟡 paged reads pinned; analytics cost-class declaration owed (rides VENUE-GRAPH-TRUST audit tool) | -| DP6 | Single write: ack at the canonical commit; visibility committed at ack (the atomic vector update kills the remove→add dark window); maintenance NEVER holds the ack (background flush cadence — THE ACK LAW pins: a hung flush cannot block a write, a hung EMBEDDER cannot block a write). | ✅ `tests/unit/brainy/persistence-policy` + `tests/unit/hnsw/update-item-atomic` + `tests/integration/deferred-embedding` | -| DP7 | Bulk ingest: sustained rate holds flat — per-write maintenance taxes must not grow with brain size (A4 removed caller-flush convoys; deferred embedding removes the per-write embed tax where opted). | 🟡 the decay-curve row is a pair speed-table RED GATE; brainy-alone sustained-rate run rides the same corpora | -| DP8 | Read under write pressure: no flicker window — a row that exists is never invisible to recall, even transiently (same-vector re-index is a no-op; changed-vector swaps in place, node never leaves the index; deferred updates serve the OLD vector until the atomic swap — stale-beats-absent). | ✅ brainy leg pinned (`tests/unit/hnsw/update-item-atomic` 9/9 + `deferred-embedding` stale-beats-absent); the symmetry property suite + runtime sentinels remain the B4 program | -| — | **As-of semantic recall** (time-travel vector search): `asOf(G).find()` serves the vectors AS THEY STOOD at G — byte-exact past vectors, tombstone masking, the deferred-embed cell honest on the vector leg, TYPED refusal beyond the head. Brainy-alone leg = ephemeral at-generation materialization (documented O(n log n at G) build, bounded); the at-scale leg rides the accelerated provider's as-of index. | ✅ `tests/integration/asof-semantic-recall` 4/4 (registry ID pending the master table's mint) | -| — | **The lazy-open gate honors EVERY provider's not-ready report** (a not-ready metadata provider can no longer latch the silent-empty state under `disableAutoRebuild`). | ✅ `tests/unit/brainy/lazy-notready-honor` | - -## MT — Maintenance (never in the door path) - -| ID | Brainy row | Status | -|----|-----------|--------| -| MT1 | Flush/checkpoint: ENGINE-OWNED cadence (write-count/interval/idle triggers, single-flight, background, loud on failure; callers never flush in hot paths; `flush()` stays as an awaitable barrier). | ✅ `tests/unit/brainy/persistence-policy` | -| MT2 | Compaction: never on flush (durability-only law, 8.9.0); close-time pass time-budgeted + resumable; explicit `compactHistory({timeBudgetMs})`. | ✅ 8.9.0 suites | -| MT3 | Index upkeep (mapper folds, delta promotion): native-side machinery; brainy's JS legs are small and synchronous-cheap. | 🟡 declared; yield audit rides the pair | -| MT4 | Heal/rebuild walks (`repairIndex`, backfill walks): paged; failure latches with cooldown; NOT yet yield-to-foreground installments. | 🔴 owed — the priority-isolation clause (couples to LC4; same choreography) | -| MT5 | Deferred embedding worker: ack at durability, durable pending markers (written BEFORE the commit — orphan-safe), crash-recovered at open via a bounded prefix listing, single-flight, 60s hang guard, `awaitPendingEmbeds()` barrier + `pendingEmbeds` gauge. VFS write paths adopt it end-to-end. | ✅ `tests/integration/deferred-embedding` 5/5 | -| MT6 | Retention/archival walks: retention `'all'` does nothing by design; bounded-retention reclaim is close-time/explicit only. | 🟡 8.9.0 behavior pinned; archival profile is the co-frozen D1+D3 unit | - -## FM — Failure modes - -| ID | Brainy row | Status | -|----|-----------|--------| -| FM1 | Disk full / IO error mid-op: transaction rollback + typed error; failed rollback → StoreInconsistentError quarantines writes until repairIndex(). | 🟡 rollback paths pinned; explicit disk-full depot case owed | -| FM2 | Memory pressure: query limits + reserved-memory config; unified cache eviction. | 🟡 declared budgets; cascade pin owed | -| FM3 | Torn/corrupt file on open: malformed brain-format marker → safe rebuild (never trusting a bad epoch); corrupt records surface loudly. | 🟡 marker pin ✅ (`brain-format-handshake`); broader quarantine is native-side | -| FM4 | Native module unavailable: plugin load failure is LOUD (version-coupling law throws on range mismatch — never silently version-drifted); JS engine serves with its own declared budgets, named as the active backend in op names. | ✅ `tests/unit/plugin-version-coupling` + op-name stamping | - -## FL — Fleet - -| ID | Brainy row | Status | -|----|-----------|--------| -| FL1 | Cold open on demand: LC1's adopt-everything open; warm() available for eager paths. | 🟡 open cost pinned at LC1; millisecond budget rides the speed table | -| FL2–FL4 | Boot storm / upgrade wave / isolation: fleet-layer policies over LC1/LC4 — engine leg = budgeted opens + LC4's behind-doors migration. | 🔴 owed with LC4 | -| FL5 | Brain as product object: create instant (LC2) · erase = `clear()` explicit + complete · export = portable-graph, canon-complete mode available. | ✅ clear-persistence + portable-graph + canonical-enumeration suites | - -## Status summary - -Contracted + pinned this train: **DP3, DP4, DP6, DP8(brainy leg), MT1, -MT5, LC5(aggregation), the lazy-open not-ready gate, LC1/LC3/LC9, FM4, -FL5** — each with the cited test. Owed, in production-risk order, all -coupled to the priority-isolation program the lifecycle sev opened: **LC4 -(doors-open migration), MT4 (yielding heals), LC7 (downgrade contract), -LC6 (SIGTERM budget), FL2–FL4, FM1/FM2 depot cases, B4 symmetry suite + -sentinels.** Rows move from owed to contracted only with a cited test — -none lands by prose. diff --git a/docs/performance-envelopes.md b/docs/performance-envelopes.md deleted file mode 100644 index d29677e3..00000000 --- a/docs/performance-envelopes.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: Performance Envelopes -slug: guides/performance-envelopes -public: true -category: guides -template: guide -order: 40 -description: Measured per-operation latency envelopes at stated scales — what to expect, on what hardware, and exactly how each number was produced. -next: - - guides/find-limits ---- - -# Performance Envelopes - -Every number on this page is **measured, never projected** — produced by the script -cited at the bottom, against the built package (the artifact you install), on the stated -hardware. Each entry says what was measured, at what scale, on which storage backend. -When a release touches a measured path, that operation is re-measured and this page -updates in the same release. - -Two scopes to keep straight: - -- **These envelopes are the pure-JS engine** (no native accelerator registered) on - filesystem storage. This is the floor every deployment gets from `npm install` alone. -- **Accelerated deployments** (the optional native provider) publish their own numbers — - this page never claims them. - -## Read operations - -Reads are where the architecture pays off: after the write path has done its indexing -work, queries answer from purpose-built indexes without scanning. - -| Operation | 1,000 entities | 10,000 entities | Notes | -|---|---|---|---| -| `get(id)` (warm) | p50 < 0.1ms | p50 < 0.1ms | served from cache/metadata index | -| `find` (metadata: indexed equality + range, limit 100) | p50 1.0ms · p95 1.8ms | p50 7.0ms · p95 8.9ms | column-store bitmap paths | -| `related(id)` (per-node adjacency) | p50 < 0.1ms · p95 0.2ms | p50 < 0.1ms | LSM adjacency index — O(degree), scale-independent | -| `find` (semantic: embed + HNSW, 1k docs) | p50 178ms · p95 393ms | — | dominated by WASM query embedding (measured on a machine under concurrent load — treat the p95 as an upper bound); the vector search itself is single-digit ms | - -## Write operations - -Under Model-B **every write is its own durable generation** — a single-op `add` pays -serialization, before-image staging, and fsync before it acks. That durability is priced -into the write path visibly, by design: - -| Operation | 1,000 entities | 10,000 entities | Notes | -|---|---|---|---| -| `add` (single-op) | p50 167ms · p95 171ms | p50 165ms · p95 172ms | full durable generation per write — flat across scale | -| `addMany` (bulk) | ~163ms/entity | ~187ms/entity | **currently per-item commits** — see the honest note below | -| `relateMany` | ~0.8ms/edge | ~0.9ms/edge | edges batch efficiently today | -| `flush` (steady-state, 1 pending write) | p50 8ms · p95 10ms | p50 45ms · p95 52ms | durability-only since 8.9.0 — cost no longer depends on history backlog or retention mode | - -**The honest note on bulk writes:** `addMany` today commits each item as its own -generation (the same durability as single-op `add`, serialized by the single-writer -lock), so bulk-load cost is N × single-op cost. Batched chunk commits (one generation -and one fsync window per chunk, as `removeMany` already does) are designed into the -unified-commit work on the current roadmap. Until that ships, size bulk imports -accordingly — 10k entities is minutes, not seconds, on filesystem storage. - -## Open / close - -| Operation | 1,000 entities | 10,000 entities | Notes | -|---|---|---|---| -| `open` (empty store) | ~560ms | ~190ms | includes embedder initialization | -| `open` (warm, populated, clean shutdown) | 763ms | 4.9s | pure-JS vector index load dominates and grows with entity count; the native accelerator exists precisely to remove this | -| `close` | bounded | bounded | auto-compaction pass is time-bounded (~5s max) since 8.9.0 | - -A store that was NOT cleanly closed pays index rebuilds on top of the warm-open -number (tens of seconds at 10k) — clean shutdown is worth engineering for. - -## How these were produced - -- **Hardware**: Intel Core i9-14900HX (32 threads), 62GB RAM, NVMe, Linux, Node v22. -- **Backend**: `storage: { type: 'filesystem' }`, pure JS (no native providers). -- **Embeddings**: deterministic stub for non-semantic ops (isolates engine cost); - the real WASM embedder for the semantic row (that's what you'll run). -- **Method**: p50/p95 over 50–200 samples per op against the built `dist/`; - the measuring script ships in the repo history and re-runs per release. - -Numbers on different hardware will differ; the *shape* (sub-2ms indexed reads, -~160ms embedding-bound semantic queries, durability-priced writes) is the envelope -you should hold your deployment against. If your measurements diverge from these -shapes by an order of magnitude, something is wrong — file it. diff --git a/docs/transactions.md b/docs/transactions.md index cbea39c0..fce7d10e 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing }) ### Basic Add Operation ```typescript -import { Brainy } from '@soulcraftlabs/brainy' -import { NounType } from '@soulcraftlabs/brainy/types' +import { Brainy } from '@soulcraft/brainy' +import { NounType } from '@soulcraft/brainy/types' const brain = new Brainy() await brain.init() @@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked ```typescript import { describe, it, expect } from 'vitest' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' describe('Transaction Tests', () => { it('should rollback on failure', async () => { diff --git a/docs/universal-display-augmentation.md b/docs/universal-display-augmentation.md index 464b91fb..da42874c 100644 --- a/docs/universal-display-augmentation.md +++ b/docs/universal-display-augmentation.md @@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati ### Basic Usage ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brainy = new Brainy() await brainy.init() diff --git a/docs/vfs/PROJECTION_STRATEGY_API.md b/docs/vfs/PROJECTION_STRATEGY_API.md index f1319d5b..380862e1 100644 --- a/docs/vfs/PROJECTION_STRATEGY_API.md +++ b/docs/vfs/PROJECTION_STRATEGY_API.md @@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low): ### Step 1: Create the Strategy Class ```typescript -import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' -import { Brainy } from '@soulcraftlabs/brainy' -import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs' +import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' +import { Brainy } from '@soulcraft/brainy' +import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs' export class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' @@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy { ### Step 2: Register the Strategy ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import { PriorityProjection } from './PriorityProjection' const brain = new Brainy() @@ -537,7 +537,7 @@ Use the projection's resolve cache: ```typescript import { describe, it, expect, beforeAll } from 'vitest' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import { PriorityProjection } from './PriorityProjection' describe('PriorityProjection', () => { @@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) { 3. Use appropriate limits: Don't fetch more than needed ### Type errors -1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'` +1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'` 2. Use `as VFSEntity` when mapping results 3. Check BaseProjectionStrategy import diff --git a/docs/vfs/QUICK_START.md b/docs/vfs/QUICK_START.md index 4a1f83dc..8b0efce6 100644 --- a/docs/vfs/QUICK_START.md +++ b/docs/vfs/QUICK_START.md @@ -14,11 +14,11 @@ A file explorer that: ## ⚡ Step 1: Basic Setup (1 minute) ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // ✅ CORRECT: Use filesystem storage for production const brain = new Brainy({ @@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns: ```tsx import React, { useState, useEffect } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' export function FileExplorer() { const [brain, setBrain] = useState(null) @@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next: ### "Module not found" errors ```bash # Make sure you're using the right import -npm ls @soulcraftlabs/brainy # Check version -npm install @soulcraftlabs/brainy@latest # Update if needed +npm ls @soulcraft/brainy # Check version +npm install @soulcraft/brainy@latest # Update if needed ``` ### "VFS not initialized" errors diff --git a/docs/vfs/README.md b/docs/vfs/README.md index a94910c9..b95f0d7b 100644 --- a/docs/vfs/README.md +++ b/docs/vfs/README.md @@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne ## Quick Start ```javascript -import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs' +import { VirtualFileSystem } from '@soulcraft/brainy/vfs' // Initialize the VFS const vfs = new VirtualFileSystem({ @@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system: ## Installation ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ## Requirements diff --git a/docs/vfs/ROADMAP.md b/docs/vfs/ROADMAP.md index c8d15cd2..93c5b901 100644 --- a/docs/vfs/ROADMAP.md +++ b/docs/vfs/ROADMAP.md @@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows. ```typescript // Planned (research phase) -import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse' +import { mountVFS } from '@soulcraft/brainy/vfs/fuse' await mountVFS(vfs, { mountPoint: '/mnt/brainy', @@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested ### Express.js Static Middleware ```typescript // Wanted: Community contribution -import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express' +import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express' app.use('/files', createStaticMiddleware(vfs, { index: ['index.html', 'index.md'], @@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, { ### VSCode Extension ```typescript // Wanted: Community contribution -import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode' +import { VFSProvider } from '@soulcraft/brainy/vfs/vscode' const provider = new VFSProvider(vfs) vscode.workspace.registerFileSystemProvider('brainy', provider) diff --git a/docs/vfs/SEMANTIC_VFS.md b/docs/vfs/SEMANTIC_VFS.md index f34ee9ae..9298c822 100644 --- a/docs/vfs/SEMANTIC_VFS.md +++ b/docs/vfs/SEMANTIC_VFS.md @@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true Create your own semantic dimensions: ```typescript -import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' +import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' diff --git a/docs/vfs/VFS_API_GUIDE.md b/docs/vfs/VFS_API_GUIDE.md index 5dcaaeb8..e0c6a94c 100644 --- a/docs/vfs/VFS_API_GUIDE.md +++ b/docs/vfs/VFS_API_GUIDE.md @@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha ## Quick Start ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Initialize Brainy const brain = new Brainy({ @@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123') VFS uses standard POSIX-style errors: ```typescript -import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy' +import { VFSError, VFSErrorCode } from '@soulcraft/brainy' try { await vfs.readFile('/nonexistent.txt') diff --git a/docs/vfs/VFS_CORE.md b/docs/vfs/VFS_CORE.md index c1d502c0..1eeaf9f8 100644 --- a/docs/vfs/VFS_CORE.md +++ b/docs/vfs/VFS_CORE.md @@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities: #### GitBridge Usage ```javascript // Import and instantiate GitBridge -import { GitBridge } from '@soulcraftlabs/brainy' +import { GitBridge } from '@soulcraft/brainy' const gitBridge = new GitBridge(vfs, brain) // Export VFS to Git repository structure @@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare ## Complete Example ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' async function vfsExample() { // Initialize diff --git a/docs/vfs/VFS_GRAPH_TYPES.md b/docs/vfs/VFS_GRAPH_TYPES.md index 478bef7f..3c1f30f0 100644 --- a/docs/vfs/VFS_GRAPH_TYPES.md +++ b/docs/vfs/VFS_GRAPH_TYPES.md @@ -196,5 +196,5 @@ await brain.relate({ Always import and use the type enums: ```javascript -import { NounType, VerbType } from '@soulcraftlabs/brainy' +import { NounType, VerbType } from '@soulcraft/brainy' ``` \ No newline at end of file diff --git a/docs/vfs/VFS_INITIALIZATION.md b/docs/vfs/VFS_INITIALIZATION.md index fd12fc71..97e6b0bf 100644 --- a/docs/vfs/VFS_INITIALIZATION.md +++ b/docs/vfs/VFS_INITIALIZATION.md @@ -5,7 +5,7 @@ The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed! ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Create and initialize Brainy const brain = new Brainy({ @@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e ## Complete Example ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' async function useVFS() { // Initialize Brainy @@ -100,7 +100,7 @@ useVFS().catch(console.error) ## TypeScript Usage ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' class FileManager { private brain: Brainy diff --git a/docs/vfs/building-file-explorers.md b/docs/vfs/building-file-explorers.md index 7514c12e..6bb31871 100644 --- a/docs/vfs/building-file-explorers.md +++ b/docs/vfs/building-file-explorers.md @@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues: ### Method 1: Use `getDirectChildren()` (Recommended) ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -97,7 +97,7 @@ Here's a complete example using React: ```tsx import React, { useState, useEffect } from 'react' -import { VirtualFileSystem } from '@soulcraftlabs/brainy' +import { VirtualFileSystem } from '@soulcraft/brainy' interface FileNode { name: string @@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) { If you must build trees manually from flat lists, use the `VFSTreeUtils`: ```typescript -import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs' +import { VFSTreeUtils } from '@soulcraft/brainy/vfs' // Get all entities somehow const allEntities = await vfs.getDescendants('/root') diff --git a/examples/bluesky-distributed-setup.js b/examples/bluesky-distributed-setup.js index e3b33506..9e83cf25 100644 --- a/examples/bluesky-distributed-setup.js +++ b/examples/bluesky-distributed-setup.js @@ -7,7 +7,7 @@ * the Bluesky firehose with Brainy's distributed architecture */ -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import { WebSocket } from 'ws' // ===================================================== diff --git a/examples/monitor-cache-performance.ts b/examples/monitor-cache-performance.ts index 87c965a2..9d50d476 100644 --- a/examples/monitor-cache-performance.ts +++ b/examples/monitor-cache-performance.ts @@ -14,7 +14,7 @@ * ts-node examples/monitor-cache-performance.ts */ -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' // ANSI color codes for pretty output const colors = { diff --git a/integrations/README.md b/integrations/README.md index de156623..aa3d795b 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu ## Quick Start ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature. ### Minimal (in-memory): ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions()) ```typescript import express from 'express' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const app = express() const brain = new Brainy({ @@ -232,7 +232,7 @@ app.listen(3000, () => { ```typescript import { Hono } from 'hono' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const app = new Hono() diff --git a/integrations/google-sheets/README.md b/integrations/google-sheets/README.md index 8309a30a..b2b0af3a 100644 --- a/integrations/google-sheets/README.md +++ b/integrations/google-sheets/README.md @@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings. The simplest way to enable all integrations: ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -112,7 +112,7 @@ With Express: ```javascript import express from 'express' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const app = express() const brain = new Brainy({ integrations: true }) diff --git a/package-lock.json b/package-lock.json index c4f66561..c4e4f9ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "name": "@soulcraft/brainy", + "version": "8.3.3", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "name": "@soulcraft/brainy", + "version": "8.3.3", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 06ce0253..ee74980b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,6 @@ { - "name": "@soulcraftlabs/brainy", - "version": "10.4.4", - "brainyContract": 1, + "name": "@soulcraft/brainy", + "version": "8.3.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", @@ -127,16 +126,15 @@ "license": "MIT", "private": false, "publishConfig": { - "access": "public", - "registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" + "access": "public" }, - "homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy", + "homepage": "https://github.com/soulcraftlabs/brainy", "bugs": { - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues" + "url": "https://github.com/soulcraftlabs/brainy/issues" }, "repository": { "type": "git", - "url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git" + "url": "git+https://github.com/soulcraftlabs/brainy.git" }, "files": [ "dist/**/*.js", diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3–4× faster imports", - "items": [ - "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", - "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine — Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place — no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 582f4847..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} diff --git a/scripts/buildEmbeddedPatterns.ts b/scripts/buildEmbeddedPatterns.ts index c046df45..73e51224 100644 --- a/scripts/buildEmbeddedPatterns.ts +++ b/scripts/buildEmbeddedPatterns.ts @@ -10,7 +10,6 @@ import { TransformerEmbedding } from '../src/utils/embedding.js' import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' -import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -98,22 +97,13 @@ async function buildEmbeddedPatterns() { // Convert to base64 for embedding in TypeScript const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - - // Deterministic stamp: derived from the git commit time of this - // generator's inputs, never from wall-clock time — two builds of the - // same source tree must produce byte-identical output. - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') - const generatedStamp = resolveDeterministicStamp( - [path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath], - outputPath - ) - + // Generate TypeScript file with everything embedded const tsContent = `/** * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${generatedStamp} + * Generated: ${new Date().toISOString()} * Patterns: ${libraryData.patterns.length} * Coverage: 94-98% of all queries * @@ -207,6 +197,7 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length} ` // Write the TypeScript file + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/buildTypeEmbeddings.ts b/scripts/buildTypeEmbeddings.ts index 688d6ac1..61bcf238 100644 --- a/scripts/buildTypeEmbeddings.ts +++ b/scripts/buildTypeEmbeddings.ts @@ -11,7 +11,6 @@ import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' import { NounType, VerbType } from '../src/types/graphTypes.js' -import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -374,24 +373,12 @@ async function buildTypeEmbeddings() { const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - // Deterministic stamp: derived from the git commit time of this - // generator's inputs, never from wall-clock time — two builds of the - // same source tree must produce byte-identical output. - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') - const generatedStamp = resolveDeterministicStamp( - [ - path.join(__dirname, 'buildTypeEmbeddings.ts'), - path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts') - ], - outputPath - ) - // Generate TypeScript file const tsContent = `/** * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${generatedStamp} + * Generated: ${new Date().toISOString()} * Noun Types: ${nounTypes.length} * Verb Types: ${verbTypes.length} * @@ -408,7 +395,7 @@ export const TYPE_METADATA = { verbTypes: ${verbTypes.length}, totalTypes: ${totalTypes}, embeddingDimensions: ${embeddingDim}, - generatedAt: "${generatedStamp}", + generatedAt: "${new Date().toISOString()}", sizeBytes: { embeddings: ${buffer.byteLength}, base64: ${base64.length} @@ -507,6 +494,7 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n ` // Write the TypeScript file + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs deleted file mode 100644 index be73d4ca..00000000 --- a/scripts/emit-contract-manifest.mjs +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env node -/** - * Emit this build's API-contract manifest to docs/api-contract.json. - * - * WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the - * code the first time somebody adds one. This reads the surface the build - * actually exposes — the prototype's own methods and accessors, the exported - * error classes, the `where` operator sets, the field-addressing vocabulary, - * the health verdicts — so a diff between two engines' manifests is a diff - * between two engines, never between two authors. - * - * Requirement marking (required / optional per door) is NOT derivable from the - * surface — it is a commitment, recorded with the contract's owner rather than - * here. This manifest carries the surface; the promise lives with the contract. - * - * Usage: node scripts/emit-contract-manifest.mjs [--check] - * --check exits non-zero when the committed manifest is stale. - */ - -import { writeFileSync, readFileSync, existsSync } from 'node:fs' -import { join, dirname } from 'node:path' -import { fileURLToPath } from 'node:url' - -const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..') -const OUT = join(ROOT, 'docs', 'api-contract.json') - -const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js')) -const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js')) -const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js')) -const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js')) - -/** Every own method and accessor on the class's prototype, minus the private ones. */ -function surfaceOf(ctor) { - const doors = [] - for (const name of Object.getOwnPropertyNames(ctor.prototype)) { - if (name === 'constructor' || name.startsWith('_')) continue - const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name) - if (!descriptor) continue - if (typeof descriptor.value === 'function') { - doors.push({ name, kind: 'method', arity: descriptor.value.length }) - } else if (descriptor.get) { - doors.push({ name, kind: 'accessor' }) - } - } - return doors.sort((a, b) => a.name.localeCompare(b.name)) -} - -const errors = Object.entries(errorsModule) - .filter(([name, value]) => typeof value === 'function' && /Error$/.test(name)) - .map(([name]) => name) - .sort() - -// The operator sets, read from the engine's own refusal message so the -// manifest can never disagree with the validator. -const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8') -const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set\(\[([\s\S]*?)\]\)/) -if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess') -const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() - -const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8') -const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) => - // Proven by the refusal path: these are the tokens with no case in the - // index's operator switch, so they fall to its default and are refused. - !new RegExp(`case '${op}':`).test(indexSource) -) -const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) - -const manifest = { - contractVersion: versionModule.contractVersion(), - engine: '@soulcraftlabs/brainy', - compatibility: { - minor: - 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', - major: - 'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused' - }, - doors: surfaceOf(Brainy), - errors, - operators: { - accepted, - servedOnIndexPath: servedOnIndex, - refusedByIndexPath: refusedByIndex, - combinators: ['allOf', 'anyOf', 'not'] - }, - fieldAddressing: { - systemKeyPrefix: 'system.', - systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(), - systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(), - plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort() - }, - health: { - verdicts: ['pass', 'warn', 'fail'], - healKinds: ['none', 'repair', 'rebuild'], - servingWithholdingInvariants: [ - 'index-initialized', - 'durable-state-present', - 'manifest-residency', - 'replay-clean', - 'strand-latch' - ] - } -} - -const rendered = `${JSON.stringify(manifest, null, 2)}\n` - -if (process.argv.includes('--check')) { - if (!existsSync(OUT)) { - console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`) - process.exit(1) - } - if (readFileSync(OUT, 'utf-8') !== rendered) { - console.error( - `docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` + - `the addition (minor = additive; a removal is a contract major).` - ) - process.exit(1) - } - console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`) - process.exit(0) -} - -writeFileSync(OUT, rendered) -console.log( - `Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` + - `${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` + - `${manifest.operators.accepted.length} operators ` + - `(${manifest.operators.refusedByIndexPath.length} refused by the index path).` -) diff --git a/scripts/gate/README.md b/scripts/gate/README.md deleted file mode 100644 index 0a8afab0..00000000 --- a/scripts/gate/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Gate Guards - -Two standalone scripts that stand between a test/build gate and a false -verdict: one refuses to let the gate start on a noisy machine, the other -refuses to let a truncated or crashed vitest run be read as green. - -## Why these exist - -Both guards exist because of the 2026-08-13 lost-day ledger: a gate ran on -a machine under load, and separately a vitest worker pool died mid-suite -while still printing a plausible-looking summary line, and in both cases -the bad result was trusted and acted on for the better part of a day before -anyone noticed. Neither failure mode announces itself — a loaded machine -still finishes and reports numbers, and a truncated test run still prints a -`Test Files` / `Tests` line — so both guards check the evidence explicitly -rather than trusting that a gate finishing means the gate was valid. - -## gate-preflight.sh - -Run before any gate lane starts. Exits 1 the moment the machine isn't -gate-clean, with one `FATAL:` line per violation naming the exact offender -(the pid and command, the path, the measured value). Prints one `OK:` line -per check that passes. `WARNING:` lines mark checks that were skipped, not -failures. - -Checks: - -| # | Check | Default threshold | Override | -|---|-------|--------------------|----------| -| a | 1-minute load average | `nproc / 2` | `GATE_MAX_LOAD` | -| b | any non-allowlisted process over 50% of one core | 50% | `GATE_ALLOW_REGEX` (extra pattern matched against the process's args) | -| c | cpu0 scaling governor must be `performance` | — | none (warns and skips if the sysfs path is absent) | -| d | free space on `/` and `/tmp` | 10G each | `GATE_SKIP_DISK_CHECK=1` to skip entirely | - -The allowlist for check (b) is always: this script's own process tree -(its ancestors and its direct child processes), `sshd`, `systemd`, and -kernel threads (recognizable by args wrapped in brackets, e.g. -`[kworker/0:1]`). `GATE_ALLOW_REGEX` extends it — it does not replace it. - -## vitest-verdict-check.sh - -Run after every vitest lane, against that lane's captured log. Fails -loudly, quoting the exact line or string that tripped it, when the log's -own summary can't be trusted: - -- no `Test Files` (or, in `--count-tests` mode, `Tests`) summary line is - present at all -- the parenthesized total in that line doesn't match what was expected -- fewer files/tests are accounted for (passed + failed + skipped) than the - total claims — a truncated run -- the log contains `Unhandled Error` or `Timeout calling` anywhere — a dead - worker pool, regardless of what the summary line claims - -``` -vitest-verdict-check.sh -vitest-verdict-check.sh --count-tests -``` - -The first form checks `Test Files` for an exact match. The second checks -`Tests` for a minimum (a floor, not an exact count, since the total number -of individual tests moves more often than the number of test files). - -## Wiring into a CI lane - -```sh -# Before any lane that will report a verdict: -scripts/gate/gate-preflight.sh || exit 1 - -# Run the suite, capturing its output: -npx vitest run tests/unit 2>&1 | tee /tmp/unit.log - -# After every vitest lane, check the log against the actual file count: -EXPECTED_FILES=$(ls tests/unit/**/*.test.ts | wc -l) -scripts/gate/vitest-verdict-check.sh /tmp/unit.log "$EXPECTED_FILES" || exit 1 -``` - -## Exit-code contract - -| Script | Exit 0 | Exit 1 | -|--------|--------|--------| -| `gate-preflight.sh` | machine is gate-clean | one or more `FATAL:` violations printed | -| `vitest-verdict-check.sh` | log's summary is trustworthy and matches | usage error, missing/unreadable log, or one or more `FATAL:` violations printed | - -Non-zero from either script means: do not trust the gate that was about to -run, or the result of the one that just ran. diff --git a/scripts/gate/gate-preflight.sh b/scripts/gate/gate-preflight.sh deleted file mode 100755 index c6208f49..00000000 --- a/scripts/gate/gate-preflight.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Brainy Gate Preflight -# Refuses to let a test/build gate run on a machine that isn't clean enough -# to trust the numbers it produces. See scripts/gate/README.md for why (the -# 2026-08-13 lost-day ledger). -# -# Checks: 1-minute load average, any non-allowlisted process pinning a core, -# the cpu0 scaling governor, and free space on / and /tmp. -# -# Exit 0 and print one OK line per passing check when the machine is clean. -# Exit 1 and print one FATAL line per violation, naming the offender, when -# it is not. -# -# Known trap: a helper function whose last executed statement is a `while` -# (or any command whose own exit status happens to be nonzero) hands that -# status back as the function's return value. Called as a plain statement, -# that silently kills this script under `set -e`. Every helper below ends -# on an explicit `return 0` as its own statement, never on a loop or test. -# -# The same failure mode hides in plainer-looking lines too: `var=$(cmd)` is -# a bare assignment, so `set -e` DOES treat a nonzero `cmd` (or, under -# `pipefail`, a nonzero stage anywhere in `cmd`'s pipeline) as a failure of -# that statement and kills the script right there — even mid-loop, even -# when the "failure" is routine (a process that exited before a second -# lookup, a path that doesn't exist). Every such assignment below is paired -# with an explicit `|| var=""` fallback so a routine miss degrades to an -# empty value instead of an exit. - -VIOLATIONS=0 -ANCESTOR_PIDS="" - -fatal() { - echo "FATAL: $1" - VIOLATIONS=$((VIOLATIONS + 1)) -} - -ok() { - echo "OK: $1" -} - -# Walks this process's parent chain up to pid 1, then takes one snapshot of -# its direct children (the ps/read pipeline in check_processes), and -# records both in ANCESTOR_PIDS — so the process-scan below can recognize -# its own tree (the shell/terminal/session that launched it, plus its own -# helper commands) instead of flagging it. Children are captured once, up -# front, rather than re-queried per row later, so a helper command that has -# already exited by the time it's looked up can't be mistaken for a miss. -build_ancestor_pids() { - local pid="$$" - local ppid child - ANCESTOR_PIDS=" $pid " - while [ "$pid" != "1" ]; do - ppid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') || ppid="" - if [ -z "$ppid" ]; then - break - fi - ANCESTOR_PIDS="${ANCESTOR_PIDS}${ppid} " - pid="$ppid" - done - - while IFS= read -r child; do - [ -z "$child" ] && continue - ANCESTOR_PIDS="${ANCESTOR_PIDS}${child} " - done < <(ps --ppid "$$" -o pid= 2>/dev/null || true) - - return 0 -} - -# (a) 1-minute load average vs. threshold (default: nproc / 2). -check_load() { - local max_load="${GATE_MAX_LOAD:-}" - if [ -z "$max_load" ]; then - max_load=$(( $(nproc) / 2 )) - if [ "$max_load" -lt 1 ]; then - max_load=1 - fi - fi - - local load_1m - load_1m=$(cut -d' ' -f1 /proc/loadavg) - - if awk -v l="$load_1m" -v m="$max_load" 'BEGIN { exit !(l > m) }'; then - fatal "1-minute load average ${load_1m} exceeds threshold ${max_load} (GATE_MAX_LOAD=${max_load})" - else - ok "1-minute load average ${load_1m} is within threshold ${max_load}" - fi - return 0 -} - -# (b) any process outside the allowlist pinning more than half a core. -# Parsed with `read` into named fields, not an awk/cut chain — a fixed-column -# awk/cut split on `ps` output duplicated fields the first time this was -# tried, because process args vary in word count. `read` with a fixed list -# of variables dumps everything left over into the last one (args), which -# handles that correctly. -check_processes() { - local max_pcpu=50 - local extra_regex="${GATE_ALLOW_REGEX:-}" - local violation_found=0 - local line pcpu pid args pcpu_int - - while IFS= read -r line; do - [ -z "$line" ] && continue - read -r pcpu pid args <<< "$line" - - # Kernel threads report their comm in brackets, e.g. "[kworker/0:1]". - case "$args" in - \[*\]) continue ;; - esac - - # This script's own tree: its ancestors (shell, terminal, session) and - # its direct children, both captured once by build_ancestor_pids. - case " $ANCESTOR_PIDS " in - *" $pid "*) continue ;; - esac - - case "$args" in - *sshd*|*systemd*) continue ;; - esac - - if [ -n "$extra_regex" ] && [[ "$args" =~ $extra_regex ]]; then - continue - fi - - pcpu_int="${pcpu%.*}" - if [ -z "$pcpu_int" ]; then - pcpu_int=0 - fi - if [ "$pcpu_int" -gt "$max_pcpu" ]; then - fatal "pid ${pid} ('${args}') is using ${pcpu}% of one core" - violation_found=1 - fi - done < <(ps -eo pcpu,pid,args --sort=-pcpu | tail -n +2) - - if [ "$violation_found" -eq 0 ]; then - ok "no process outside the allowlist exceeds ${max_pcpu}% of one core" - fi - return 0 -} - -# (c) cpu0 scaling governor must be "performance". Skipped with a warning -# (not a violation) when the sysfs path doesn't exist on this machine. -check_governor() { - local gov_path="/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" - if [ ! -r "$gov_path" ]; then - echo "WARNING: ${gov_path} not present; skipping governor check" - return 0 - fi - - local governor - governor=$(cat "$gov_path" 2>/dev/null) || governor="" - if [ "$governor" != "performance" ]; then - fatal "cpu0 governor is '${governor}', not 'performance'" - else - ok "cpu0 governor is 'performance'" - fi - return 0 -} - -# (d) free-space floors on / and /tmp (default 10G each). Skip entirely via -# GATE_SKIP_DISK_CHECK=1. -check_disk() { - if [ "${GATE_SKIP_DISK_CHECK:-0}" = "1" ]; then - echo "WARNING: disk free-space check skipped (GATE_SKIP_DISK_CHECK=1)" - return 0 - fi - - local floor_gb=10 - local floor_bytes=$((floor_gb * 1024 * 1024 * 1024)) - local path avail_bytes avail_gb - - for path in / /tmp; do - avail_bytes=$(df --output=avail -B1 "$path" 2>/dev/null | tail -n 1 | tr -d ' ') || avail_bytes="" - if [ -z "$avail_bytes" ]; then - echo "WARNING: could not determine free space on ${path}; skipping" - continue - fi - if [ "$avail_bytes" -lt "$floor_bytes" ]; then - avail_gb=$((avail_bytes / 1024 / 1024 / 1024)) - fatal "${path} has only ${avail_gb}G free, below the ${floor_gb}G floor" - else - ok "${path} has enough free space (floor ${floor_gb}G)" - fi - done - return 0 -} - -echo "Brainy gate preflight" -echo "----------------------" - -build_ancestor_pids -check_load -check_processes -check_governor -check_disk - -echo "----------------------" -if [ "$VIOLATIONS" -gt 0 ]; then - echo "FATAL: gate preflight failed with ${VIOLATIONS} violation(s) — machine is not gate-clean" - exit 1 -fi - -echo "gate preflight passed — machine is gate-clean" -exit 0 diff --git a/scripts/gate/vitest-verdict-check.sh b/scripts/gate/vitest-verdict-check.sh deleted file mode 100755 index 36243a1d..00000000 --- a/scripts/gate/vitest-verdict-check.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Brainy Vitest Verdict Check -# Confirms a vitest run's own summary line is trustworthy before anything -# downstream treats a green run as green. See scripts/gate/README.md for why -# (the 2026-08-13 lost-day ledger). -# -# Usage: -# vitest-verdict-check.sh -# vitest-verdict-check.sh --count-tests -# -# The first form checks the "Test Files" summary line's total against an -# exact expected count. The second checks the "Tests" summary line's total -# against a minimum. Both also fail on any sign the worker pool died -# mid-run, whether or not a summary line still made it into the log. -# -# Exit 0 and print one OK line per passing check when the log is clean. -# Exit 1 and print one FATAL line per violation, quoting the exact line or -# string that tripped it, when it is not. -# -# Known trap (shared with gate-preflight.sh): every helper below ends on an -# explicit `return 0` as its own statement, never on a loop or test, so a -# helper's last command can never hand its own exit status back as the -# function's under `set -e`. The same applies to `var=$(cmd)` assignments -# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that -# legitimately finds nothing (exit 1) would otherwise kill the script -# instead of just leaving the variable empty — every such assignment below -# is paired with an explicit `|| true` inside the substitution. - -usage() { - echo "Usage: $0 " - echo " $0 --count-tests " - exit 1 -} - -MODE="files" -if [ "${1:-}" = "--count-tests" ]; then - MODE="tests" - shift -fi - -LOG_FILE="${1:-}" -THRESHOLD="${2:-}" - -if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then - usage -fi - -if [ ! -f "$LOG_FILE" ]; then - echo "FATAL: log file '${LOG_FILE}' does not exist" - exit 1 -fi - -if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then - echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer" - exit 1 -fi - -VIOLATIONS=0 - -fatal() { - echo "FATAL: $1" - VIOLATIONS=$((VIOLATIONS + 1)) -} - -ok() { - echo "OK: $1" -} - -# Vitest colorizes its summary with ANSI escapes; strip them before parsing -# anything, or the color codes end up embedded in the fields we grep for. -CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")" - -# Worker-pool death: if either string appears, the run's own summary line — -# even if present and even if its numbers look fine — cannot be trusted, -# because the process died mid-suite and vitest's own accounting is what -# died with it. -check_worker_death() { - if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then - fatal "log contains 'Unhandled Error' — worker pool died mid-run" - fi - if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then - fatal "log contains 'Timeout calling' — worker pool died mid-run" - fi - return 0 -} - -# Shared shape between the "Test Files" and "Tests" summary lines: -#