diff --git a/.claude/skills/architecture.md b/.claude/skills/architecture.md index 4de3c287..de046b17 100644 --- a/.claude/skills/architecture.md +++ b/.claude/skills/architecture.md @@ -2,7 +2,7 @@ ## What Is Brainy -@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. +@soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. ## Core Architecture diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index da5887f6..5e93cd96 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -5,10 +5,6 @@ name: CI # sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the # tag's publish-source run and starve every release (observed on 8.10.3 and # 9.0.0: the publish sat behind the tag's own redundant CI). -concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true - on: push: branches: ['**'] diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml index 6bd42b2a..a08875ae 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-source.yml @@ -12,11 +12,6 @@ on: push: tags: - 'v*' - workflow_dispatch: - inputs: - ref_reason: - description: 'why this manual run (e.g. tag event dropped)' - required: false jobs: publish: @@ -37,7 +32,7 @@ jobs: run: | set -eo pipefail - SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" + SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" VERSION="$(node -p "require('./package.json').version")" # The dist-tag follows the version: a prerelease (any hyphen — # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — @@ -48,13 +43,13 @@ jobs: case "$VERSION" in *-*) NPM_TAG="rc" ;; esac - echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." + echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." TMPRC="$(mktemp)" chmod 600 "$TMPRC" { - echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}" - echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}" + echo "@soulcraft:registry=${SOURCE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" } > "$TMPRC" # The release script bumps package.json's version before it tags, so @@ -69,7 +64,7 @@ jobs: # exit code: a benign duplicate publish (a prior run, or a mirror, already # landed this exact version) reports failure even though the registry # already holds the right content. - LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" + LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" rm -f "$TMPRC" if [ "$LANDED_VERSION" != "$VERSION" ]; then @@ -78,7 +73,7 @@ jobs: fi if [ "$PUBLISH_OK" = true ]; then - echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry." + echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry." else - echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." fi diff --git a/CHANGELOG.md b/CHANGELOG.md index a54d609e..cd64d88f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,101 +2,7 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) - -- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) -- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25) -- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40) -- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2) -- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927) -- docs: measurements in public history carry numbers, not provenance (02c61636) -- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010) -- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950) -- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9) -- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5) -- perf(open): answer "are there any entities?" with one directory read (417ddb51) -- perf(generations): discover generations by directory name, not by walking the log (9dd39921) -- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc) -- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca) -- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f) -- chore: keep the generated neural stamps at main's values (c1f09723) -- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3) -- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02) -- feat(open): open never waits for a provider that is rebuilding itself (131daa08) -- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f) -- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6) -- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b) -- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f) -- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c) -- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b) - - -### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) - -- Merge branch 'next/open-brainy-rename' (a58372f0) -- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) -- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) - - -### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) - -- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) - - -### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - -- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) -- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) -- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1) -- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4) -- Merge branch 'next/enumeration-identity-rekey' (204d74c1) -- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16) -- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a) -- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab) -- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9) -- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245) -- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee) -- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb) -- chore(release): 10.4.1-rc.1 (7870dc40) - - -### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - -- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) -- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) - - -### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) - -- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) - - -### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) - -- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) - - -### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) - -- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) -- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) -- fix(add): empty string is real data, not a missing field (258e9042) -- feat(vfs): implement readdir's recursive option — typed since 7.30, never read (fc516da6) -- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) - - -### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) - -- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) -- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) -- fix(storage): an unknown nested storage config can never silently land on the shared default root (ddd5e719) -- docs(release): the 10.4.0 entry, the index-health concept doc, and the API surfaces — written from the tree, not the plan (8cced871) -- fix(plugins): the silent-degrade doors close — a broken accelerator install can never read as absent (b9ba50fb) -- feat(recovery): the catchup verdict is consumed; verb rows go live; the metadata rebuild goes online (18f172e0) -- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) - - -### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) +### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) - ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) - chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) @@ -109,13 +15,13 @@ All notable changes to this project will be documented in this file. See [standa - ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) -### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) +### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18) - docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) - fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) -### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) +### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) - fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) @@ -124,14 +30,14 @@ All notable changes to this project will be documented in this file. See [standa - feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) -### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) +### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17) - docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) - ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) - fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) -### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) +### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) - fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) @@ -140,7 +46,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) -### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) +### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12) - fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) - fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) @@ -172,7 +78,7 @@ All notable changes to this project will be documented in this file. See [standa - test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) -### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) +### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04) - docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) - fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) @@ -207,7 +113,7 @@ All notable changes to this project will be documented in this file. See [standa - feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) -### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) +### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27) - docs: the last two archived-host links point home (91ef1c8b) - feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) @@ -216,19 +122,19 @@ All notable changes to this project will be documented in this file. See [standa - ci: run the pipeline on the forge (999d0ebb) -### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) +### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03) - docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) - fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) -### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) +### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) - fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) -### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) +### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) - fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) diff --git a/CLAUDE.md b/CLAUDE.md index 56df0b72..c7336a18 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md` **Brainy's current open actions:** None. MIT open-source — no platform-specific actions. -**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` +**Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` --- ## Project Overview -Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license. +Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license. ## Getting Started @@ -91,7 +91,7 @@ test: add/update tests (patch version bump) ## Docs Pipeline — soulcraft.com/docs -Docs in `docs/**/*.md` are published with the npm package (included in `files`) and go live on soulcraft.com/docs via the docs ingest API: the release script's `scripts/push-docs.js` step POSTs every public doc to `https://soulcraft.com/api/docs/ingest` (auth: `DOCS_INGEST_SECRET` in the environment). No separate deploy step is involved (the old deploy-to-publish flow was retired in a platform change, 2026-08). Frontmatter controls what appears publicly. +Docs in `docs/**/*.md` are published with the npm package (included in `files`) and synced to soulcraft.com/docs on every portal deploy. Frontmatter controls what appears publicly. ### Docs check triggers @@ -161,9 +161,9 @@ npm run release:major # Breaking changes (rare, manual decision) The script: verifies clean git state, builds, tests, bumps version, updates CHANGELOG.md, commits, tags, pushes, publishes to npm, and creates a GitHub release. After a successful release, remind the user: -> "Published. Docs are live on soulcraft.com/docs (pushed via the ingest API during the release) — spot-check a changed page with curl." +> "Published. Deploy portal to pick up the new docs → go to the portal project and deploy." -There is no separate deploy step anymore. If the docs push failed (the script warns loudly), re-run `node scripts/push-docs.js` with `DOCS_INGEST_SECRET` set. +Do NOT deploy portal from here. Portal is always deployed separately from within the portal project. ## Closed-Source Product Names — HARD RULE diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54d4f784..d277091d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ may find elsewhere in the repo's history. ## Where the project lives -The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. +The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**. It's anonymously readable and cloneable — no account needed to browse, clone, or build. @@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework. ## Development setup ```bash -git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git +git clone https://source.soulcraft.com/soulcraft/brainy.git cd brainy npm install npm run build @@ -57,17 +57,6 @@ see `package.json` for `test:integration`, `test:coverage`, and friends. description states a number, cite the benchmark that produced it (see [docs/performance-envelopes.md](docs/performance-envelopes.md) for the pattern). Don't state an estimate as if it were measured. -- **Measurements carry numbers, not provenance.** Public commit messages and - docs give the SHAPE a number was taken at and never where it was taken: no - hostnames, no store or deployment identities, no operational anecdotes about - someone's running system. "A 14,056-noun / 72,679-verb production-shaped - store, measured solo under an exclusive lock" tells a reader everything the - number depends on; the machine it ran on and whose data it was tell them - nothing except where somebody's infrastructure lives. -- **Documents that answer or reference a confidential specification never enter - this repository, even summarized.** The public docs describe THIS engine and - the published contract, and nothing else — a summary of a private document is - still that document's contents. ## License diff --git a/README.md b/README.md index 762c9ec3..ca558340 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- Brainy + Brainy

Brainy

@@ -11,9 +11,9 @@

- Package on The Source - Repository - CI + npm version + npm downloads + CI Documentation MIT License TypeScript @@ -30,8 +30,6 @@ --- -**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license. - Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: | You write | Brainy indexes it as | You query it with | @@ -47,14 +45,12 @@ It runs **inside your process** — no server, no Docker, nothing to operate — ## Quick start ```bash -bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended -npm install @soulcraftlabs/brainy # Node.js ≥ 22 +bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended +npm install @soulcraft/brainy # Node.js ≥ 22 ``` -> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read). - ```javascript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() // in-memory; one line swaps to disk await brain.init() diff --git a/RELEASES.md b/RELEASES.md index c875cb26..cc0272c3 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,14 +1,7 @@ # @soulcraft/brainy — Release Notes for Consumers -Machine-readable release notes are published at -https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json -(this engine) and -https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json -(the product engine) — read by HQ's `/hq/releases` door, and the source of -truth ahead of this file. - This file is the **quick reference for downstream sessions** tracking Brainy changes. -Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases +Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases **How to use:** Brainy is the underlying data engine for downstream applications. Read this when: - Upgrading `@soulcraft/brainy` in your application @@ -38,330 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v10.4.4 — 2026-08-28 - -**A correctness and observability release.** The headline is not speed: it is that a -restart now tells you the truth about itself, a store stops lying about how much it -holds, and the engine stops doing work nobody asked for. There is a performance -improvement and it is modest; it is stated exactly below rather than rounded up. - -### The dark restart — fixed at the root - -A service could stop cleanly, exit 0, having awaited `close()` on every store it held, -and its next boot would announce `Overwriting stale writer lock … appears dead` for -every one of them. Nothing had crashed. Two deployments hit this; the same defect also -made those boots pay a crash-recovery fold they did not owe. - -The cause was not the lock. `close()` released it correctly — when it got there. A -failure part-way through close skipped both the release AND the clean-shutdown marker, -and "the recorded pid is gone" reads identically for an orderly restart and a crash. - -- `close()` is now two parts and the second is unconditional: the flush-request watcher, - the **writer lock**, the VFS timers and the terminal `closed` flag are released whether - the durable steps succeeded or not. The original failure is narrated with what it costs - the next open, then rethrown. -- Releasing the lock writes a **clean-close record** naming the lock generation it gave - up. The next open reads that record instead of guessing: recorded → nothing to recover; - absent → it says so, and names the recovery it is about to run. This also ends two - long-standing false alarms — a recycled pid locking a store out of its own reopen, and - `Re-acquiring writer lock … this is a bug` after a perfectly clean close. -- The signal path stopped failing in a batch. One store's failing flush used to strand - every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the - generation store's close (the marker) is part of shutdown, the lock goes in a `finally`, - and the handler no longer calls `process.exit()` when the host application has its own - signal handler, a race that truncated the host's own shutdown mid-flight. - -### The count ledger stops lying, and `counts.json` is written atomically - -The all-tier scalars are the denominator a coverage check subtracts against. A ledger -derived under the old rule — one entity per id DIRECTORY — counted ghost and scar -containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for -the life of the store. Two copies of one archive could disagree, and a downstream index -heal reported remaining work that did not exist. - -- Such a ledger now derives itself honestly **in the background** after the open, counting - identity records, and persists the correction stamped. Nothing waits for it, because no - read is served from a denominator. -- A derivation that raced a write refuses to stamp its number: one retry on a quiet store, - then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under - a barrier. -- `counts.json` is written temp+rename. A truncating write left a window in which a - concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open - down the full-rescan path, so the cheapest file in the store was buying the most - expensive recovery. - -### An open and a repair narrate themselves — on a channel a log level cannot silence - -A store could open for three minutes and print nothing at all. The phase timings existed; -they were written to a channel that every production-looking environment clamps away. - -- Narration moved to an always-visible channel. An open now heartbeats the phase it is in, - names each phase as it ends with what it was paying for, and names the expensive STEP - inside a phase. `repairIndex()` does the same and its receipt carries a per-family - `durationMs` — a repair that ran for half an hour with no output could only be watched - through `top`. -- A brain nobody has written to now does nothing: a flush over a clean store is a no-op - and says nothing, the graph index's auto-flush asks before it acts, and the - cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a - directory every 500 ms per store forever, with a slow safety sweep behind it and a - narrated fall back to polling where a filesystem cannot be watched. -- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()` - does not wait for it, every other family serves, and that family's doors refuse **by - name, carrying the provider's own progress**, saying plainly that they open by - themselves and no action is needed. Health narration dedupes by content, so an unchanged - verdict is silent however a provider's generation counter moves. - -### For operators — one behaviour change - -**Four `where` operators that previously returned an empty page now raise -`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range -posting index cannot evaluate a substring, a pattern or an array length without reading -every row, and it now refuses by name instead of answering with an empty result that -looks like an answer. - -**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and -`excludes`. All 25 accepted operator tokens now agree between this engine and its -accelerated counterpart. - -### Performance — stated exactly - -Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under -an exclusive lock: - -- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is - one fix — generation discovery reads directory NAMES instead of recursively walking the - entire generation log (−9.2 s, and it scales with history rather than row count). The - VFS phase is **unchanged**. -- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving - off the critical path accounts for storage-init dropping 5,941 ms → 25 ms. -- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own - init is under 2 s of that phase. It is the log-authority adoption and/or the - pending-embed log recovery, both now instrumented so the next measurement names the - culprit outright. - -Continuing work, named so nobody has to rediscover it: that ~38 s term; making the -generation store's committed-range set lazy; the hydration path that substitutes -`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix -filter built with a `$startsWith` spelling no operator set accepts, so -`searchFiles({ path })` throws today. - ---- - -## v10.4.3 — 2026-08-27 (Open Brainy's first release) - -**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for -byte — only the name, the registry, and the pointers changed.** Install: - -```bash -npm install @soulcraftlabs/brainy -``` - -with the registry line in your `.npmrc` (anonymous read): - -``` -@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/ -``` - -- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the - npmjs republish step is retired from the release rail. Existing npmjs versions of - `@soulcraft/brainy` stay as they are and receive no new versions. -- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects. -- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one - install-line change (`@soulcraft/brainy` → `@soulcraftlabs/brainy`), which downstream - applications make together with their native-engine bump. - -## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector) - -**This is the last release of the MIT engine under the `@soulcraft/brainy` name.** -The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API, -client library, types and protocol, an openly specified canonical format, and the TypeScript -reference engine, scoped honestly as a single-node engine for stores up to roughly one -million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a -major version bump; that engine implements the same API over the same open format at -production scale, requires a license, and refuses loudly without one. Nothing changes -for existing installs until that major ships; the move is announced with it. - -Six fixes, one law: a vector with no magnitude carries no information, so it must -never reach a vector index — in any engine — and the canonical store must say so. - -- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item - shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted - with an empty vector leg, never embedded, never indexed, counted as unvectored in - the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing — - no longer need a placeholder vector and never enter the vector leg. `vector: []` - together with `deferEmbedding: true` is refused with a typed error (a supplied - vector has nothing to defer). Previously `vector: []` threw a dimension error. -- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is - the sanctioned way to strip a vector from an existing row: canonical vector → `[]`, - removal from the vector index, the vectored ledger decremented exactly once — and - idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and - it clears a pending deferred-embed marker durably so the background worker cannot - re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the - index from canonical rows); shedding historical vectors needs this door. -- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on - any write path is persisted as unvectored (`[]`) with one warning naming the row; - the vector-index operations keep their own refusal as a second line. The engine's - own VFS root, which used to persist a deliberate all-zero placeholder (harmless - under cosine distance, a false attractor under a downstream engine's - squared-euclidean serving — a production incident this week), is now created - unvectored, and an existing store's legacy root is migrated on open by a single - fixed-path read before the health gate runs — never a walk. -- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the - cursor walks behind them enumerate by the metadata record, the same key the - canonical ledger counts by — previously the walk keyed on the vector file, so a - row holding metadata but no vector was counted yet never yielded (a permanent - "missing" phantom in coverage math), while an orphaned vector-only directory - could be yielded as a phantom id. The recovery fold also never deletes an existing - vector when it replays a metadata-only after-image (preserve-if-absent). One - documented gap remains: a verb's endpoints live only in its vector leg, so a - metadata-only verb is counted and loudly skipped, never fabricated — the fix is a - canonical-format change and lands with the open format. -- **The ledger's one-time derivation counts identity records.** Stores upgraded from - pre-ledger versions derived their ALL-visibility scalars once by counting id - directories, which included ghost and scar containers left by an old partial-delete - defect — an inflated denominator whose coverage row could never reach exact. The - derivation now counts only directories holding a metadata record, `counts.json` - carries a derivation-rule stamp, and a ledger derived under the old rule is marked - `suspect` at open (one O(1) field read, one warning) so the online `repairIndex()` - path clears it with a real recount. -- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and - zero-norm rows (one summary line), re-pins the vector dimension from the first real - vector after a restart (previously a restart left the pin unset, so a wrong-length - insert became the new pin instead of being rejected), and `addItem` / `updateItem` - throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing - a vector-less node. -- **Smaller:** a failing plugin activation now rethrows with the original error as - `cause` (the originating file and line survive to the caller's log); build - generators stamp from the repository history of their inputs instead of wall clock, - so two builds of the same tree are byte-identical. - -Adoption: one restart, paired with its native-engine release. The first open of an -existing store runs the legacy-root migration (one narrated line) and, on stores that -upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned -recount — no rebuild in either case. - -## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds) - -Two production defects from the same week, fixed together as a patch to 10.4.0. - -- **The read gate is per family.** A read now refuses only when the index family it - actually consults is unhealthy: a metadata filter is served while the vector leg is - rebuilding; a semantic query is refused only by the vector family; a graph - traversal only by the graph family. Previously any unhealthy family refused every - read on the brain — under a long vector rebuild, a production deployment's - metadata-only reads were refused for the duration, and the retries became a write - pump of their own. -- **Unchanged data never re-embeds.** `update()` compares the incoming `data` - structurally with the stored record; an update carrying identical data (a common - shape for periodic upserts) no longer embeds again and no longer churns the vector - leg. Previously every such update re-embedded and re-inserted, which under load - saturated the vector index with near-identical vectors. - -Adoption: one restart, paired with its native-engine release. - -## v10.4.0 — 2026-08-25 (the health report has a name) - -Three related cures, one root cause: an index deciding whether it could be trusted -by sampling itself instead of by exact accounting. This release replaces every -sampled self-probe with ledger-derived truth, and a read against an unhealthy index -now refuses loudly instead of guessing. - -- **The canonical count ledger.** Storage now tracks two scalars per family - (nouns/verbs) on the write path: the user-facing `counted` total — unchanged, - still what `getNounCount()` / `getVerbCount()` return — and a new ALL-visibility - `all` total covering every tier, the real denominator a derived index's own - coverage math needs. The unfiltered storage-level `totalCount` returned by - `getNouns()` / `getVerbs()` is now this unclamped ALL scalar; previously it could - only ever move up (`Math.max(scalar, scanned)`), so an inflated counter could - never self-correct. A delete that cannot prove the record it removed actually - existed (no canonical read, no prior image available) no longer decrements on - faith — it marks the ledger `suspect` (narrated once per session) instead of - silently drifting, and the next `repairIndex()` clears the flag with a real - recount. -- **One contract for a throwing health probe.** A provider's `validateInvariants()` - is documented to never throw — but if one does anyway (a bug, a transient fault), - it is now read the same way everywhere: `heal: 'none'`, the error named in the - report, never synthesized into a rebuild trigger and never swallowed into "looks - fine." A flaky check can no longer buy itself a rebuild. `repairIndex()`'s - per-family receipt also gains `missing` (an exact count plus a capped id sample), - `rebuilt` (a full rebuild ran, vs. an incremental heal), and `reason`. -- **The named health report; reads refuse instead of rebuilding.** Any index - provider may now expose a synchronous, O(1) `healthReport()` — composed from the - provider's own exact ledgers, never a sample — and this is the one signal - Brainy's read gate trusts. The first-query lazy-build path is gone: `brain.init()` - now runs every needed rebuild to completion before it returns, always, regardless - of dataset size. A read that lands on a provider whose health report says it - isn't serving throws a typed error instead of triggering a rebuild mid-query — - `GraphIndexNotReadyError`, `MetadataIndexNotReadyError`, or - `VectorIndexNotReadyError` (all exported from `@soulcraft/brainy`), naming the - reasons. `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] | 'all' })` is - the new explicit operator door: it rebuilds the named family unconditionally, no - health check consulted — reach for it when you have independent reason to - distrust a family regardless of what it self-reports. Bare `repairIndex()` is - unchanged in spirit: report-driven, heals only what its own checks say needs it. -- New concept doc: [Index Health](docs/concepts/index-health.md) walks the whole - story from a consumer's side — degraded-but-serving vs. not-ready, what - `repairIndex()` checks and heals per family, what `suspect` counts mean. - -**Nothing to change to adopt this.** No API removed, no signature narrowed — -`repairIndex()` gains an optional options bag and its return value gains fields, -both additive. The honest notes: if your code ever relied on a `find()` against a -cold/not-yet-built index quietly triggering a rebuild and returning results a beat -later, that behavior is gone — it now throws one of the three typed -`*NotReadyError` classes instead (catch them if you need to distinguish "not ready -yet" from "no results"). And `disableAutoRebuild: true` no longer defers index -construction to the first query — a needed rebuild always runs at `open()` now; -the flag has no effect on timing. Full manual control still lives in -`repairIndex({ rebuild: [...] })`. - -- **Crash-reopen catchup.** After an unclean shutdown, the metadata index now - folds the exact fact window it missed — `find()` serves every acked write on - reopen, closing the gap where canonical reads and counts recovered a - crash-window write but the index kept serving its pre-crash state until the - next full rebuild. Related root-cause fixed alongside: `close()` never - stamped the index watermarks (only `flush()` did), so a close without a - prior flush caused a needless full rescan verdict on the next open. -- **Relation rows are live in the metadata index.** Previously verb rows - entered the metadata index only during a rebuild — so a rebuilt store's - relation postings went stale from the first `relate()` after it. Relations - are now posted and retracted on the live write path (relate / unrelate / - updateRelation / remove's cascade, and their `transact()` forms), in the - same commit as the graph leg. -- **The metadata rebuild is online.** `rebuild()` for the metadata family no - longer clears and rebuilds in place (reads went empty for the duration): it - builds a complete replacement beside the serving index, mirrors concurrent - writes to both, swaps atomically, and persists once after the swap. Reads - never observe a partial index. `repairIndex({ rebuild: ['metadata'] })` uses - it automatically. -- **Incremental heal is routed.** A provider invariant that asks for the - incremental heal (`heal: 'repair'`) now routes to the provider's own - `repair()` when it exposes one — re-posting exactly what its ledger names, - never a store-sized rebuild — and the post-heal re-read of the report decides - success; a repair that doesn't converge is recorded with the escalation named. -- **The vector family joins the count ledger.** `getCanonicalCounts()` gains - `vectors: { all }` — the count of canonical entities holding a real vector - (deferred-embed entities count when their vector lands). And the open gate - closes the vector leg: a store whose canonical rows hold vectors but whose - derived vector index is empty now builds at `open()` (or refuses with the - typed error) instead of silently serving empty vector-search results. -- **An unknown storage config shape fails loudly.** A nested `config` object - carrying a path-shaped key (a shape that was never supported) used to fall - through silently to the default shared directory — every instance writing one - store while callers believed each had its own. It now throws, naming the - canonical `path` key. -- **Relation index rows are JSON-safe.** Internal endpoint identifiers can no - longer ride the metadata-index crossing (a native provider serializes it); - they stay on the graph operations where they belong. -- **A broken accelerator install can never read as "not installed."** The - auto-detection free pass now requires the resolution error to name the - accelerator package itself, exactly — a missing platform-binary sibling - package, an inner file path, or a dependency failure is a broken install and - `init()` throws loudly. And a plugin that declines activation is narrated on - the always-on log channel, so `silent: true` can no longer hide a fallback - to the default engines. - ---- - ## v10.3.1 — 2026-08-18 (the fold that behaves) Three recovery cures from one production first-boot incident (a brain's first diff --git a/SECURITY.md b/SECURITY.md index 91d40d49..1f3c4732 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -30,7 +30,7 @@ commit to backporting fixes to unsupported lines. ## Scope -This policy covers the `@soulcraftlabs/brainy` package itself — the code in +This policy covers the `@soulcraft/brainy` package itself — the code in this repository. If you're evaluating a deployment that also uses `@soulcraft/cor`, report issues in that package the same way, to the same address; we'll route internally. diff --git a/bin/brainy-ts.js b/bin/brainy-ts.js index 90a35e98..4e9aedb8 100644 --- a/bin/brainy-ts.js +++ b/bin/brainy-ts.js @@ -3,7 +3,7 @@ /** * Modern TypeScript CLI Runner * - * This is the entry point after npm install @soulcraftlabs/brainy + * This is the entry point after npm install @soulcraft/brainy * It runs the compiled TypeScript CLI code */ diff --git a/bun.lock b/bun.lock index 1e3e66e2..c31b3865 100644 --- a/bun.lock +++ b/bun.lock @@ -3,7 +3,7 @@ "configVersion": 0, "workspaces": { "": { - "name": "@soulcraftlabs/brainy", + "name": "@soulcraft/brainy", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", "@azure/identity": "^4.0.0", diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md index b2d22fb1..4134ae63 100644 --- a/docs/DEVELOPER_LEARNING_PATH.md +++ b/docs/DEVELOPER_LEARNING_PATH.md @@ -25,13 +25,13 @@ ### Prerequisites ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Your First Neural Database ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' // Step 1: Create and initialize Brainy const brain = new Brainy({ @@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou ### Building a Knowledge Graph ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**. ### Triple Intelligence in Action ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in ### Files as Intelligent Entities ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**. ### Production-Ready Deployment ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' // 1. PRODUCTION STORAGE - Filesystem with off-site snapshots console.log('Initializing production storage...\n') diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 77fbbd79..1cc38ce9 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -1217,7 +1217,7 @@ where: { await brain.find({ type: 'Document' }) // ✅ Correct: Use NounType enum -import { NounType } from '@soulcraftlabs/brainy' +import { NounType } from '@soulcraft/brainy' await brain.find({ type: NounType.Document }) // ❌ Error: Operator not recognized diff --git a/docs/MIGRATION-V3-TO-V4.md b/docs/MIGRATION-V3-TO-V4.md index 680b6928..29c409ac 100644 --- a/docs/MIGRATION-V3-TO-V4.md +++ b/docs/MIGRATION-V3-TO-V4.md @@ -153,13 +153,13 @@ brainy-data/ ### Step 1: Update Brainy Package ```bash -npm install @soulcraftlabs/brainy@latest +npm install @soulcraft/brainy@latest ``` **Check your version:** ```bash -npm list @soulcraftlabs/brainy -# Should show: @soulcraftlabs/brainy@4.0.0 +npm list @soulcraft/brainy +# Should show: @soulcraft/brainy@4.0.0 ``` ### Step 2: No Code Changes Required! ✅ @@ -374,7 +374,7 @@ If you encounter issues, you can rollback: ```bash # Reinstall v3 -npm install @soulcraftlabs/brainy@^3.50.0 +npm install @soulcraft/brainy@^3.50.0 # Restart application ``` @@ -389,7 +389,7 @@ rm -rf ./data cp -r ./data-backup ./data # Reinstall v3 -npm install @soulcraftlabs/brainy@^3.50.0 +npm install @soulcraft/brainy@^3.50.0 ``` ## Common Migration Scenarios @@ -539,7 +539,7 @@ console.log('Storage type:', status.type) **Migration Checklist:** - ✅ Backup data -- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`) +- ✅ Update npm package (`npm install @soulcraft/brainy@latest`) - ✅ Restart application (automatic migration) - ✅ Verify data integrity - ✅ Enable lifecycle policies diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index b543e84a..248a2c70 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -323,24 +323,58 @@ Only the graph adjacency index carries a committed scale assertion: - ✅ **Single-Node by Design**: One process owns one `path`; scale out at the service layer - ✅ **Zero Stubs**: Every line of code is production-ready -## Index Build at Open (10.4+) +## Lazy Loading Performance -As of 10.4, `brain.init()` runs every needed index rebuild to completion before -it returns — always, regardless of dataset size. There is no lazy, -first-query rebuild path: a brain either finishes opening healthy, or `init()` -fails loudly. `disableAutoRebuild` no longer defers index construction to a -first query; it has no effect on *when* a rebuild runs. Manual control over -rebuilds is `repairIndex({ rebuild: [...] })`. See -[Index Health](concepts/index-health.md) for the full read-gate contract -(providers self-report readiness via `healthReport()`; a read against a -not-serving provider throws a typed `*NotReadyError` rather than rebuilding -mid-query). +Brainy supports two initialization modes for optimal performance across different use cases: - +### Mode 1: Auto-Rebuild (Default) + +```javascript +const brain = new Brainy() +await brain.init() // Rebuilds indexes during init (~500ms-3s for 10K entities) +``` + +**Performance:** +- Init time: 500ms-3s (depends on dataset size) +- First query: Instant (indexes already loaded) +- Use case: Traditional applications, long-running servers + +### Mode 2: Lazy Loading + +```javascript +const brain = new Brainy({ disableAutoRebuild: true }) +await brain.init() // Returns instantly (0-10ms) + +const results = await brain.find({ limit: 10 }) // First query triggers rebuild (~50-200ms) +const more = await brain.find({ limit: 100 }) // Subsequent queries instant (0ms check) +``` + +**Performance:** +- Init time: 0-10ms (instant) +- First query: 50-200ms (includes index rebuild for 1K-10K entities) +- Subsequent queries: 0ms check (instant) +- Concurrent queries: Wait for same rebuild (mutex prevents duplicates) + +**Concurrency Safety:** +```javascript +// 100 concurrent queries immediately after init +await brain.init() + +const promises = Array.from({ length: 100 }, () => + brain.find({ limit: 10 }) +) + +const results = await Promise.all(promises) +// ✅ Only 1 rebuild triggered (mutex) +// ✅ All 100 queries return correct results +// ✅ Total time: ~60ms (not 6000ms!) +``` + +**Use Cases for Lazy Loading:** +- **Serverless/Edge**: Minimize cold start time (0-10ms init) +- **Development**: Faster restarts during development +- **Large datasets**: Defer index loading until needed +- **Read-heavy workloads**: Writes don't wait for index rebuild ## Zero Configuration Required @@ -350,6 +384,10 @@ Brainy is designed to be **smart enough to tune itself dynamically**. No configu // That's it. Brainy handles everything. const brain = new Brainy() await brain.init() + +// Or with lazy loading for serverless +const brain = new Brainy({ disableAutoRebuild: true }) +await brain.init() // Instant (0-10ms) ``` ### Automatic Self-Tuning @@ -357,6 +395,7 @@ await brain.init() - **Metadata Index**: Auto-builds sorted indices for range queries on first use - **Graph Index**: Auto-flushes every 30 seconds - **Default Tuning**: Research-based vector index defaults +- **Lazy Loading**: Indices built only when needed - **Cache Management**: LRU caches with TTL ### Intelligent Defaults diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index 238d2252..d9a4d3e7 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -10,7 +10,7 @@ next: - guides/storage-adapters --- -# Plugin System +# Plugin Development Guide Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cor` provides optional native acceleration, and it's the same system available to any developer. @@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement ### 1. Implement the `BrainyPlugin` interface ```typescript -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' const myPlugin: BrainyPlugin = { name: 'my-brainy-plugin', // Must be unique (typically your npm package name) @@ -90,7 +90,7 @@ await brain.init() **Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`: ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import myPlugin from './my-plugin.js' const brain = new Brainy() @@ -200,30 +200,15 @@ members so a warm reopen never pays a redundant rebuild-from-canonical: - **`init?(): Promise`** — eager cold-load. Brainy awaits it once during `brain.init()`, after the metadata provider's `init()` (the id-mapper hydrates first) and **before the rebuild gate**. -- **`healthReport?(): HealthReport`** — the PREFERRED signal (10.4+). A named, - synchronous, O(1) verdict derived from the provider's own exact ledgers — never a - sample, never I/O, must never throw for a well-formed provider. Brainy's read gate - (`assessProviderHealth()`) reads this INSTEAD of `isReady()` / size heuristics when - present: `serving: false` refuses the read with a typed `*NotReadyError` rather than - triggering a rebuild — a read never starts a store walk. `healthy` marks every - *verified* invariant holding; a family named in `unledgered` counts as neither - healthy nor broken. See `HealthReport` / `LedgerInvariantResult` / - `InvariantSource` in `src/plugin.ts`, and - [Index Health](concepts/index-health.md) for the consumer-facing story. -- **`isReady?(): boolean`** — honest durability signal, the fallback when - `healthReport()` is absent. `true` ⇔ the persisted index is loaded (or cheaply - demand-loadable) and consistent with what was last persisted. When exposed, the - gate defers to this signal **instead of** the `size() === 0` / `totalEntries === 0` - heuristics — a disk-native index may report 0 resident entries while fully durable. - Never return `true` if the durable state failed to load: the signal is honest in - both directions, and a not-ready provider gets its rebuild even when `size() > 0`. +- **`isReady?(): boolean`** — honest durability signal. `true` ⇔ the persisted index is + loaded (or cheaply demand-loadable) and consistent with what was last persisted. When + exposed, the rebuild gate defers to this signal **instead of** the `size() === 0` / + `totalEntries === 0` heuristics — a disk-native index may report 0 resident entries + while fully durable. Never return `true` if the durable state failed to load: the + signal is honest in both directions, and a not-ready provider gets its rebuild even + when `size() > 0`. - **`isMigrating?(): boolean`** — while `true`, the provider owns its index (background migration); brainy skips its rebuild entirely. -- **`validateInvariants?(): Promise`** — the async DEEP - diagnostic (full scans allowed), distinct from the bounded, sync `healthReport()`. - Must never throw — a failure is `healthy: false` data, not an exception; a provider - that throws anyway is read as a loud, unverified failure (never as "healthy") by - every caller, never silently retried into a rebuild. Providers that implement none of these keep the size/count heuristics — correct for engines whose `rebuild()` *is* their load path (like brainy's built-in JS vector index). @@ -272,10 +257,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor #### `cache` **Type:** `UnifiedCache` -Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`). +Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`). ```typescript -import type { UnifiedCache } from '@soulcraftlabs/brainy/internals' +import type { UnifiedCache } from '@soulcraft/brainy/internals' context.registerProvider('cache', myNativeCache) ``` @@ -325,8 +310,8 @@ Plugins can register custom storage backends that users reference by name. ### Implementing a Storage Adapter ```typescript -import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin' -import type { StorageAdapter } from '@soulcraftlabs/brainy' +import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin' +import type { StorageAdapter } from '@soulcraft/brainy' class MyStorageAdapter implements StorageAdapter { async init(): Promise { /* ... */ } @@ -360,9 +345,9 @@ Brainy provides three entry points for plugin developers: | Import Path | Contents | Stability | |-------------|----------|-----------| -| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) | -| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | -| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | +| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) | +| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | +| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | ## Diagnostics @@ -440,7 +425,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations ```typescript // simd-distance-plugin/src/plugin.ts -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' // Hypothetical native module import { simdCosineDistance } from './native.js' @@ -470,7 +455,7 @@ export default simdDistancePlugin "main": "./dist/plugin.js", "types": "./dist/plugin.d.ts", "peerDependencies": { - "@soulcraftlabs/brainy": ">=7.0.0" + "@soulcraft/brainy": ">=7.0.0" } } ``` @@ -478,7 +463,7 @@ export default simdDistancePlugin Usage: ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) await brain.init() diff --git a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md index ad4a4a40..4568cd31 100644 --- a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md +++ b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md @@ -54,7 +54,7 @@ After 40 API calls: ```typescript // server.ts -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // SINGLETON INSTANCE let brainInstance: Brainy | null = null @@ -174,7 +174,7 @@ process.on('SIGTERM', async () => { ```typescript // server.ts - Clean Bun implementation -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain: Brainy | null = null diff --git a/docs/README.md b/docs/README.md index ddb37d20..3290001f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,7 +5,7 @@ ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md index 4b120bd8..94c6a6cb 100644 --- a/docs/RELEASE-GUIDE.md +++ b/docs/RELEASE-GUIDE.md @@ -99,7 +99,7 @@ Examples: ```bash # 1. Deprecate wrong version on npm -npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y" +npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y" # 2. Fix version in package.json # 3. Republish correct version diff --git a/docs/SCALING.md b/docs/SCALING.md index 054d2096..e9ae1136 100644 --- a/docs/SCALING.md +++ b/docs/SCALING.md @@ -13,7 +13,7 @@ ### In-Memory ```typescript -import Brainy from '@soulcraftlabs/brainy' +import Brainy from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) ``` @@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the -open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native +open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`. `find()` query latency, p50 / p95 (200 queries each): diff --git a/docs/api-contract.json b/docs/api-contract.json deleted file mode 100644 index aafd838a..00000000 --- a/docs/api-contract.json +++ /dev/null @@ -1,1544 +0,0 @@ -{ - "contractVersion": 1, - "engine": "@soulcraftlabs/brainy", - "compatibility": { - "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", - "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" - }, - "doors": [ - { - "name": "adaptiveHistoryBudgetBytes", - "kind": "method", - "arity": 1 - }, - { - "name": "add", - "kind": "method", - "arity": 1 - }, - { - "name": "addMany", - "kind": "method", - "arity": 1 - }, - { - "name": "adoptLogAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "adoptLogAuthorityInner", - "kind": "method", - "arity": 0 - }, - { - "name": "aggViewFromEntity", - "kind": "method", - "arity": 1 - }, - { - "name": "anyProviderMigrating", - "kind": "method", - "arity": 0 - }, - { - "name": "applyFusionScoring", - "kind": "method", - "arity": 2 - }, - { - "name": "applyGraphConstraints", - "kind": "method", - "arity": 2 - }, - { - "name": "armIdleFlushTimer", - "kind": "method", - "arity": 2 - }, - { - "name": "asOf", - "kind": "method", - "arity": 2 - }, - { - "name": "assertGenerationStoreReady", - "kind": "method", - "arity": 1 - }, - { - "name": "assertWritable", - "kind": "method", - "arity": 1 - }, - { - "name": "audit", - "kind": "method", - "arity": 0 - }, - { - "name": "auditGraph", - "kind": "method", - "arity": 0 - }, - { - "name": "autoAdoptLegacyVfsBlobsIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "autoAlpha", - "kind": "method", - "arity": 1 - }, - { - "name": "autoCompactHistory", - "kind": "method", - "arity": 0 - }, - { - "name": "awaitMigrationLock", - "kind": "method", - "arity": 1 - }, - { - "name": "awaitPendingEmbeds", - "kind": "method", - "arity": 0 - }, - { - "name": "backfillAggregateIfNeeded", - "kind": "method", - "arity": 1 - }, - { - "name": "batchGet", - "kind": "method", - "arity": 2 - }, - { - "name": "brainWideStrictRequiresSubtype", - "kind": "method", - "arity": 1 - }, - { - "name": "bridgeLegacyPendingEmbedSidecars", - "kind": "method", - "arity": 0 - }, - { - "name": "buildAtGenerationVectors", - "kind": "method", - "arity": 2 - }, - { - "name": "buildGraphView", - "kind": "method", - "arity": 4 - }, - { - "name": "buildMetadataFilter", - "kind": "method", - "arity": 1 - }, - { - "name": "buildMigrationUpdate", - "kind": "method", - "arity": 5 - }, - { - "name": "buildRelationMigrationUpdate", - "kind": "method", - "arity": 5 - }, - { - "name": "cacheVerbInt", - "kind": "method", - "arity": 2 - }, - { - "name": "canServeVectorAtGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "checkHealth", - "kind": "method", - "arity": 0 - }, - { - "name": "checkMigrations", - "kind": "method", - "arity": 0 - }, - { - "name": "clear", - "kind": "method", - "arity": 0 - }, - { - "name": "clearPendingEmbed", - "kind": "method", - "arity": 1 - }, - { - "name": "close", - "kind": "method", - "arity": 0 - }, - { - "name": "closeDurableSteps", - "kind": "method", - "arity": 0 - }, - { - "name": "cluster", - "kind": "method", - "arity": 1 - }, - { - "name": "collectProviderInvariants", - "kind": "method", - "arity": 0 - }, - { - "name": "compactHistory", - "kind": "method", - "arity": 1 - }, - { - "name": "consumeMetadataWatermarkVerdict", - "kind": "method", - "arity": 1 - }, - { - "name": "convertMetadataToEntity", - "kind": "method", - "arity": 2 - }, - { - "name": "convertNounToEntity", - "kind": "method", - "arity": 1 - }, - { - "name": "counts", - "kind": "accessor" - }, - { - "name": "createIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "createMigrationBackupIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "createPinnedDb", - "kind": "method", - "arity": 1 - }, - { - "name": "createResult", - "kind": "method", - "arity": 4 - }, - { - "name": "dbFinalizationRegistry", - "kind": "accessor" - }, - { - "name": "dbHost", - "kind": "accessor" - }, - { - "name": "defineAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "detectIdKind", - "kind": "method", - "arity": 3 - }, - { - "name": "diagnostics", - "kind": "method", - "arity": 0 - }, - { - "name": "diff", - "kind": "method", - "arity": 2 - }, - { - "name": "embed", - "kind": "method", - "arity": 1 - }, - { - "name": "embedBatch", - "kind": "method", - "arity": 2 - }, - { - "name": "emitCommitted", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceSubtypeOnAdd", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceSubtypeOnRelate", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceTrackedFieldValues", - "kind": "method", - "arity": 2 - }, - { - "name": "enhanceNLPResult", - "kind": "method", - "arity": 2 - }, - { - "name": "enqueuePendingEmbed", - "kind": "method", - "arity": 1 - }, - { - "name": "ensureAggregationIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "ensureIndexesLoaded", - "kind": "method", - "arity": 0 - }, - { - "name": "ensureInitialized", - "kind": "method", - "arity": 1 - }, - { - "name": "entityForAggFromRawRecord", - "kind": "method", - "arity": 1 - }, - { - "name": "entityFromGenerationRecord", - "kind": "method", - "arity": 3 - }, - { - "name": "entityIntsToUuids", - "kind": "method", - "arity": 1 - }, - { - "name": "entityViewFromRawRecord", - "kind": "method", - "arity": 2 - }, - { - "name": "excludedVisibilityTiers", - "kind": "method", - "arity": 1 - }, - { - "name": "executeGraphSearch", - "kind": "method", - "arity": 2 - }, - { - "name": "executeProximitySearch", - "kind": "method", - "arity": 1 - }, - { - "name": "executeTextSearch", - "kind": "method", - "arity": 2 - }, - { - "name": "executeVectorSearch", - "kind": "method", - "arity": 3 - }, - { - "name": "explain", - "kind": "method", - "arity": 1 - }, - { - "name": "export", - "kind": "method", - "arity": 0 - }, - { - "name": "extract", - "kind": "method", - "arity": 2 - }, - { - "name": "extractConcepts", - "kind": "method", - "arity": 2 - }, - { - "name": "extractEntities", - "kind": "method", - "arity": 2 - }, - { - "name": "factSegmentPaths", - "kind": "method", - "arity": 1 - }, - { - "name": "fieldCountsAggregateName", - "kind": "method", - "arity": 1 - }, - { - "name": "fillSubtypes", - "kind": "method", - "arity": 1 - }, - { - "name": "filterIdsBelted", - "kind": "method", - "arity": 2 - }, - { - "name": "find", - "kind": "method", - "arity": 1 - }, - { - "name": "findAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "findDuplicates", - "kind": "method", - "arity": 1 - }, - { - "name": "findMatchingWords", - "kind": "method", - "arity": 3 - }, - { - "name": "flush", - "kind": "method", - "arity": 0 - }, - { - "name": "formatInfo", - "kind": "method", - "arity": 0 - }, - { - "name": "formatSubtypeError", - "kind": "method", - "arity": 1 - }, - { - "name": "generation", - "kind": "method", - "arity": 0 - }, - { - "name": "generationDigest", - "kind": "method", - "arity": 1 - }, - { - "name": "get", - "kind": "method", - "arity": 2 - }, - { - "name": "getActivePlugins", - "kind": "method", - "arity": 0 - }, - { - "name": "getAvailableFields", - "kind": "method", - "arity": 0 - }, - { - "name": "getBackgroundDeduplicator", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldsForType", - "kind": "method", - "arity": 1 - }, - { - "name": "getFieldStatistics", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldsWithCardinality", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldValues", - "kind": "method", - "arity": 1 - }, - { - "name": "getIndexStats", - "kind": "method", - "arity": 0 - }, - { - "name": "getIndexStatus", - "kind": "method", - "arity": 0 - }, - { - "name": "getMemoryStats", - "kind": "method", - "arity": 0 - }, - { - "name": "getNeighborUuids", - "kind": "method", - "arity": 2 - }, - { - "name": "getNounCount", - "kind": "method", - "arity": 0 - }, - { - "name": "getOptimalQueryPlan", - "kind": "method", - "arity": 1 - }, - { - "name": "getStats", - "kind": "method", - "arity": 1 - }, - { - "name": "getStorageType", - "kind": "method", - "arity": 0 - }, - { - "name": "getSubtypeRule", - "kind": "method", - "arity": 1 - }, - { - "name": "getTripleIntelligence", - "kind": "method", - "arity": 0 - }, - { - "name": "getTypedNeighbors", - "kind": "method", - "arity": 4 - }, - { - "name": "getVerbCount", - "kind": "method", - "arity": 0 - }, - { - "name": "graph", - "kind": "accessor" - }, - { - "name": "graphAccelerationProvider", - "kind": "method", - "arity": 0 - }, - { - "name": "graphCommunities", - "kind": "method", - "arity": 1 - }, - { - "name": "graphCommunitiesFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphCommunitiesNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphEntityInt", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExport", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExportFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExportNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphPath", - "kind": "method", - "arity": 3 - }, - { - "name": "graphPathFallback", - "kind": "method", - "arity": 3 - }, - { - "name": "graphPathNative", - "kind": "method", - "arity": 4 - }, - { - "name": "graphRank", - "kind": "method", - "arity": 1 - }, - { - "name": "graphRankFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphRankNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphSubgraph", - "kind": "method", - "arity": 2 - }, - { - "name": "graphSubgraphFallback", - "kind": "method", - "arity": 4 - }, - { - "name": "graphSubgraphFromQuery", - "kind": "method", - "arity": 5 - }, - { - "name": "graphSubgraphNative", - "kind": "method", - "arity": 5 - }, - { - "name": "groupByLabel", - "kind": "method", - "arity": 2 - }, - { - "name": "hasStorageMethod", - "kind": "method", - "arity": 1 - }, - { - "name": "hasVectorOrTextCriteria", - "kind": "method", - "arity": 1 - }, - { - "name": "health", - "kind": "method", - "arity": 0 - }, - { - "name": "highlight", - "kind": "method", - "arity": 1 - }, - { - "name": "highlightSemanticPhase", - "kind": "method", - "arity": 5 - }, - { - "name": "history", - "kind": "method", - "arity": 2 - }, - { - "name": "historyStats", - "kind": "method", - "arity": 0 - }, - { - "name": "hub", - "kind": "accessor" - }, - { - "name": "hydrateIdMapperForGraphRebuild", - "kind": "method", - "arity": 0 - }, - { - "name": "hydrateNativeSubgraph", - "kind": "method", - "arity": 2 - }, - { - "name": "import", - "kind": "method", - "arity": 2 - }, - { - "name": "importPluginPackage", - "kind": "method", - "arity": 1 - }, - { - "name": "incidentEdges", - "kind": "method", - "arity": 3 - }, - { - "name": "indexStats", - "kind": "method", - "arity": 0 - }, - { - "name": "init", - "kind": "method", - "arity": 1 - }, - { - "name": "insights", - "kind": "method", - "arity": 0 - }, - { - "name": "isEmbeddingReady", - "kind": "method", - "arity": 0 - }, - { - "name": "isInfrastructureWrite", - "kind": "method", - "arity": 1 - }, - { - "name": "isInitialized", - "kind": "accessor" - }, - { - "name": "isReadOnly", - "kind": "accessor" - }, - { - "name": "kickBackgroundFlush", - "kind": "method", - "arity": 1 - }, - { - "name": "kickEmbedWorker", - "kind": "method", - "arity": 0 - }, - { - "name": "legacyLayoutMigrationPhase", - "kind": "method", - "arity": 0 - }, - { - "name": "loadAnalyticsGraph", - "kind": "method", - "arity": 1 - }, - { - "name": "loadPlugins", - "kind": "method", - "arity": 0 - }, - { - "name": "logAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "maintenanceDebt", - "kind": "method", - "arity": 0 - }, - { - "name": "materializeAtGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "metadataIndexRetractionOp", - "kind": "method", - "arity": 3 - }, - { - "name": "migrate", - "kind": "method", - "arity": 1 - }, - { - "name": "migrateField", - "kind": "method", - "arity": 1 - }, - { - "name": "migrateInternal", - "kind": "method", - "arity": 2 - }, - { - "name": "migrateLegacyZeroNormVfsRootIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "migrationSnapshot", - "kind": "method", - "arity": 0 - }, - { - "name": "neededFamiliesMigrating", - "kind": "method", - "arity": 1 - }, - { - "name": "neighbors", - "kind": "method", - "arity": 2 - }, - { - "name": "newId", - "kind": "method", - "arity": 0 - }, - { - "name": "nlp", - "kind": "method", - "arity": 0 - }, - { - "name": "normalizeConfig", - "kind": "method", - "arity": 1 - }, - { - "name": "noteWriteForPersistence", - "kind": "method", - "arity": 0 - }, - { - "name": "now", - "kind": "method", - "arity": 0 - }, - { - "name": "onChange", - "kind": "method", - "arity": 1 - }, - { - "name": "pagination", - "kind": "accessor" - }, - { - "name": "parseMigrationPath", - "kind": "method", - "arity": 1 - }, - { - "name": "parseNaturalQuery", - "kind": "method", - "arity": 1 - }, - { - "name": "pathExists", - "kind": "method", - "arity": 2 - }, - { - "name": "pendingEmbedCount", - "kind": "method", - "arity": 0 - }, - { - "name": "performInit", - "kind": "method", - "arity": 1 - }, - { - "name": "persistPinnedGeneration", - "kind": "method", - "arity": 2 - }, - { - "name": "persistSingleOp", - "kind": "method", - "arity": 6 - }, - { - "name": "pickMetadataProbe", - "kind": "method", - "arity": 1 - }, - { - "name": "pickVectorProbe", - "kind": "method", - "arity": 0 - }, - { - "name": "pinGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "planGetEntity", - "kind": "method", - "arity": 3 - }, - { - "name": "planTransact", - "kind": "method", - "arity": 1 - }, - { - "name": "planTxAdd", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxRelate", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxRemove", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxUnrelate", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxUpdate", - "kind": "method", - "arity": 3 - }, - { - "name": "projectionGauges", - "kind": "method", - "arity": 0 - }, - { - "name": "providerForFamily", - "kind": "method", - "arity": 1 - }, - { - "name": "providerIsMigrating", - "kind": "method", - "arity": 1 - }, - { - "name": "providerMigrationStatus", - "kind": "method", - "arity": 0 - }, - { - "name": "queryAggregate", - "kind": "method", - "arity": 2 - }, - { - "name": "queryIndexFamilies", - "kind": "method", - "arity": 1 - }, - { - "name": "readPath", - "kind": "method", - "arity": 2 - }, - { - "name": "ready", - "kind": "accessor" - }, - { - "name": "rebuildIndexesIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "rebuildMetadataIndexOnline", - "kind": "method", - "arity": 0 - }, - { - "name": "reconcileLogDivergence", - "kind": "method", - "arity": 2 - }, - { - "name": "reconstructPath", - "kind": "method", - "arity": 4 - }, - { - "name": "recordStateAt", - "kind": "method", - "arity": 3 - }, - { - "name": "recoverPendingEmbedsFromLog", - "kind": "method", - "arity": 0 - }, - { - "name": "registerShutdownHooks", - "kind": "method", - "arity": 0 - }, - { - "name": "relate", - "kind": "method", - "arity": 1 - }, - { - "name": "related", - "kind": "method", - "arity": 1 - }, - { - "name": "relateMany", - "kind": "method", - "arity": 1 - }, - { - "name": "relationFromGenerationRecord", - "kind": "method", - "arity": 2 - }, - { - "name": "relationshipSubtypesOf", - "kind": "method", - "arity": 1 - }, - { - "name": "releaseGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "remove", - "kind": "method", - "arity": 1 - }, - { - "name": "removeAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "removeMany", - "kind": "method", - "arity": 1 - }, - { - "name": "removeMigrationBackupSafe", - "kind": "method", - "arity": 0 - }, - { - "name": "repackHistory", - "kind": "method", - "arity": 1 - }, - { - "name": "repairIndex", - "kind": "method", - "arity": 1 - }, - { - "name": "requestFlush", - "kind": "method", - "arity": 1 - }, - { - "name": "requireProviders", - "kind": "method", - "arity": 1 - }, - { - "name": "requireSubtype", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveAsOfGeneration", - "kind": "method", - "arity": 2 - }, - { - "name": "resolveDiffEndpoint", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveHiddenIds", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveHNSWPersistMode", - "kind": "method", - "arity": 0 - }, - { - "name": "resolveRawGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveRetentionPolicy", - "kind": "method", - "arity": 0 - }, - { - "name": "resolveVerbEndpointInts", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveVerbIntsToIds", - "kind": "method", - "arity": 1 - }, - { - "name": "restore", - "kind": "method", - "arity": 2 - }, - { - "name": "rrfFusion", - "kind": "method", - "arity": 4 - }, - { - "name": "runAggregationBackfillWalk", - "kind": "method", - "arity": 0 - }, - { - "name": "runAggregationCatchUp", - "kind": "method", - "arity": 0 - }, - { - "name": "runEmbedWorker", - "kind": "method", - "arity": 0 - }, - { - "name": "runOracle", - "kind": "method", - "arity": 1 - }, - { - "name": "runRepairIndexPhases", - "kind": "method", - "arity": 5 - }, - { - "name": "scanFacts", - "kind": "method", - "arity": 1 - }, - { - "name": "seedIdsToInts", - "kind": "method", - "arity": 1 - }, - { - "name": "selectorToSeedIds", - "kind": "method", - "arity": 1 - }, - { - "name": "setRetentionBudget", - "kind": "method", - "arity": 1 - }, - { - "name": "setupEmbedder", - "kind": "method", - "arity": 0 - }, - { - "name": "setupIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "setupStorage", - "kind": "method", - "arity": 0 - }, - { - "name": "similar", - "kind": "method", - "arity": 1 - }, - { - "name": "similarity", - "kind": "method", - "arity": 2 - }, - { - "name": "splitForHighlighting", - "kind": "method", - "arity": 2 - }, - { - "name": "stampBrainFormat", - "kind": "method", - "arity": 0 - }, - { - "name": "stampBrainFormatIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "stampEntityTree", - "kind": "method", - "arity": 0 - }, - { - "name": "stampProjectionWatermarks", - "kind": "method", - "arity": 0 - }, - { - "name": "stats", - "kind": "method", - "arity": 0 - }, - { - "name": "storageAdapter", - "kind": "accessor" - }, - { - "name": "stream", - "kind": "method", - "arity": 0 - }, - { - "name": "streaming", - "kind": "accessor" - }, - { - "name": "subtypesOf", - "kind": "method", - "arity": 1 - }, - { - "name": "trackField", - "kind": "method", - "arity": 1 - }, - { - "name": "transact", - "kind": "method", - "arity": 2 - }, - { - "name": "transactionLog", - "kind": "method", - "arity": 1 - }, - { - "name": "unrelate", - "kind": "method", - "arity": 1 - }, - { - "name": "unvectorNounForRootMigration", - "kind": "method", - "arity": 1 - }, - { - "name": "update", - "kind": "method", - "arity": 1 - }, - { - "name": "updateMany", - "kind": "method", - "arity": 1 - }, - { - "name": "updateRelation", - "kind": "method", - "arity": 1 - }, - { - "name": "upsertMergeParams", - "kind": "method", - "arity": 2 - }, - { - "name": "use", - "kind": "method", - "arity": 1 - }, - { - "name": "usesDefaultWasmEmbedder", - "kind": "method", - "arity": 0 - }, - { - "name": "validateIndexConsistency", - "kind": "method", - "arity": 0 - }, - { - "name": "vectorSearchAtGeneration", - "kind": "method", - "arity": 4 - }, - { - "name": "verbsToRelations", - "kind": "method", - "arity": 1 - }, - { - "name": "verbToRelationLike", - "kind": "method", - "arity": 1 - }, - { - "name": "verifyEntityTreeStamp", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyGraphAdjacencyLive", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyLogAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyMetadataLive", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyVectorLive", - "kind": "method", - "arity": 0 - }, - { - "name": "versionedIndexProviders", - "kind": "method", - "arity": 0 - }, - { - "name": "vfs", - "kind": "accessor" - }, - { - "name": "waitForIndexed", - "kind": "method", - "arity": 2 - }, - { - "name": "warm", - "kind": "method", - "arity": 0 - }, - { - "name": "warmupEmbeddings", - "kind": "method", - "arity": 0 - }, - { - "name": "warnIfReadsDegraded", - "kind": "method", - "arity": 1 - }, - { - "name": "wireConnectionsCodec", - "kind": "method", - "arity": 0 - }, - { - "name": "wireGraphIdResolver", - "kind": "method", - "arity": 0 - } - ], - "errors": [ - "BrainyError", - "DerivedArtifactMissingError", - "GraphIndexNotReadyError", - "MetadataIndexNotReadyError", - "MigrationInProgressError", - "ProtectedArtifactError", - "VectorIndexNotReadyError" - ], - "operators": { - "accepted": [ - "between", - "contains", - "endsWith", - "eq", - "equals", - "excludes", - "exists", - "greaterThan", - "greaterThanOrEqual", - "gt", - "gte", - "hasAll", - "in", - "length", - "lessThan", - "lessThanOrEqual", - "lt", - "lte", - "matches", - "missing", - "ne", - "noneOf", - "notEquals", - "oneOf", - "startsWith" - ], - "servedOnIndexPath": [ - "between", - "contains", - "eq", - "equals", - "excludes", - "exists", - "greaterThan", - "greaterThanOrEqual", - "gt", - "gte", - "hasAll", - "in", - "lessThan", - "lessThanOrEqual", - "lt", - "lte", - "missing", - "ne", - "noneOf", - "notEquals", - "oneOf" - ], - "refusedByIndexPath": [ - "endsWith", - "length", - "matches", - "startsWith" - ], - "combinators": [ - "allOf", - "anyOf", - "not" - ] - }, - "fieldAddressing": { - "systemKeyPrefix": "system.", - "systemEntityScalars": [ - "confidence", - "createdAt", - "createdBy", - "id", - "service", - "subtype", - "type", - "updatedAt", - "visibility", - "weight" - ], - "systemRelationScalars": [ - "confidence", - "createdAt", - "createdBy", - "service", - "sourceId", - "subtype", - "targetId", - "updatedAt", - "verb", - "visibility", - "weight" - ], - "plumbingFields": [ - "_rev", - "connections", - "data", - "level", - "vector" - ] - }, - "health": { - "verdicts": [ - "pass", - "warn", - "fail" - ], - "healKinds": [ - "none", - "repair", - "rebuild" - ], - "servingWithholdingInvariants": [ - "index-initialized", - "durable-state-present", - "manifest-residency", - "replay-clean", - "strand-latch" - ] - } -} diff --git a/docs/api/README.md b/docs/api/README.md index ba49ff48..4ca84364 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -24,7 +24,7 @@ next: ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() // Zero config! await brain.init() // VFS auto-initialized! @@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization ### Db API errors -All exported from `@soulcraftlabs/brainy`: +All exported from `@soulcraft/brainy`: | Error | Thrown by | Meaning | |---|---|---| @@ -1451,34 +1451,6 @@ const count = await brain.getVerbCount() --- -### The canonical count ledger (`StorageAdapter.getCanonicalCounts()`) - -An OPTIONAL method on the `StorageAdapter` interface (implemented by both -built-in adapters), not a method on `Brainy` itself — relevant if you're -writing a custom storage adapter or composing a provider's own -`healthReport()`. O(1), no I/O. Per family (`nouns`/`verbs`): - -```typescript -interface CanonicalCounts { - nouns: { counted: number; all: number } - verbs: { counted: number; all: number } - suspect: boolean -} -``` - -- `counted` mirrors `getNounCount()` / `getVerbCount()` (public + internal tiers). -- `all` is the ALL-visibility scalar — every tier, including system/internal - records — the denominator a derived index's own coverage math is measured - against. -- `suspect` is `true` when an unprovable delete has left `all` unverified since - the last recount; `brain.repairIndex()` clears it with a real canonical walk. - -Adapters without the ledger omit the method; treat absence as "no -denominator," never as zero. See -**[Index Health](../concepts/index-health.md)** for the full story. - ---- - ### Subtype & facet APIs Full guide: **[Subtypes & Facets](../guides/subtypes-and-facets.md)**. @@ -1859,104 +1831,6 @@ const semanticOnly = await brain.getStats({ excludeVFS: true }) --- -### `repairIndex(options?)` → `Promise` - -The ceremony door for index repair. Bare `repairIndex()` is report-driven: it -prunes orphaned containers, recomputes count rollups, reconciles VFS -containment, and rebuilds only a derived-index family whose own health check -asks for it. Pass `options.rebuild` to force one or more families to rebuild -UNCONDITIONALLY — no health check is consulted — when an operator has -independent reason to reconcile a family regardless of what it self-reports. - -```typescript -// Report-driven: only heals what actually needs it -const report = await brain.repairIndex() -console.log(report.healedTotal, report.families) - -// Explicit: force the graph adjacency to rebuild from canonical, unconditionally -await brain.repairIndex({ rebuild: ['graph'] }) - -// Explicit: force all three derived indexes to rebuild -await brain.repairIndex({ rebuild: 'all' }) -``` - -**`RepairReport`:** -- `families: RepairFamilyReport[]` — one row per family checked -- `healedTotal: number` — items healed across every family -- `durationMs: number` - -**`RepairFamilyReport`** (one row): -- `family: string` — e.g. `'orphaned-containers'`, `'count-rollups'`, - `'vfs-containment'`, `'metadata-corruption'`, `'provider:metadata'`, - `'provider:graph'`, `'provider:vector'` -- `checked: boolean` — was this family actually examined (`false` ⇒ see `skipped`) -- `healed: number` — items re-posted/corrected in place (the incremental heal count) -- `missing?: { count: number; sample: string[] }` — exact count plus a capped id - sample when the check can name what diverged (never the full list) -- `rebuilt?: boolean` — a full generational rebuild ran (vs. an incremental heal) -- `detail?: string` / `reason?: string` — narration -- `skipped?: string` — why the family wasn't checked - -Full walkthrough — what each family checks, degraded-but-serving vs. not-ready, -and what `suspect` counts mean — in -**[Index Health](../concepts/index-health.md)**. - ---- - -### Index readiness: typed errors, `healthReport()`, `disableAutoRebuild` - -Every derived-index provider (vector, graph, metadata) may expose a named, -synchronous, O(1) `healthReport()` composed from its own exact ledgers — the -signal Brainy's read gate trusts over sampling or size heuristics. `init()` -brings every provider to serving before it returns; there is no first-query -lazy-rebuild path. A read that reaches a provider whose health report says it -isn't serving throws instead of rebuilding mid-query: - -| Error | Thrown by | Meaning | -|---|---|---| -| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | Graph adjacency isn't serving | -| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | -| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | - -All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish -"index not ready" from a genuine empty result: - -```typescript -import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy' - -try { - const rows = await brain.find({ where: { status: 'active' } }) -} catch (err) { - if (err instanceof MetadataIndexNotReadyError) { - // reconcile: await brain.repairIndex(), then retry - } else { - throw err - } -} -``` - -**`disableAutoRebuild`** no longer defers index construction to the first -query. A needed rebuild always runs at `open()`, regardless of this flag or -dataset size; the flag has no effect on *when* a rebuild runs. Full manual -control lives in `repairIndex({ rebuild: [...] })`, above. - -### `validateIndexConsistency()` → `Promise<...>` - -The deep, async diagnostic counterpart to `healthReport()` — safe to run on a -live brain, but does more work (a provider's `validateInvariants()` may run a -full scan, not just read a ledger). Aggregates the JS metadata index's own -consistency check with every derived-index provider's invariant report. - -```typescript -const validation = await brain.validateIndexConsistency() -if (!validation.healthy) { - console.log(validation.recommendation) // what to run, e.g. repairIndex() - console.log(validation.providers) // each provider's own invariant report, when exposed -} -``` - ---- - ## Lifecycle ### Initialization @@ -2208,7 +2082,7 @@ For the full taxonomy with all 169 types and their descriptions, see: - **📖 Documentation:** [Full Documentation](../) - **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues) - **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions) -- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy) +- **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy) - **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy) --- diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 83b9e23a..48064757 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with .ack | **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) | A pluggable index provider (the 8.0 plugin contract in -`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the +`@soulcraft/brainy/plugin`) may replace any of the JS implementations; the persisted formats above are contract-bound so JS and native implementations can interleave on the same directory. diff --git a/docs/architecture/finite-type-system.md b/docs/architecture/finite-type-system.md index 48a8b1fe..76492ee5 100644 --- a/docs/architecture/finite-type-system.md +++ b/docs/architecture/finite-type-system.md @@ -126,7 +126,7 @@ class TypeAwareMetadataIndex { **The Design**: Specify types clearly in your API calls: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' // Add entity with explicit type await brain.add({ @@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation { **Brainy's Approach**: Extract **typed** concepts: ```typescript -import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy' +import { NaturalLanguageProcessor } from '@soulcraft/brainy' const nlp = new NaturalLanguageProcessor() const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco") @@ -382,7 +382,7 @@ import { getVerbTypes, BrainyTypes, suggestType -} from '@soulcraftlabs/brainy' +} from '@soulcraft/brainy' // Get all available noun types const nounTypes = getNounTypes() diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md index 6a754b56..8b3dc540 100644 --- a/docs/architecture/index-architecture.md +++ b/docs/architecture/index-architecture.md @@ -723,14 +723,6 @@ async stats(): Promise { ### 5. Index Rebuilding (Lazy Loading Support) -> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is -> RETIRED.** `disableAutoRebuild` no longer defers index construction to a -> first query; `brain.init()` now runs every needed rebuild to completion -> before it returns, unconditionally, and a read against a not-serving -> provider throws a typed `*NotReadyError` instead of rebuilding mid-query. -> See `docs/concepts/index-health.md` for the current contract. Left below -> as historical background on the rebuild mechanics. - **Two modes of index loading:** #### Mode 1: Auto-Rebuild on init() (default) diff --git a/docs/architecture/initialization-and-rebuild.md b/docs/architecture/initialization-and-rebuild.md index e19bdd9f..a1645744 100644 --- a/docs/architecture/initialization-and-rebuild.md +++ b/docs/architecture/initialization-and-rebuild.md @@ -1,15 +1,5 @@ # Initialization and Rebuild Processes -> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is RETIRED.** -> `disableAutoRebuild` no longer defers index construction to a first query; -> `brain.init()` now runs every needed rebuild to completion before it -> returns, unconditionally. A read against a not-serving provider throws a -> typed `*NotReadyError` instead of rebuilding mid-query. See -> `docs/concepts/index-health.md` for the current contract; this document's -> line-number references to `src/brainy.ts` also predate the file's current -> size and are unreliable. Left as historical background on the rebuild -> mechanics, not as a current API description. - This document explains how Brainy's four indexes (MetadataIndex, vector index, GraphAdjacencyIndex, DeletedItemsIndex) initialize and rebuild from persisted storage. ## Core Principle: All Indexes Are Disk-Based diff --git a/docs/architecture/multiprocess-storage-mixin.md b/docs/architecture/multiprocess-storage-mixin.md index 1593bf8f..46f98398 100644 --- a/docs/architecture/multiprocess-storage-mixin.md +++ b/docs/architecture/multiprocess-storage-mixin.md @@ -127,7 +127,7 @@ For reference, a clean migration path: `isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for build/install artifact protection. 5. Document the new contract in `concepts/storage-adapters.md`. -6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by +6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by plugins. Estimated work: ~half a day of code, ~2 hours of doc/example updates, diff --git a/docs/architecture/noun-verb-taxonomy.md b/docs/architecture/noun-verb-taxonomy.md index 3dac6892..286464be 100644 --- a/docs/architecture/noun-verb-taxonomy.md +++ b/docs/architecture/noun-verb-taxonomy.md @@ -20,7 +20,7 @@ next: Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu - **Multi-hop Graph Traversals = Relationship Complexity** - **Result: Model data across virtually any industry** -Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. +Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. ## The Power of Standardization: Universal Interoperability diff --git a/docs/architecture/zero-config.md b/docs/architecture/zero-config.md index a35e6416..d42d6784 100644 --- a/docs/architecture/zero-config.md +++ b/docs/architecture/zero-config.md @@ -35,7 +35,7 @@ constructor and `init()`. ## Instant Start ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // That's it. No config needed. const brain = new Brainy() diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md index d24dd66b..c459021b 100644 --- a/docs/concepts/field-addressing.md +++ b/docs/concepts/field-addressing.md @@ -167,7 +167,7 @@ await brain.find({ orderBy: 'createdAt' }) `UnresolvableFieldError` is exported from the package root: ```typescript -import { UnresolvableFieldError } from '@soulcraftlabs/brainy' +import { UnresolvableFieldError } from '@soulcraft/brainy' try { await brain.find({ orderBy: 'createdAt' }) diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md deleted file mode 100644 index 923267df..00000000 --- a/docs/concepts/index-health.md +++ /dev/null @@ -1,217 +0,0 @@ ---- -title: Index Health -slug: concepts/index-health -public: true -category: concepts -template: concept -order: 8 -description: How Brainy knows whether a derived index can be trusted — exact accounting instead of sampling, the named health report, degraded-but-serving vs. not-ready, and what repairIndex() checks, heals, and rebuilds. -next: - - concepts/generation-fact-log - - guides/inspection ---- - -# Index Health - -Brainy keeps one **canonical** copy of every entity and relationship, and three -**derived** indexes built from it — vector, metadata, and graph — so `find()` can -answer semantically, by filter, and by traversal without re-deriving the answer from -scratch on every query. A derived index is a cache with a serving structure: it can -be present but stale, present but only partially loaded, or fully out of sync with -canonical after a crash. This page is about how Brainy decides whether to trust one, -what it does when it can't, and how you reconcile the two. - -## Exact accounting instead of sampling - -Older health checks worked by inference: does `size()` return something greater -than zero, does a spot-check on one known item come back correct. Both are proxies. -A cold index can report a nonzero count while its actual serving structure never -loaded, and a spot-check only proves the one item it happened to ask about. - -Every derived-index provider may now expose a named, synchronous, O(1) -`healthReport()` — composed from the provider's own **exact ledgers** (real counters -it already maintains on the write path), never a sample or a walk. This is the one -signal Brainy's read gate consults. A provider that doesn't yet expose one falls -back to an honest `isReady()` boolean, and finally to a size heuristic for engines -with neither — but wherever a `healthReport()` exists, it wins. - -Underneath, storage itself keeps an analogous **canonical count ledger**: a -`counted` scalar (the user-facing total — what `getNounCount()` / `getVerbCount()` -return) and an `all` scalar (every tier, including internal records a derived -index's own coverage math needs to compare against). This is the real denominator -a provider's `healthReport()` measures itself by, rather than a total that can only -ever ratchet upward. See [What `suspect` counts mean](#what-suspect-counts-mean) -below for the one case that ledger can't stay exact through on its own. - -## The named report - -A `HealthReport` carries, per provider (`'vector'` / `'graph'` / `'metadata'`): - -- **`healthy`** — `true` iff every *verified* invariant holds. An invariant whose - family has no ledger yet is `unledgered`, never counted either way — unknown, - not passing. -- **`serving`** — can this provider answer a query right now. A failing invariant - graded `heal: 'repair'` or `heal: 'none'` still leaves `serving: true` — this is - **degraded-but-serving**: something is off (say, a stale rollup on an - `employee` record's relationship count) but reads keep working. Only a failure - graded `heal: 'rebuild'` flips `serving` to `false` — **not-ready** — because the - provider itself is telling you its serving structure cannot answer correctly. -- **`invariants`** — each checked condition, with its provenance - (`source: 'ledger'` — an exact count; `'deep'` — a full scan, diagnostic-only; - `'unledgered'` — not yet tracked) and, for a failing one, an exact `missing` - count plus a capped sample of the affected ids — a verdict, never a dump. -- **`generation`** — bumps on every ledger mutation and rebuild, so a caller can - cache a verdict per generation instead of re-deriving it. - -The distinction that matters day to day: `healthy: false` can be entirely benign — -a maintenance window, a divergence `repairIndex()` will clean up on its own -schedule. `serving: false` is not benign. It means this provider is refusing to -answer, on its own word, right now. - -**How a failure gets its grade — the serving law.** A provider grades `heal` by -one question only: *could an answer be wrong?* — never *how expensive is the -fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post -exactly what the ledger names, reads serving throughout); it can never withhold -serving just because healing it takes work. `serving` is withheld only by a -small, named set of rebuild-graded conditions — the index not initialized, its -durable state absent, a manifest naming files that are not resident, a replay -that did not complete cleanly — the states in which an answer could genuinely be -wrong. And a read is only ever refused by the family it actually consults: a -metadata filter is answered by the metadata index alone, vector search by the -vector index, traversal by the graph index — one family's refusal never blocks -another family's reads. - -## Reads refuse — they never rebuild - -A query that reaches a not-serving provider does not trigger a rebuild from inside -the read. Brainy retired that path deliberately: a rebuild kicked off by an ordinary -`find({ where: { status: 'active' } })` call is a dark, unpredictable cost hiding -behind a request that looks like a cheap read. Instead, the read throws a typed, -catchable error naming the reason: - -| Error | Thrown when | Meaning | -|---|---|---| -| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | The graph adjacency index isn't serving — traversal would otherwise return `[]` indistinguishable from "no relationships" | -| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | -| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | - -All three are exported from `@soulcraftlabs/brainy`. Catch them where your application -needs to distinguish "this index isn't ready yet" from "there's genuinely nothing -here" — a health dashboard, a retry policy, an operator alert. The fix is always -the same: reconcile the index, either by reopening the brain (which brings every -provider to serving before `init()` returns — see the next section) or by calling -`repairIndex()` explicitly. - -```typescript -try { - const active = await brain.find({ where: { status: 'active' } }) -} catch (err) { - if (err instanceof MetadataIndexNotReadyError) { - // not a "no results" — the index itself refused; alert or retry after repair - } else { - throw err - } -} -``` - -### Rebuilds happen at open, not on first query - -`brain.init()` runs every needed rebuild to completion **before it returns**, -unconditionally, regardless of dataset size. There is no lazy, first-query -rebuild path anymore — a brain either finishes opening healthy, or it fails -open loudly. `disableAutoRebuild: true` no longer defers index construction to -the first query: it has no effect on *when* a needed rebuild runs. Full manual -control over rebuilds is `repairIndex({ rebuild: [...] })` (below), not this flag. - -## `repairIndex()` — checking and healing - -Bare `repairIndex()` is **report-driven**: it only heals what its own checks say -actually needs it, and it always returns a full per-family receipt. - -```typescript -const report = await brain.repairIndex() -report.healedTotal // total items healed across every family -report.durationMs -report.families // one row per family checked -``` - -Each `RepairFamilyReport` row names what happened: - -- **`checked`** — was this family actually examined (`false` means skipped — - see `skipped` for why). -- **`healed`** — items re-posted or corrected in place. -- **`missing`** — when the check can name what diverged: an exact `count` plus a - capped `sample` of ids. -- **`rebuilt`** — a full generational rebuild ran (as opposed to an incremental - heal). -- **`detail`** / **`reason`** / **`skipped`** — the receipt's narration; a row is - always either checked or explains why it wasn't. Nothing is silent. - -On every call, bare `repairIndex()`: - -1. Prunes orphaned canonical containers left by a partial delete. -2. Recomputes the count rollups from one canonical walk (unconditional — this is - also what clears a `suspect` ledger; see below). -3. Reconciles VFS containment edges, if the VFS is initialized. -4. Runs the metadata index's own corruption detection pass. -5. Consults each of the three derived-index providers' own health check and - rebuilds only a family whose failing invariant actually asks for it - (`heal: 'rebuild'`) — never a provider that reports `healthy` or a lesser - grade. - -### The explicit rebuild door - -`options.rebuild` skips the health check and rebuilds one or more families -**unconditionally** — the operator override for when you have independent reason -to distrust a family regardless of what it self-reports (a suspicious deploy, a -storage-layer incident, a support ticket that doesn't match what the health report -says): - -```typescript -// Force the graph adjacency to rebuild from canonical, no invariant consulted -await brain.repairIndex({ rebuild: ['graph'] }) - -// Force all three derived indexes -await brain.repairIndex({ rebuild: 'all' }) -``` - -A family named this way is recorded with `rebuilt: true` and -`reason: 'explicit rebuild requested'`, and is skipped by the normal -health-driven pass in the same call — it was already rebuilt unconditionally. - -Reach for the explicit door when you need certainty regardless of self-report; -reach for bare `repairIndex()` for routine maintenance and after any incident -where you're not sure which family (if any) needs it. - -## What `suspect` counts mean - -Storage's canonical count ledger increments the ALL-visibility total on every new -record and decrements it on every *proven* delete — one where the record was read, -or the caller supplied its prior image. A delete that cannot prove what it removed -existed doesn't guess: it flags the ledger `suspect` (an operator-visible -`console.warn`, narrated once per session, not once per delete) rather than risk -decrementing a total that was never incremented for that record in the first -place. This is intentionally rare — it's a defensive fallback for callers on an -unusual removal path, not a per-delete cost. - -`suspect` is not directly exposed on any `Brainy` method today — it lives on the -`StorageAdapter`'s optional `getCanonicalCounts()`, primarily consulted by -`repairIndex()`'s recount step and by custom storage adapters composing their own -`healthReport()`. What matters for an application: a `suspect` ledger is not -incorrect, just *unverified since the last recount* — and `repairIndex()`'s -unconditional count-rollup step (step 2, above) recomputes the ALL scalars from a -real canonical walk on every call, clearing the flag with proof either way. - -## Practical guidance - -- **On a normal restart**, do nothing — `init()` brings every provider to - serving before it returns, or fails loudly. -- **On a `*NotReadyError`** from a live read, reconcile with `repairIndex()` - (report-driven is almost always sufficient) and retry. -- **After an incident** where you distrust a specific family regardless of what - it reports healthy — a storage-layer fault, a suspicious restore — use the - explicit door: `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] })`. -- **To audit before trusting a report**, `brain.auditGraph()` walks every stored - relationship and proves (or disproves) that reads return canonical truth, - independent of what any provider self-reports — see - [Inspecting a Live Brainy](../guides/inspection.md). diff --git a/docs/concepts/storage-adapters.md b/docs/concepts/storage-adapters.md index 82aa01e8..af6d068f 100644 --- a/docs/concepts/storage-adapters.md +++ b/docs/concepts/storage-adapters.md @@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from to call `acquireWriterLock()` at init. ```typescript -import { FileSystemStorage } from '@soulcraftlabs/brainy' +import { FileSystemStorage } from '@soulcraft/brainy' export class MmapFileSystemStorage extends FileSystemStorage { public supportsMultiProcessLocking(): boolean { @@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom network backend), extend `BaseStorage` directly: ```typescript -import { BaseStorage } from '@soulcraftlabs/brainy' +import { BaseStorage } from '@soulcraft/brainy' export class MyCloudStorage extends BaseStorage { // BaseStorage's default no-op implementations of the multi-process @@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`, `hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM import (verify in your plugin's `dist/`: `import { FileSystemStorage } from -'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype +'@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype chain at runtime resolves to whatever Brainy version your consumer has installed. @@ -109,8 +109,8 @@ installed. the prototype chain at the consumer-app level: - **Stale `node_modules`** — a lingering install from before the consumer - upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but - `node_modules/@soulcraftlabs/brainy` is still 7.20.x. + upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but + `node_modules/@soulcraft/brainy` is still 7.20.x. - **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy version older than the package.json range, and `bun install` honors the lockfile. @@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint: methods on its prototype chain. Writer locking and the flush-request RPC are disabled for this directory. Likely fix: clean install (`rm -rf node_modules bun.lockb && bun install`) or rebuild your container image to refresh -`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. +`@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. ``` ## Authoring a new storage adapter — minimum checklist @@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh install time — fix install, not your plugin. 6. **Pin your peer dep generously.** `"peerDependencies": { - "@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin + "@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin to an exact patch unless you're tracking a known regression. ## Future direction @@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change. heartbeat semantics, what the lock protects. - [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the read-only mode. -- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the +- `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the authoritative type signatures for every method this page references. diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index 616c8fc4..11d86ec8 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -22,7 +22,7 @@ they share a single scan. ## Quick Start ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/framework-integration.md b/docs/guides/framework-integration.md index 8f85da00..984466c5 100644 --- a/docs/guides/framework-integration.md +++ b/docs/guides/framework-integration.md @@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server: -- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'` +- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'` - **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node - **Cleaner code**: No browser polyfills, no conditional client/server imports - **Better DX**: One instance shared across your server routes @@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers ### Install Brainy ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Basic Integration ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Run on the server (API route, server component, backend service) // new Brainy() auto-detects filesystem persistence on Node @@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod ```javascript // lib/brain.server.js -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests: ```javascript // server/brain.js (server-only module) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun): ```typescript // server: api/search -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() // auto-detects filesystem persistence on Node await brain.init() @@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or ```javascript // lib/brain.server.js (imported only by server code) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul ```javascript // src/lib/server/brain.js (server-only — note the .server suffix) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -432,7 +432,7 @@ import { defineConfig } from 'vite' export default defineConfig({ ssr: { - external: ['@soulcraftlabs/brainy'] + external: ['@soulcraft/brainy'] } }) ``` @@ -440,7 +440,7 @@ export default defineConfig({ ```javascript // rollup.config.js (server bundle) export default { - external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto'] + external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto'] } ``` @@ -466,7 +466,7 @@ export async function load({ url }) { ```javascript // For build-time usage (runs in Node during the build) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' export async function generateStaticProps() { const brain = new Brainy({ @@ -513,7 +513,7 @@ export async function generateStaticProps() { ### Issue: Large client bundle size **Cause**: A client module is pulling in Brainy. -**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle. +**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle. ### Issue: SSR hydration mismatch **Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup. diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index ffabe55c..b1bb15ef 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING: ## The Ultimate Simplicity ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md index 18c3cb9a..66f50713 100644 --- a/docs/guides/import-progress-examples.md +++ b/docs/guides/import-progress-examples.md @@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats** ### Basic Progress Tracking ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import * as fs from 'fs' const brain = await Brainy.create() diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 3bc26dae..7837d49e 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -7,7 +7,7 @@ ## Basic Import ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -187,7 +187,7 @@ await brain.import(file, { ## Complete Example ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import * as fs from 'fs' async function importCatalog() { diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 8560b543..240e81ae 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI. ## Programmatic inspection ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 20d40ea2..0a36f632 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -21,21 +21,21 @@ next: ## Install ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` Or with your preferred package manager: ```bash -bun add @soulcraftlabs/brainy -yarn add @soulcraftlabs/brainy -pnpm add @soulcraftlabs/brainy +bun add @soulcraft/brainy +yarn add @soulcraft/brainy +pnpm add @soulcraft/brainy ``` ## Verify ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -52,7 +52,7 @@ npm install @soulcraft/cor ``` ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) await brain.init() // native providers registered during init @@ -71,7 +71,7 @@ remains available on npm if you need it. Brainy ships with full TypeScript types. No `@types/` package needed: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/migration-3.36.0.md b/docs/guides/migration-3.36.0.md index 5f00534a..8b1f239e 100644 --- a/docs/guides/migration-3.36.0.md +++ b/docs/guides/migration-3.36.0.md @@ -66,7 +66,7 @@ const results = await brain.search("query") **New diagnostics for capacity planning and performance tuning.** ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')} ### Step 1: Update Package ```bash -npm install @soulcraftlabs/brainy@latest +npm install @soulcraft/brainy@latest ``` ### Step 2: Restart Your Application @@ -134,7 +134,7 @@ npm run start ### Check Adaptive Sizing is Working ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -218,7 +218,7 @@ For debugging or compatibility testing: If you need to rollback to v3.35.0: ```bash -npm install @soulcraftlabs/brainy@3.35.0 +npm install @soulcraft/brainy@3.35.0 ``` **Note:** We don't anticipate any issues, but rollback is straightforward if needed. @@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) { ## Next Steps -1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest` +1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest` 2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements 3. 🎯 **Tune:** Adjust based on recommendations (if needed) 4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning diff --git a/docs/guides/model-loading.md b/docs/guides/model-loading.md index cc1b2b6a..e5b7b1d6 100644 --- a/docs/guides/model-loading.md +++ b/docs/guides/model-loading.md @@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings. ```bash # Bun as a runtime — supported and recommended -bun add @soulcraftlabs/brainy +bun add @soulcraft/brainy bun run server.ts ``` diff --git a/docs/guides/namespace-migration.md b/docs/guides/namespace-migration.md index f7d2c7f7..fad3c766 100644 --- a/docs/guides/namespace-migration.md +++ b/docs/guides/namespace-migration.md @@ -80,7 +80,7 @@ If you read raw stored records (fact-log scanners, export tooling), use the exported shape-aware splitters — they handle both record eras: ```typescript -import { splitNounMetadataRecord } from '@soulcraftlabs/brainy' +import { splitNounMetadataRecord } from '@soulcraft/brainy' const { reserved, custom } = splitNounMetadataRecord(rawRecord) // reserved = engine fields · custom = the user's bag, ANY names ``` @@ -88,7 +88,7 @@ const { reserved, custom } = splitNounMetadataRecord(rawRecord) Feature detection (never version-sniff): ```typescript -import * as brainy from '@soulcraftlabs/brainy' +import * as brainy from '@soulcraft/brainy' const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1' ``` diff --git a/docs/guides/nextjs-integration.md b/docs/guides/nextjs-integration.md index 25d6062d..ab55e51f 100644 --- a/docs/guides/nextjs-integration.md +++ b/docs/guides/nextjs-integration.md @@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou ```bash npx create-next-app@latest my-brainy-app cd my-brainy-app -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Basic Setup @@ -18,7 +18,7 @@ npm install @soulcraftlabs/brainy // app/components/BrainyProvider.jsx 'use client' import { createContext, useContext, useEffect, useState } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const BrainyContext = createContext() @@ -271,7 +271,7 @@ export default function SearchPage() { ```javascript // app/api/search/route.js (App Router) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -332,7 +332,7 @@ export async function GET() { ```javascript // pages/api/search.js (Pages Router) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -374,7 +374,7 @@ export default async function handler(req, res) { ```javascript // app/api/data/route.js -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -418,7 +418,7 @@ export async function POST(request) { ```jsx // app/actions/brainy.js 'use server' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brain = null @@ -630,7 +630,7 @@ CMD ["npm", "start"] /** @type {import('next').NextConfig} */ const nextConfig = { experimental: { - serverComponentsExternalPackages: ['@soulcraftlabs/brainy'] + serverComponentsExternalPackages: ['@soulcraft/brainy'] }, webpack: (config, { isServer }) => { if (!isServer) { @@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) { // app/contexts/BrainyContext.jsx 'use client' import { createContext, useContext, useReducer, useEffect } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const BrainyContext = createContext() @@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider' import { Search } from '../app/components/Search' // Mock Brainy -jest.mock('@soulcraftlabs/brainy', () => ({ +jest.mock('@soulcraft/brainy', () => ({ Brainy: jest.fn().mockImplementation(() => ({ init: jest.fn().mockResolvedValue(undefined), find: jest.fn().mockResolvedValue([ diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index 2984998b..268bc5fa 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin Every distributed-job scheduler eventually wants this exact loop: ```ts -import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy' +import { Brainy, RevisionConflictError } from '@soulcraft/brainy' const LOCK_ID = '...uuid for this job slot...' @@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved: ```ts -import { GenerationConflictError } from '@soulcraftlabs/brainy' +import { GenerationConflictError } from '@soulcraft/brainy' async function addIfMissingByEmail(email: string, data: string) { for (let attempt = 0; attempt < 5; attempt++) { diff --git a/docs/guides/quick-start.md b/docs/guides/quick-start.md index d9a4e896..097c55fe 100644 --- a/docs/guides/quick-start.md +++ b/docs/guides/quick-start.md @@ -18,13 +18,13 @@ Get Brainy running in under a minute. ## 1. Install ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ## 2. Initialize ```typescript -import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' +import { Brainy, NounType, VerbType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -67,7 +67,7 @@ await brain.relate({ ## 5. Query with Triple Intelligence ```typescript -import type { Result } from '@soulcraftlabs/brainy' +import type { Result } from '@soulcraft/brainy' // All three search paradigms in one call const results: Result[] = await brain.find({ diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md index 27dabe75..9f2e2e5b 100644 --- a/docs/guides/standard-import-progress.md +++ b/docs/guides/standard-import-progress.md @@ -11,7 +11,7 @@ ### One Interface for Everything ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = await Brainy.create() @@ -78,7 +78,7 @@ interface ImportProgress { ```typescript import { useState } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' function UniversalImportProgress({ file }: { file: File }) { const [progress, setProgress] = useState({ @@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) { ```typescript import ora from 'ora' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' async function importWithProgress(filePath: string) { const spinner = ora('Starting import...').start() diff --git a/docs/guides/storage-adapters.md b/docs/guides/storage-adapters.md index a4224bc8..06ec9f3a 100644 --- a/docs/guides/storage-adapters.md +++ b/docs/guides/storage-adapters.md @@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map). ## Quick start ```ts -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Filesystem (recommended for any persistent workload): const brain = new Brainy({ @@ -134,7 +134,7 @@ config; the `type` is optional. If you want to skip the factory: ```ts -import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy' +import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy' const fsStorage = new FileSystemStorage('./brainy-data') const memStorage = new MemoryStorage() diff --git a/docs/guides/subtypes-and-facets.md b/docs/guides/subtypes-and-facets.md index 74311528..ff5de320 100644 --- a/docs/guides/subtypes-and-facets.md +++ b/docs/guides/subtypes-and-facets.md @@ -34,7 +34,7 @@ Three layers solve this: ### Write ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -240,7 +240,7 @@ await brain.migrateField({ A realistic adoption sequence for a brain that started without these primitives: ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } }) await brain.init() diff --git a/docs/guides/upgrading-7-to-8.md b/docs/guides/upgrading-7-to-8.md index 53aa2a5c..a3c64fb9 100644 --- a/docs/guides/upgrading-7-to-8.md +++ b/docs/guides/upgrading-7-to-8.md @@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you. ## TL;DR -- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.** +- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.** If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, with no operator action. - Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. @@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and open it.** ```ts -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Opening the store is all that is required — recovery runs during init(). const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) @@ -182,5 +182,5 @@ and opening each store is sufficient. The recovery is copy-only, so no rollback of the recovery itself is ever needed. If you need to roll back the **whole** 7→8 upgrade, restore the directory from your pre-upgrade backup (retained automatically while recovery is incomplete, or -your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old +your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old branch layout in place, so a directory-level restore is the rollback path. diff --git a/docs/guides/vue-integration.md b/docs/guides/vue-integration.md index 34d18ebf..7f7c6a06 100644 --- a/docs/guides/vue-integration.md +++ b/docs/guides/vue-integration.md @@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N npm create vue@latest my-brainy-app cd my-brainy-app npm install -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ### Basic Setup @@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun ```javascript // server/utils/brain.js (server-only — Nitro never bundles this into the client) -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' let brainPromise @@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue' export default defineConfig({ plugins: [vue()], ssr: { - external: ['@soulcraftlabs/brainy'] + external: ['@soulcraft/brainy'] } }) ``` diff --git a/docs/neural-extraction.md b/docs/neural-extraction.md index cfb6d764..989b1b60 100644 --- a/docs/neural-extraction.md +++ b/docs/neural-extraction.md @@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c ### Method 1: Brain Instance (Recommended) ```typescript -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', { import { SmartExtractor, SmartRelationshipExtractor -} from '@soulcraftlabs/brainy' +} from '@soulcraft/brainy' // Or use subpath imports: -import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor' +import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor' const brain = new Brainy() await brain.init() @@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, { **Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration. ```typescript -import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy' +import { SmartExtractor, FormatContext } from '@soulcraft/brainy' const extractor = new SmartExtractor(brain, { minConfidence: 0.7, // Threshold @@ -229,7 +229,7 @@ interface ExtractionResult { **Relationship type classifier.** Determines verb/relationship types between entities. ```typescript -import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' +import { SmartRelationshipExtractor } from '@soulcraft/brainy' const relExtractor = new SmartRelationshipExtractor(brain, { minConfidence: 0.6, @@ -286,7 +286,7 @@ const rel = await relExtractor.infer( **Full extraction orchestrator.** Handles candidate detection, classification, and deduplication. ```typescript -import { NeuralEntityExtractor } from '@soulcraftlabs/brainy' +import { NeuralEntityExtractor } from '@soulcraft/brainy' const extractor = new NeuralEntityExtractor(brain) @@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location) ### Example 2: Excel Data Classification ```typescript -import { SmartExtractor } from '@soulcraftlabs/brainy' +import { SmartExtractor } from '@soulcraft/brainy' const extractor = new SmartExtractor(brain) @@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) { ### Example 3: Relationship Extraction ```typescript -import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' +import { SmartRelationshipExtractor } from '@soulcraft/brainy' const relExtractor = new SmartRelationshipExtractor(brain) diff --git a/docs/transactions.md b/docs/transactions.md index cbea39c0..fce7d10e 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing }) ### Basic Add Operation ```typescript -import { Brainy } from '@soulcraftlabs/brainy' -import { NounType } from '@soulcraftlabs/brainy/types' +import { Brainy } from '@soulcraft/brainy' +import { NounType } from '@soulcraft/brainy/types' const brain = new Brainy() await brain.init() @@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked ```typescript import { describe, it, expect } from 'vitest' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' describe('Transaction Tests', () => { it('should rollback on failure', async () => { diff --git a/docs/universal-display-augmentation.md b/docs/universal-display-augmentation.md index 464b91fb..da42874c 100644 --- a/docs/universal-display-augmentation.md +++ b/docs/universal-display-augmentation.md @@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati ### Basic Usage ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brainy = new Brainy() await brainy.init() diff --git a/docs/vfs/PROJECTION_STRATEGY_API.md b/docs/vfs/PROJECTION_STRATEGY_API.md index f1319d5b..380862e1 100644 --- a/docs/vfs/PROJECTION_STRATEGY_API.md +++ b/docs/vfs/PROJECTION_STRATEGY_API.md @@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low): ### Step 1: Create the Strategy Class ```typescript -import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' -import { Brainy } from '@soulcraftlabs/brainy' -import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs' +import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' +import { Brainy } from '@soulcraft/brainy' +import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs' export class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' @@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy { ### Step 2: Register the Strategy ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import { PriorityProjection } from './PriorityProjection' const brain = new Brainy() @@ -537,7 +537,7 @@ Use the projection's resolve cache: ```typescript import { describe, it, expect, beforeAll } from 'vitest' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import { PriorityProjection } from './PriorityProjection' describe('PriorityProjection', () => { @@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) { 3. Use appropriate limits: Don't fetch more than needed ### Type errors -1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'` +1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'` 2. Use `as VFSEntity` when mapping results 3. Check BaseProjectionStrategy import diff --git a/docs/vfs/QUICK_START.md b/docs/vfs/QUICK_START.md index 4a1f83dc..8b0efce6 100644 --- a/docs/vfs/QUICK_START.md +++ b/docs/vfs/QUICK_START.md @@ -14,11 +14,11 @@ A file explorer that: ## ⚡ Step 1: Basic Setup (1 minute) ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // ✅ CORRECT: Use filesystem storage for production const brain = new Brainy({ @@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns: ```tsx import React, { useState, useEffect } from 'react' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' export function FileExplorer() { const [brain, setBrain] = useState(null) @@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next: ### "Module not found" errors ```bash # Make sure you're using the right import -npm ls @soulcraftlabs/brainy # Check version -npm install @soulcraftlabs/brainy@latest # Update if needed +npm ls @soulcraft/brainy # Check version +npm install @soulcraft/brainy@latest # Update if needed ``` ### "VFS not initialized" errors diff --git a/docs/vfs/README.md b/docs/vfs/README.md index a94910c9..b95f0d7b 100644 --- a/docs/vfs/README.md +++ b/docs/vfs/README.md @@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne ## Quick Start ```javascript -import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs' +import { VirtualFileSystem } from '@soulcraft/brainy/vfs' // Initialize the VFS const vfs = new VirtualFileSystem({ @@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system: ## Installation ```bash -npm install @soulcraftlabs/brainy +npm install @soulcraft/brainy ``` ## Requirements diff --git a/docs/vfs/ROADMAP.md b/docs/vfs/ROADMAP.md index c8d15cd2..93c5b901 100644 --- a/docs/vfs/ROADMAP.md +++ b/docs/vfs/ROADMAP.md @@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows. ```typescript // Planned (research phase) -import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse' +import { mountVFS } from '@soulcraft/brainy/vfs/fuse' await mountVFS(vfs, { mountPoint: '/mnt/brainy', @@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested ### Express.js Static Middleware ```typescript // Wanted: Community contribution -import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express' +import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express' app.use('/files', createStaticMiddleware(vfs, { index: ['index.html', 'index.md'], @@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, { ### VSCode Extension ```typescript // Wanted: Community contribution -import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode' +import { VFSProvider } from '@soulcraft/brainy/vfs/vscode' const provider = new VFSProvider(vfs) vscode.workspace.registerFileSystemProvider('brainy', provider) diff --git a/docs/vfs/SEMANTIC_VFS.md b/docs/vfs/SEMANTIC_VFS.md index f34ee9ae..9298c822 100644 --- a/docs/vfs/SEMANTIC_VFS.md +++ b/docs/vfs/SEMANTIC_VFS.md @@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true Create your own semantic dimensions: ```typescript -import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' +import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' diff --git a/docs/vfs/VFS_API_GUIDE.md b/docs/vfs/VFS_API_GUIDE.md index 5dcaaeb8..e0c6a94c 100644 --- a/docs/vfs/VFS_API_GUIDE.md +++ b/docs/vfs/VFS_API_GUIDE.md @@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha ## Quick Start ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Initialize Brainy const brain = new Brainy({ @@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123') VFS uses standard POSIX-style errors: ```typescript -import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy' +import { VFSError, VFSErrorCode } from '@soulcraft/brainy' try { await vfs.readFile('/nonexistent.txt') diff --git a/docs/vfs/VFS_CORE.md b/docs/vfs/VFS_CORE.md index c1d502c0..1eeaf9f8 100644 --- a/docs/vfs/VFS_CORE.md +++ b/docs/vfs/VFS_CORE.md @@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities: #### GitBridge Usage ```javascript // Import and instantiate GitBridge -import { GitBridge } from '@soulcraftlabs/brainy' +import { GitBridge } from '@soulcraft/brainy' const gitBridge = new GitBridge(vfs, brain) // Export VFS to Git repository structure @@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare ## Complete Example ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' async function vfsExample() { // Initialize diff --git a/docs/vfs/VFS_GRAPH_TYPES.md b/docs/vfs/VFS_GRAPH_TYPES.md index 478bef7f..3c1f30f0 100644 --- a/docs/vfs/VFS_GRAPH_TYPES.md +++ b/docs/vfs/VFS_GRAPH_TYPES.md @@ -196,5 +196,5 @@ await brain.relate({ Always import and use the type enums: ```javascript -import { NounType, VerbType } from '@soulcraftlabs/brainy' +import { NounType, VerbType } from '@soulcraft/brainy' ``` \ No newline at end of file diff --git a/docs/vfs/VFS_INITIALIZATION.md b/docs/vfs/VFS_INITIALIZATION.md index fd12fc71..97e6b0bf 100644 --- a/docs/vfs/VFS_INITIALIZATION.md +++ b/docs/vfs/VFS_INITIALIZATION.md @@ -5,7 +5,7 @@ The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed! ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' // Create and initialize Brainy const brain = new Brainy({ @@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e ## Complete Example ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' async function useVFS() { // Initialize Brainy @@ -100,7 +100,7 @@ useVFS().catch(console.error) ## TypeScript Usage ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' class FileManager { private brain: Brainy diff --git a/docs/vfs/building-file-explorers.md b/docs/vfs/building-file-explorers.md index 7514c12e..6bb31871 100644 --- a/docs/vfs/building-file-explorers.md +++ b/docs/vfs/building-file-explorers.md @@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues: ### Method 1: Use `getDirectChildren()` (Recommended) ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' const brain = new Brainy() await brain.init() @@ -97,7 +97,7 @@ Here's a complete example using React: ```tsx import React, { useState, useEffect } from 'react' -import { VirtualFileSystem } from '@soulcraftlabs/brainy' +import { VirtualFileSystem } from '@soulcraft/brainy' interface FileNode { name: string @@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) { If you must build trees manually from flat lists, use the `VFSTreeUtils`: ```typescript -import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs' +import { VFSTreeUtils } from '@soulcraft/brainy/vfs' // Get all entities somehow const allEntities = await vfs.getDescendants('/root') diff --git a/examples/bluesky-distributed-setup.js b/examples/bluesky-distributed-setup.js index e3b33506..9e83cf25 100644 --- a/examples/bluesky-distributed-setup.js +++ b/examples/bluesky-distributed-setup.js @@ -7,7 +7,7 @@ * the Bluesky firehose with Brainy's distributed architecture */ -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' import { WebSocket } from 'ws' // ===================================================== diff --git a/examples/monitor-cache-performance.ts b/examples/monitor-cache-performance.ts index 87c965a2..9d50d476 100644 --- a/examples/monitor-cache-performance.ts +++ b/examples/monitor-cache-performance.ts @@ -14,7 +14,7 @@ * ts-node examples/monitor-cache-performance.ts */ -import { Brainy, NounType } from '@soulcraftlabs/brainy' +import { Brainy, NounType } from '@soulcraft/brainy' // ANSI color codes for pretty output const colors = { diff --git a/integrations/README.md b/integrations/README.md index de156623..aa3d795b 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu ## Quick Start ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature. ### Minimal (in-memory): ```typescript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions()) ```typescript import express from 'express' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const app = express() const brain = new Brainy({ @@ -232,7 +232,7 @@ app.listen(3000, () => { ```typescript import { Hono } from 'hono' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const app = new Hono() diff --git a/integrations/google-sheets/README.md b/integrations/google-sheets/README.md index 8309a30a..b2b0af3a 100644 --- a/integrations/google-sheets/README.md +++ b/integrations/google-sheets/README.md @@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings. The simplest way to enable all integrations: ```javascript -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -112,7 +112,7 @@ With Express: ```javascript import express from 'express' -import { Brainy } from '@soulcraftlabs/brainy' +import { Brainy } from '@soulcraft/brainy' const app = express() const brain = new Brainy({ integrations: true }) diff --git a/package-lock.json b/package-lock.json index c4f66561..8cee0038 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "name": "@soulcraft/brainy", + "version": "10.4.0-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "name": "@soulcraft/brainy", + "version": "10.4.0-rc.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 06ce0253..ef803c11 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,6 @@ { - "name": "@soulcraftlabs/brainy", - "version": "10.4.4", - "brainyContract": 1, + "name": "@soulcraft/brainy", + "version": "10.4.0-rc.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", @@ -127,16 +126,15 @@ "license": "MIT", "private": false, "publishConfig": { - "access": "public", - "registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" + "access": "public" }, - "homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy", + "homepage": "https://source.soulcraft.com/soulcraft/brainy", "bugs": { - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues" + "url": "https://source.soulcraft.com/soulcraft/brainy/issues" }, "repository": { "type": "git", - "url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git" + "url": "git+https://source.soulcraft.com/soulcraft/brainy.git" }, "files": [ "dist/**/*.js", diff --git a/scripts/buildEmbeddedPatterns.ts b/scripts/buildEmbeddedPatterns.ts index c046df45..73e51224 100644 --- a/scripts/buildEmbeddedPatterns.ts +++ b/scripts/buildEmbeddedPatterns.ts @@ -10,7 +10,6 @@ import { TransformerEmbedding } from '../src/utils/embedding.js' import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' -import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -98,22 +97,13 @@ async function buildEmbeddedPatterns() { // Convert to base64 for embedding in TypeScript const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - - // Deterministic stamp: derived from the git commit time of this - // generator's inputs, never from wall-clock time — two builds of the - // same source tree must produce byte-identical output. - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') - const generatedStamp = resolveDeterministicStamp( - [path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath], - outputPath - ) - + // Generate TypeScript file with everything embedded const tsContent = `/** * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${generatedStamp} + * Generated: ${new Date().toISOString()} * Patterns: ${libraryData.patterns.length} * Coverage: 94-98% of all queries * @@ -207,6 +197,7 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length} ` // Write the TypeScript file + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/buildTypeEmbeddings.ts b/scripts/buildTypeEmbeddings.ts index 688d6ac1..61bcf238 100644 --- a/scripts/buildTypeEmbeddings.ts +++ b/scripts/buildTypeEmbeddings.ts @@ -11,7 +11,6 @@ import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' import { NounType, VerbType } from '../src/types/graphTypes.js' -import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -374,24 +373,12 @@ async function buildTypeEmbeddings() { const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - // Deterministic stamp: derived from the git commit time of this - // generator's inputs, never from wall-clock time — two builds of the - // same source tree must produce byte-identical output. - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') - const generatedStamp = resolveDeterministicStamp( - [ - path.join(__dirname, 'buildTypeEmbeddings.ts'), - path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts') - ], - outputPath - ) - // Generate TypeScript file const tsContent = `/** * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${generatedStamp} + * Generated: ${new Date().toISOString()} * Noun Types: ${nounTypes.length} * Verb Types: ${verbTypes.length} * @@ -408,7 +395,7 @@ export const TYPE_METADATA = { verbTypes: ${verbTypes.length}, totalTypes: ${totalTypes}, embeddingDimensions: ${embeddingDim}, - generatedAt: "${generatedStamp}", + generatedAt: "${new Date().toISOString()}", sizeBytes: { embeddings: ${buffer.byteLength}, base64: ${base64.length} @@ -507,6 +494,7 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n ` // Write the TypeScript file + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs deleted file mode 100644 index be73d4ca..00000000 --- a/scripts/emit-contract-manifest.mjs +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env node -/** - * Emit this build's API-contract manifest to docs/api-contract.json. - * - * WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the - * code the first time somebody adds one. This reads the surface the build - * actually exposes — the prototype's own methods and accessors, the exported - * error classes, the `where` operator sets, the field-addressing vocabulary, - * the health verdicts — so a diff between two engines' manifests is a diff - * between two engines, never between two authors. - * - * Requirement marking (required / optional per door) is NOT derivable from the - * surface — it is a commitment, recorded with the contract's owner rather than - * here. This manifest carries the surface; the promise lives with the contract. - * - * Usage: node scripts/emit-contract-manifest.mjs [--check] - * --check exits non-zero when the committed manifest is stale. - */ - -import { writeFileSync, readFileSync, existsSync } from 'node:fs' -import { join, dirname } from 'node:path' -import { fileURLToPath } from 'node:url' - -const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..') -const OUT = join(ROOT, 'docs', 'api-contract.json') - -const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js')) -const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js')) -const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js')) -const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js')) - -/** Every own method and accessor on the class's prototype, minus the private ones. */ -function surfaceOf(ctor) { - const doors = [] - for (const name of Object.getOwnPropertyNames(ctor.prototype)) { - if (name === 'constructor' || name.startsWith('_')) continue - const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name) - if (!descriptor) continue - if (typeof descriptor.value === 'function') { - doors.push({ name, kind: 'method', arity: descriptor.value.length }) - } else if (descriptor.get) { - doors.push({ name, kind: 'accessor' }) - } - } - return doors.sort((a, b) => a.name.localeCompare(b.name)) -} - -const errors = Object.entries(errorsModule) - .filter(([name, value]) => typeof value === 'function' && /Error$/.test(name)) - .map(([name]) => name) - .sort() - -// The operator sets, read from the engine's own refusal message so the -// manifest can never disagree with the validator. -const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8') -const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set\(\[([\s\S]*?)\]\)/) -if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess') -const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() - -const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8') -const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) => - // Proven by the refusal path: these are the tokens with no case in the - // index's operator switch, so they fall to its default and are refused. - !new RegExp(`case '${op}':`).test(indexSource) -) -const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) - -const manifest = { - contractVersion: versionModule.contractVersion(), - engine: '@soulcraftlabs/brainy', - compatibility: { - minor: - 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', - major: - 'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused' - }, - doors: surfaceOf(Brainy), - errors, - operators: { - accepted, - servedOnIndexPath: servedOnIndex, - refusedByIndexPath: refusedByIndex, - combinators: ['allOf', 'anyOf', 'not'] - }, - fieldAddressing: { - systemKeyPrefix: 'system.', - systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(), - systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(), - plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort() - }, - health: { - verdicts: ['pass', 'warn', 'fail'], - healKinds: ['none', 'repair', 'rebuild'], - servingWithholdingInvariants: [ - 'index-initialized', - 'durable-state-present', - 'manifest-residency', - 'replay-clean', - 'strand-latch' - ] - } -} - -const rendered = `${JSON.stringify(manifest, null, 2)}\n` - -if (process.argv.includes('--check')) { - if (!existsSync(OUT)) { - console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`) - process.exit(1) - } - if (readFileSync(OUT, 'utf-8') !== rendered) { - console.error( - `docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` + - `the addition (minor = additive; a removal is a contract major).` - ) - process.exit(1) - } - console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`) - process.exit(0) -} - -writeFileSync(OUT, rendered) -console.log( - `Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` + - `${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` + - `${manifest.operators.accepted.length} operators ` + - `(${manifest.operators.refusedByIndexPath.length} refused by the index path).` -) diff --git a/scripts/lib/deterministicStamp.ts b/scripts/lib/deterministicStamp.ts deleted file mode 100644 index c2a66604..00000000 --- a/scripts/lib/deterministicStamp.ts +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Deterministic generation-stamp resolution for Brainy's build-time code - * generators. - * - * Two builds of the same source tree must produce byte-identical output. - * A wall-clock stamp (`new Date()`) breaks that guarantee, so every - * generator that writes a "Generated:" header or a `generatedAt` field - * into its output must resolve the stamp through this module instead. - * - * Resolution order: - * 1. The newest git commit timestamp among the generator's input files - * (the generator script itself always counts as an input). - * 2. If git metadata is unavailable (for example, building from a - * published npm tarball with no `.git` directory), the stamp already - * recorded in the previously generated output file. - * 3. If neither is available, the fixed epoch string - * `1970-01-01T00:00:00.000Z`. - * - * Every fallback logs a line to stderr — deterministic degradation is - * loud, never a silent divergence. - */ - -import { execFileSync } from 'child_process' -import * as fs from 'fs' - -const EPOCH_STAMP = '1970-01-01T00:00:00.000Z' -const STAMP_PATTERN = /\*\s*Generated:\s*(\S+)/ - -/** - * Resolve the deterministic stamp for a generator run. - * - * @param inputPaths Absolute paths to every file whose content determines - * the generator's output, including the generator script itself. - * @param previousOutputPath Absolute path to the previously generated - * file, used for the existing-stamp fallback when git is unavailable. - * @returns An ISO-8601 timestamp string that is deterministic for a given - * source tree. - */ -export function resolveDeterministicStamp( - inputPaths: string[], - previousOutputPath: string -): string { - const gitStamp = newestGitCommitTimestamp(inputPaths) - if (gitStamp) { - return gitStamp - } - - const existingStamp = readExistingStamp(previousOutputPath) - if (existingStamp) { - process.stderr.write( - `[deterministic-stamp] no git commit history found for generator inputs; ` + - `reusing existing stamp from ${previousOutputPath}: ${existingStamp}\n` - ) - return existingStamp - } - - process.stderr.write( - `[deterministic-stamp] no git commit history and no previous output at ` + - `${previousOutputPath}; falling back to fixed epoch stamp ${EPOCH_STAMP}\n` - ) - return EPOCH_STAMP -} - -/** - * Find the newest git commit timestamp among the given input paths. - * Returns null if git is unavailable, the tree is not a git repository, - * or none of the inputs have any commit history yet. - */ -function newestGitCommitTimestamp(inputPaths: string[]): string | null { - let newest: string | null = null - - for (const inputPath of inputPaths) { - if (!fs.existsSync(inputPath)) { - continue - } - - let out: string - try { - out = execFileSync( - 'git', - ['log', '-1', '--format=%cI', '--', inputPath], - { stdio: ['ignore', 'pipe', 'ignore'] } - ) - .toString() - .trim() - } catch { - // git missing, not a repository, or no permissions — handled by the - // caller's fallback chain. - continue - } - - if (!out) { - // Path exists but has no commit history yet (e.g. newly created, - // uncommitted file). - continue - } - - if (!newest || new Date(out).getTime() > new Date(newest).getTime()) { - newest = out - } - } - - return newest -} - -/** - * Parse the `* Generated: ` header out of a previously - * generated file, if one exists. - */ -function readExistingStamp(outputPath: string): string | null { - if (!fs.existsSync(outputPath)) { - return null - } - - const content = fs.readFileSync(outputPath, 'utf-8') - const match = content.match(STAMP_PATTERN) - return match ? match[1] : null -} diff --git a/scripts/release.sh b/scripts/release.sh index 142fa06f..5d03e66b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -15,11 +15,11 @@ NC='\033[0m' # No Color RELEASE_TYPE="${1:-patch}" # patch, minor, or major SKIP_TESTS=false DRY_RUN=false -# --source-only is now a no-op: The Source is the one registry, so every -# release already ships Source-only — tag, CI's publish to The Source, the -# release page, and the docs push, with no separate storefront leg to skip. -# The flag is still accepted (for backward-compatible invocations) and just -# prints a notice; it no longer changes behavior. +# --source-only: the HOME leg only — tag, CI's publish to The Source, and the +# release page; NO storefront (npmjs) publish, NO pair verification, NO docs +# push. The pair-gate shape: a prerelease the fleet's other engine devDeps +# from our own registry while the pair is proven, never a public artifact. +# Refused for a non-prerelease version — a public floor is always a pair. SOURCE_ONLY=false for arg in "$@"; do @@ -109,7 +109,7 @@ else ;; *) echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" - echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run] [--source-only (no-op; The Source is the one registry)]" + echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run] [--source-only (prereleases only)]" exit 1 ;; esac @@ -129,7 +129,11 @@ if [ "$PRERELEASE" = true ]; then echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}" fi if [ "$SOURCE_ONLY" = true ]; then - echo -e "${YELLOW}⚠️ The Source is the one registry; --source-only is implied${NC}" + if [ "$PRERELEASE" != true ]; then + echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}" + exit 1 + fi + echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}" fi echo "" @@ -154,8 +158,7 @@ else fi # Create new changelog entry -RELEASE_DATE=$(date +%Y-%m-%d) -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE}) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) ${COMMITS} " @@ -175,19 +178,6 @@ if [ -f "CHANGELOG.md" ]; then fi echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" -# Step 6b: Update the releases wall entry — mechanical, derived from the -# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json -# from the one shared releases repo, soulcraftlabs/releases on The Source — -# this used to be hand-written after every release (David: never again — -# make it a step of the rail, landed in the one shared home; this repo no -# longer hosts its own copy). This step clones/fetches that repo into a -# local cache, prepends the entry, and pushes it directly — a real -# cross-repo push, refusing loudly (never skipping) on any -# clone/validation/commit/push failure. -echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}" -node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md -echo -e "${GREEN}✅ Releases wall updated${NC}\n" - # Step 7: Create release commit echo -e "${BLUE}6️⃣ Creating release commit...${NC}" git add package.json package-lock.json CHANGELOG.md @@ -219,9 +209,9 @@ echo -e "${GREEN}✅ Pushed to origin${NC}\n" # .forgejo/workflows/publish-source.yml, which builds and publishes on The # Source's own runner (datacenter-side: seconds, not the laptop's WAN timing # out on an 87MB tarball PUT). The laptop holds no home-registry publish -# credential anymore; it only waits for CI's result before continuing on to -# the release page and the docs push. -SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" +# credential anymore; it only waits for CI's result before trusting the +# home/npmjs pair enough to publish the storefront leg. +SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" SOURCE_POLL_INTERVAL_S=15 SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml # backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0); @@ -229,7 +219,7 @@ SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequen echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}" SOURCE_LANDED=false for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do - LANDED_VERSION=$(npm view "@soulcraftlabs/brainy@${NEW_VERSION}" version "--@soulcraftlabs:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") + LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then SOURCE_LANDED=true break @@ -242,16 +232,62 @@ if [ "$SOURCE_LANDED" = true ]; then echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n" else echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" - echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraftlabs/brainy@${NEW_VERSION} never became visible on the${NC}" - echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting.${NC}" + echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}" + echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}" exit 1 fi +if [ "$SOURCE_ONLY" = true ]; then + echo -e "${YELLOW}9️⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n" +else + echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" + # BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download + # the tarball The Source serves and publish that file, never a fresh local pack + # (a local rebuild can differ byte-wise, and the fleet verifies the pair by + # shasum across registries). + STOREFRONT_TMP="$(mktemp -d)" + (cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null) + SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)" + echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}" + npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" + rm -rf "$STOREFRONT_TMP" + # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. + npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true + # Verify the pair is byte-identical by registry-reported shasum — divergence + # here means the storefront leg must be treated as failed, loudly. RETRIED + # with raw curl: npmjs metadata propagates with a lag measured in minutes, + # and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a + # raw curl of the registry document already confirmed byte-identity. The + # probe now reads the registry JSON directly (no npm cache in the path) and + # gives propagation up to 5 minutes before calling the pair divergent. + NPMJS_VERIFY_ATTEMPTS=20 + NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace + SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable") + PAIR_IDENTICAL=false + for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do + NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \ + | node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \ + || echo "") + if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then + PAIR_IDENTICAL=true + break + fi + echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}" + sleep "$NPMJS_VERIFY_INTERVAL_S" + done + if [ "$PAIR_IDENTICAL" = true ]; then + echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n" + else + echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n" + exit 1 + fi +fi + # Step 11: Release object on The Source (presentational — the tag, CHANGELOG, # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then - if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then echo -e "${GREEN}✅ Release page created on The Source${NC}\n" @@ -262,15 +298,29 @@ else echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): -# soulcraft.com/docs carries the paid product's documentation only. This -# engine's documentation home is THIS repository — README and docs/ — and the -# site serves 301s for the slugs this rail used to push. The push script stays -# in the tree for history; the rail no longer calls it. -echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" +# Step 12: Push public docs to the soulcraft.com docs ingest door +# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when +# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — +# that already happened) when a push errors, so the docs site never +# silently trails npm. +if [ "$SOURCE_ONLY" = true ]; then + echo -e "${YELLOW}1️⃣2️⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n" +else + echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" + if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" + else + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" + fi +fi echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}" +if [ "$SOURCE_ONLY" = true ]; then + echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}" +else + echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" +fi +echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs deleted file mode 100644 index d4ec7ba5..00000000 --- a/scripts/wall-entry.mjs +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/env node -/** - * @module scripts/wall-entry - * @description The releases-wall entry, made mechanical. The fleet's HQ page - * reads one public JSON per product from the ONE releases repo on The Source - * (soulcraftlabs/releases, files .json at its root — shape - * {product, entries:[{version, date, headline, items, url, thumb?}]}), at - * https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/.json. - * Those entries were hand-written after every release, then briefly written - * into this repo's own releases/.json; this script is the one door - * that composes an entry and lands it in the shared repo, so it is never - * hand-written and never forked across repos again. - * - * Two modes: - * - * 1. Generate + publish (default): - * node wall-entry.mjs --product

--version --date \ - * --from-changelog - * Derives an entry from the CHANGELOG.md entry for (headline = the - * entry's first bullet, items = every bullet, trimmed of its trailing - * commit hash), then: - * - clones (or, if a cached clone already exists, fetches and resets) - * the releases repo into a local cache directory, - * - prepends the entry to /

.json, newest first — replacing - * any existing entry for the same version so a re-run is idempotent, - * - validates the file's shape before and after, - * - commits the change as "chore(wall):

" and pushes main. - * A failure at any step (clone, validation, commit, push, a - * non-fast-forward remote) exits non-zero naming the cure. Nothing is - * ever skipped — the wall either lands correctly or the release fails. - * - * 2. Dry run: - * node wall-entry.mjs --dry-run --product

--version \ - * --date --from-changelog - * Derives the entry exactly as above and prints it, along with the file - * it would be written to, but touches no clone and no remote — usable - * from a fresh checkout with no cache and no network. - * - * 3. Validate only (--check): - * node wall-entry.mjs --check --file - * Validates an arbitrary wall file's exact key set (top-level and - * per-entry), field types, and strict-descending semver ordering with - * no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 = - * named violations printed to stderr. - * - * The remote and the local cache directory are each overridable - * (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE / - * WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local - * bare repo and a throwaway cache directory — never the real remote or the - * real developer cache. - * - * No dependencies beyond the system `git` binary — CHANGELOG parsing, - * semver comparison, and JSON shape checking are all hand-rolled below. - */ - -import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' -import { execFileSync } from 'node:child_process' -import { homedir } from 'node:os' -import { dirname, join } from 'node:path' - -const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git' - -/** @returns {string} */ -function defaultCacheDir() { - const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache') - return join(base, 'soulcraft-releases') -} - -// Required on every entry; "thumb" is optional (may be absent, or present as -// string | null) — matching the HQ contract's {..., thumb?}. -const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url'] -const ENTRY_OPTIONAL_KEYS = ['thumb'] -const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] -const FILE_KEYS = ['product', 'entries'] - -// The public permalink pattern, by product. Every entry MUST carry an https -// permalink: HQ's parser rejects a wall whose entries carry url: null (the -// whole feed became unreadable on 2026-09-02). A product whose forge repo is -// private links its PUBLIC package page on The Source instead of a release -// page that would 404 for HQ's readers. -const RELEASE_URL_PATTERNS = { - 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, - 'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`, -} - -/** - * Parse argv into a flag map. `--flag value` sets a string; `--flag` alone - * (end of argv, or followed by another `--flag`) sets boolean true. - * @param {string[]} argv - * @returns {Record} - */ -function parseArgs(argv) { - /** @type {Record} */ - const args = {} - for (let i = 0; i < argv.length; i++) { - const a = argv[i] - if (!a.startsWith('--')) continue - const key = a.slice(2) - const next = argv[i + 1] - if (next === undefined || next.startsWith('--')) { - args[key] = true - } else { - args[key] = next - i++ - } - } - return args -} - -/** - * Print a loud, named error and exit 1. Every refusal in this script goes - * through here so the failure mode is always the same shape: "wall-entry: ". - * @param {string} message - * @returns {never} - */ -function fail(message) { - console.error(`wall-entry: ${message}`) - process.exit(1) -} - -/** - * @param {string} version - * @returns {{major: number, minor: number, patch: number, pre: string | null} | null} - */ -function parseSemver(version) { - const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version) - if (!m) return null - return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null } -} - -/** - * @param {string} a - * @param {string} b - * @returns {number} positive if a > b, negative if a < b, 0 if equal. - */ -function compareSemver(a, b) { - const pa = parseSemver(a) - const pb = parseSemver(b) - if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`) - if (pa.major !== pb.major) return pa.major - pb.major - if (pa.minor !== pb.minor) return pa.minor - pb.minor - if (pa.patch !== pb.patch) return pa.patch - pb.patch - if (pa.pre === pb.pre) return 0 - if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version - if (pb.pre === null) return -1 - return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0 -} - -/** - * Validate a wall file's full shape: top-level keys ("product", "entries" — - * no more, no less), per-entry keys and field types ("thumb" optional), and - * strict-descending semver ordering with no duplicates. Collects every - * violation instead of failing on the first, so a caller reports the whole - * picture in one pass. - * @param {unknown} data - * @returns {string[]} Violation messages; empty means the file is clean. - */ -function validateShape(data) { - /** @type {string[]} */ - const errors = [] - - if (typeof data !== 'object' || data === null || Array.isArray(data)) { - return ['top level: expected a JSON object'] - } - const obj = /** @type {Record} */ (data) - - const topKeys = Object.keys(obj) - const missingTop = FILE_KEYS.filter((k) => !(k in obj)) - const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k)) - if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`) - if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`) - - if (typeof obj.product !== 'string' || obj.product.trim() === '') { - errors.push('top level: "product" must be a non-empty string') - } - if (!Array.isArray(obj.entries)) { - errors.push('top level: "entries" must be an array') - return errors // nothing further to check without an array - } - - const entries = /** @type {unknown[]} */ (obj.entries) - entries.forEach((rawEntry, i) => { - const label = `entries[${i}]` - if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) { - errors.push(`${label}: expected an object`) - return - } - const entry = /** @type {Record} */ (rawEntry) - const keys = Object.keys(entry) - const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry)) - const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k)) - if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) - if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) - - if (typeof entry.version !== 'string' || !parseSemver(entry.version)) { - errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`) - } - if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) { - errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`) - } - if (typeof entry.headline !== 'string' || entry.headline.trim() === '') { - errors.push(`${label}: "headline" must be a non-empty string`) - } - if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { - errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) - } - if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) { - errors.push(`${label}: "url" must be an https permalink — never null; HQ's parser rejects the whole feed`) - } - if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { - errors.push(`${label}: "thumb" must be a string or null when present`) - } - }) - - // Ordering: newest first, strictly descending, no duplicate versions — - // checked only over entries whose version parsed (a bad version is - // already reported above; comparing it too would just be noise). - const versioned = entries - .map((e, i) => ({ i, version: /** @type {any} */ (e)?.version })) - .filter((e) => typeof e.version === 'string' && parseSemver(e.version)) - for (let i = 0; i < versioned.length - 1; i++) { - const a = versioned[i] - const b = versioned[i + 1] - const cmp = compareSemver(a.version, b.version) - if (cmp === 0) { - errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`) - } else if (cmp < 0) { - errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`) - } - } - - return errors -} - -/** - * Extract one version's entry body from a standard-version-style CHANGELOG.md - * (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines - * until the next heading or EOF). - * @param {string} changelog - * @param {string} version - * @returns {string[]} Bullet lines, trimmed of their leading "- " and - * trailing " (hash)". - */ -function extractChangelogBullets(changelog, version) { - const lines = changelog.split('\n') - const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/ - let start = -1 - for (let i = 0; i < lines.length; i++) { - const m = headingRe.exec(lines[i]) - if (m && m[1] === version) { - start = i + 1 - break - } - } - if (start === -1) { - fail( - `version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`, - ) - } - /** @type {string[]} */ - const bullets = [] - for (let i = start; i < lines.length; i++) { - if (headingRe.test(lines[i])) break // next entry starts - const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim()) - if (lines[i].trim().startsWith('- ') && bulletMatch) { - const text = bulletMatch[1].trim() - if (text) bullets.push(text) - } - } - if (bullets.length === 0) { - fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`) - } - return bullets -} - -/** - * Derive a wall entry from a CHANGELOG.md. - * @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts - * @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} - */ -function deriveEntry({ product, version, date, changelogPath, url, thumb }) { - if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) - if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) { - fail(`--date "${date}" is not a YYYY-MM-DD date`) - } - if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`) - - const changelog = readFileSync(changelogPath, 'utf8') - const items = extractChangelogBullets(changelog, version) - const headline = items[0] - - const pattern = RELEASE_URL_PATTERNS[product] - if (url === undefined && pattern === undefined) { - throw new Error(`wall-entry: no permalink pattern for product "${product}" — add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`) - } - const resolvedUrl = url !== undefined ? url : pattern(version) - const resolvedThumb = thumb !== undefined ? thumb : null - - return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } -} - -/** - * Load and shape-validate a wall file. - * @param {string} filePath - * @returns {Record} - */ -function loadWallFile(filePath) { - if (!existsSync(filePath)) fail(`"${filePath}" does not exist`) - /** @type {unknown} */ - let data - try { - data = JSON.parse(readFileSync(filePath, 'utf8')) - } catch (err) { - fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) - } - const errors = validateShape(data) - if (errors.length) { - fail(`"${filePath}" fails shape validation —\n ${errors.join('\n ')}`) - } - return /** @type {Record} */ (data) -} - -/** - * Run a git command, throwing an Error whose message is git's own stderr - * (trimmed) on failure — every caller wraps this to name the cure. - * @param {string[]} args - * @param {string} cwd - * @returns {string} stdout, trimmed. - */ -function git(args, cwd) { - try { - return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim() - } catch (err) { - const stderr = /** @type {any} */ (err).stderr - const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message - throw new Error(message) - } -} - -/** - * Ensure a clean, up-to-date local clone of the releases repo at - * `cacheDir`, checked out on `main` — cloning fresh if `cacheDir` has no - * `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a - * stray local commit or edit left by a previous failed run can never leak - * into the next one). - * @param {string} remote - * @param {string} cacheDir - */ -function ensureReleasesClone(remote, cacheDir) { - if (existsSync(join(cacheDir, '.git'))) { - try { - git(['remote', 'set-url', 'origin', remote], cacheDir) - git(['fetch', '--prune', 'origin'], cacheDir) - git(['checkout', 'main'], cacheDir) - git(['reset', '--hard', 'origin/main'], cacheDir) - git(['clean', '-fd'], cacheDir) - } catch (err) { - fail( - `cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" — ${/** @type {Error} */ (err).message}\n` + - ` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`, - ) - } - return - } - - mkdirSync(dirname(cacheDir), { recursive: true }) - try { - git(['clone', remote, cacheDir], dirname(cacheDir)) - } catch (err) { - fail( - `cannot clone "${remote}" — ${/** @type {Error} */ (err).message}\n` + - ` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`, - ) - } - try { - git(['checkout', 'main'], cacheDir) - } catch (err) { - fail( - `cloned "${remote}" into "${cacheDir}" but could not check out "main" — ${/** @type {Error} */ (err).message}\n` + - ` cure: confirm the releases repo's default branch is named "main"`, - ) - } -} - -/** - * Prepend `entry` to the wall at `/.json`, replacing any - * existing entry for the same version (idempotent re-runs), validating - * before and after, committing, and pushing — or refusing loudly, naming - * the cure, at whichever step fails. - * @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry - * @param {string} product - * @param {string} remote - * @param {string} cacheDir - */ -function publishEntry(entry, product, remote, cacheDir) { - ensureReleasesClone(remote, cacheDir) - - const filePath = join(cacheDir, `${product}.json`) - if (!existsSync(filePath)) { - fail( - `"${filePath}" does not exist in the releases repo — cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`, - ) - } - const wall = loadWallFile(filePath) - - if (wall.product !== product) { - fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given — refusing a cross-product write`) - } - - const replacing = wall.entries.some((e) => e.version === entry.version) - wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)] - - const postErrors = validateShape(wall) - if (postErrors.length) { - fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`) - } - - writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') - - const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir) - if (status === '') { - console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} — nothing to commit or push`) - return - } - - try { - git(['add', `${product}.json`], cacheDir) - git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir) - } catch (err) { - fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`) - } - - try { - git(['push', 'origin', 'main'], cacheDir) - } catch (err) { - fail( - `push to "${remote}" failed (likely a non-fast-forward — another release landed on main first) — ${/** @type {Error} */ (err).message}\n` + - ` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`, - ) - } - - const sha = git(['rev-parse', 'HEAD'], cacheDir) - console.log( - `wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) — pushed ${sha} to ${remote} main`, - ) -} - -function main() { - const args = parseArgs(process.argv.slice(2)) - - if (args.check) { - const filePath = /** @type {string | undefined} */ (args.file) - if (!filePath) fail('--check needs --file ') - const wall = loadWallFile(/** @type {string} */ (filePath)) - console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) - process.exit(0) - } - - // Generate mode (default, also covers --dry-run): --product, --version, - // --date, --from-changelog required. - const product = /** @type {string | undefined} */ (args.product) - const version = /** @type {string | undefined} */ (args.version) - const date = /** @type {string | undefined} */ (args.date) - const fromChangelog = /** @type {string | undefined} */ (args['from-changelog']) - - const missing = [] - if (!product) missing.push('--product') - if (!version) missing.push('--version') - if (!date) missing.push('--date') - if (!fromChangelog) missing.push('--from-changelog') - if (missing.length) { - fail( - `missing required flag(s): ${missing.join(', ')}\n` + - 'Usage:\n' + - ' wall-entry.mjs --product

--version --date --from-changelog [--dry-run]\n' + - ' wall-entry.mjs --check --file ', - ) - } - - const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url) - const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb) - - const entry = deriveEntry({ - product: /** @type {string} */ (product), - version: /** @type {string} */ (version), - date: /** @type {string} */ (date), - changelogPath: /** @type {string} */ (fromChangelog), - url: urlArg, - thumb: thumbArg, - }) - - const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE) - const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir()) - - if (args['dry-run']) { - console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`) - console.log(JSON.stringify(entry, null, 2)) - process.exit(0) - } - - publishEntry(entry, /** @type {string} */ (product), remote, cacheDir) -} - -main() diff --git a/src/brainy.ts b/src/brainy.ts index da04577e..db332e18 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -25,7 +25,6 @@ import { } from './storage/brainFormat.js' import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' -import { isZeroNormVector } from './utils/distance.js' import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, @@ -198,12 +197,7 @@ import { import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { - assessIndexReadiness, - assessProviderHealth, - assessProviderRebuild, - describeRebuildProgress -} from './utils/indexReadiness.js' +import { assessIndexReadiness } from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { @@ -401,15 +395,6 @@ interface PlannedTransact { * marker outlives its write. */ markerRecords: FactMarkerRecord[] - /** - * Ids the batch's `{ op: 'update' }` unvector door (`vector: []`) needs to - * decrement on the vectored-noun ledger — consumed by `transact()` with a - * proper `await this.storage.noteVectorUnlanded?.(id)` per id, AFTER - * `commitTransaction` resolves (never for a rejected batch). Kept separate - * from `postCommit` (`Array<() => void>`, called synchronously, fire-and- - * forget) because the ledger hook is async and must be awaited. - */ - vectorUnlands: string[] } /** @@ -649,6 +634,8 @@ export class Brainy implements BrainyInterface { /** One-shot guard so the degraded-reads warning fires once per degraded window * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */ private _degradedReadWarned = false + /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ + private _metadataConsistencyProbed = false /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */ private _graphAdjacencyVerified = false /** Re-entrancy guard: a verify (rebuild → reads) is in flight. */ @@ -745,24 +732,6 @@ export class Brainy implements BrainyInterface { // Write acks NEVER await it; a failed background flush is LOUD and re-armed. private _persistDirtyWrites = 0 private _persistLastFlushAt = Date.now() - /** - * Whether a write has been committed since the last flush that ran. THE - * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written - * to has nothing to make durable, and a flush over it must cost nothing and - * say nothing. Before this, a flush called every provider, stamped the - * watermarks, persisted the generation counter and re-stamped the entity - * tree whether or not anything had changed — roughly 28 writes for a store - * that had not moved. - * - * WHAT THIS DOES NOT EXPLAIN, stated so nobody reads it as solved: a - * production process holding 21 brains printed "All indexes flushed to disk - * in 216-601ms" per brain every ~35s and idled at 1.26 cores with no writes - * for ten minutes. This engine's cadence is WRITE-DRIVEN — every trigger - * runs through noteWriteForPersistence, which only a committed write calls — - * so something was calling flush() on those brains, and this gate makes such - * a call free rather than accounting for it. The caller is still unidentified. - */ - private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null private _persistBackgroundFlight: Promise | null = null @@ -775,15 +744,6 @@ export class Brainy implements BrainyInterface { private _pendingEmbedIds = new Set() private _embedWorkerFlight: Promise | null = null - // OPEN-PATH FIX: the background embedding-engine warm kicked off (never - // awaited) by `performInit()` when `eagerEmbeddings` resolves true. Stored - // for observability only — `embed()`/`embeddingManager.embed()` already - // await the engine's OWN singleton init promise internally, so nothing - // needs to explicitly await this field for correctness. Never rejects on - // its own: a `.catch` narrates the failure and swallows it so a failed - // warm never surfaces as an unhandled rejection. - private _embeddingWarmPromise: Promise | null = null - /** The stored log-authority switch, read once at open (default: tree). */ private _logAuthority: LogAuthorityRecord = { authority: 'tree' } // A failed walk latches its error: retries within the cooldown rethrow it @@ -844,31 +804,11 @@ export class Brainy implements BrainyInterface { // applies only to instances that were never closed. private closed = false - // Index-build-at-open state. `lazyRebuildCompleted` predates the health-gate - // law (it named a first-QUERY lazy rebuild) and stays for `getIndexStatus()` - // API compatibility, but its truth changed: a needed rebuild now runs - // unconditionally at open() (see `rebuildIndexesIfNeeded`), never deferred to - // a read, so this simply flips true once that open-time step has run. - // `lazyRebuildInProgress` / `lazyRebuildPromise` (the first-query rebuild's - // concurrency guard) are retired with the lazy-build path they served — - // `ensureIndexesLoaded()` is a read-time CHECK now, never a build. + // Lazy rebuild state (Production-scale lazy loading) + // Prevents race conditions when multiple queries trigger rebuild simultaneously + private lazyRebuildInProgress = false private lazyRebuildCompleted = false - - // Read-gate narration dedup: a degraded-but-serving or not-ready health - // report narrates via prodLog.warn ONCE per (provider, report.generation) — - // never once per read. Keyed on the provider instance itself. - /** - * The last health narration emitted per provider, keyed by its CONTENT. - * - * This used to dedupe on the provider's `generation` counter, which bumps on - * every ledger mutation and every rebuild boundary — so a provider that - * bumps its generation on routine work re-emitted the same unchanged health - * line on every read that consulted it, and a provider that never bumped - * could suppress a line whose reasons had genuinely changed. The dedupe key - * is now what the line SAYS: an unchanged verdict is silent however the - * generation moves, and a changed verdict is always heard. - */ - private _lastNarratedHealth = new Map() + private lazyRebuildPromise: Promise | null = null constructor(config?: BrainyConfig) { // The reserved-field write policy died with the field-addressing law: @@ -986,14 +926,14 @@ export class Brainy implements BrainyInterface { * extends FileSystemStorage`) inherit new methods Brainy adds to * `FileSystemStorage` / `BaseStorage` automatically — `typeof` walks the * prototype chain, so there's no in-package version skew to worry about as - * long as the plugin's own dist resolves `@soulcraftlabs/brainy` dynamically + * long as the plugin's own dist resolves `@soulcraft/brainy` dynamically * (which Cortex 2.2.x onward does — see * `node_modules/@soulcraft/cor/dist/storage/mmapFileSystemStorage.js`). * * This helper exists for the **build/install** failure modes the import * resolution can't catch: * - Stale `node_modules` left over from a prior `bun install` against - * `@soulcraftlabs/brainy ≤7.20.x`. + * `@soulcraft/brainy ≤7.20.x`. * - Lockfile drift pinning brainy below the version that introduced the * method. * - Docker layer caches that reuse a `node_modules` from an earlier image. @@ -1132,86 +1072,6 @@ export class Brainy implements BrainyInterface { configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging } - // OPEN-PATH NARRATION: phase timing across the five named stretches of - // init — storage init / generation-store open+fold / index init+gate / - // VFS bootstrap / embedding-warm-started. Each `markPhase()` call records - // elapsed ms SINCE THE PREVIOUS checkpoint, so the buckets always sum to - // the pre-integration/warmOnOpen total. - // - // THE LAW THIS ENFORCES: an open is never silent for more than - // OPEN_HEARTBEAT_MS. A production service opening a 16 GB store logged - // NOTHING for three minutes and then began work — the operator could not - // tell a slow open from a hung one, and restarted into the same wall. - // Two mechanisms, both on the always-visible narration channel (the old - // breakdown used `prodLog.warn`, which production clamps away — that is - // why the three minutes were silent): - // - a heartbeat that names the phase currently running and its elapsed - // wall, every OPEN_HEARTBEAT_MS, for as long as the open lasts; - // - one line per phase AS IT ENDS, naming its wall and its cause, for - // any phase over OPEN_PHASE_NARRATE_MS. - // The heartbeat is unref'd and cleared in the `finally` below, so it can - // neither hold the process open nor outlive a failed init. It cannot fire - // inside a phase that blocks the event loop synchronously; such a phase - // must narrate its own progress (the generation-log fold does). - const OPEN_HEARTBEAT_MS = 5_000 - const OPEN_PHASE_NARRATE_MS = 2_000 - /** Phase order + what each one is paying for, quoted in its narration. */ - const OPEN_PHASES: ReadonlyArray<{ name: string; cause: string }> = [ - { name: 'storage-init', cause: 'opening the store and loading its count ledger' }, - { - name: 'generation-store-open-fold', - cause: 'opening the generation store: crash-recovery replay/fold, derived-family registration, format handshake' - }, - { name: 'index-init-gate', cause: 'constructing the derived indexes and gating them for serving' }, - { name: 'vfs-bootstrap', cause: 'bootstrapping the virtual filesystem' }, - { name: 'embedding-warm-started', cause: 'starting the background embedding warm' } - ] - const initStart = Date.now() - let lastPhaseCheckpoint = initStart - let currentPhaseIndex = 0 - const phaseTimingsMs: Record = {} - const openHeartbeat: ReturnType = setInterval(() => { - const phase = OPEN_PHASES[currentPhaseIndex] - if (!phase) return - prodLog.narrate( - `[Brainy] open: still in phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + - `"${phase.name}" after ${Math.round((Date.now() - lastPhaseCheckpoint) / 1000)}s ` + - `(${Math.round((Date.now() - initStart) / 1000)}s into the open) — ${phase.cause}` - ) - }, OPEN_HEARTBEAT_MS) - if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref() - /** - * Narrate one STEP inside a phase when it turns out to be expensive. - * A phase that costs a minute and names only itself tells an operator - * where to look but not what to look at; this names the step. Silent - * under OPEN_PHASE_NARRATE_MS, so a fast open says nothing extra. - */ - const step = async (name: string, cause: string, run: () => Promise): Promise => { - const startedAt = Date.now() - try { - return await run() - } finally { - const elapsed = Date.now() - startedAt - if (elapsed >= OPEN_PHASE_NARRATE_MS) { - prodLog.narrate(`[Brainy] open: step "${name}" took ${elapsed}ms — ${cause}`) - } - } - } - const markPhase = (name: string): void => { - const now = Date.now() - const elapsed = now - lastPhaseCheckpoint - phaseTimingsMs[name] = elapsed - lastPhaseCheckpoint = now - const finished = OPEN_PHASES[currentPhaseIndex] - if (elapsed >= OPEN_PHASE_NARRATE_MS && finished && finished.name === name) { - prodLog.narrate( - `[Brainy] open: phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + - `"${name}" finished in ${elapsed}ms — ${finished.cause}` - ) - } - currentPhaseIndex++ - } - try { // Auto-detect and activate plugins BEFORE storage setup // so plugin-provided storage factories (e.g., filesystem override from cor) are available @@ -1272,7 +1132,7 @@ export class Brainy implements BrainyInterface { `and the flush-request RPC are disabled for this directory. ` + `Likely fix: clean install (\`rm -rf node_modules bun.lockb && ` + `bun install\`) or rebuild your container image to refresh ` + - `\`@soulcraftlabs/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.` + `\`@soulcraft/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.` ) } else { console.warn( @@ -1283,12 +1143,6 @@ export class Brainy implements BrainyInterface { } } - // PHASE 1 of 5 — "storage init": plugin/legacy-layout bootstrap, - // storage adapter construction+init, the OS-limit check, and the - // writer-lock claim, all folded into one bucket (everything above this - // line since performInit started). - markPhase('storage-init') - // 8.0 generational MVCC: open the record layer BEFORE any index is // created or loaded. Crash recovery may rewrite canonical entity files // (restoring before-images of an uncommitted transaction), and every @@ -1297,12 +1151,9 @@ export class Brainy implements BrainyInterface { // instances skip recovery (readers never write; the next writer // repairs). this.generationStore = new GenerationStore(this.storage) - const generationOpenResult = await step( - 'generation-store.open', - 'reading the generation manifest and committed ranges, opening the fact log and the ' + - 'packed segment tier, and folding any crash-recovery replay', - () => this.generationStore.open({ readOnly: this.config.mode === 'reader' }) - ) + const generationOpenResult = await this.generationStore.open({ + readOnly: this.config.mode === 'reader' + }) // The generation fact log is CANONICAL state, not a derived index — no // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare @@ -1340,11 +1191,7 @@ export class Brainy implements BrainyInterface { // rollup invariants against the log head + live counters. Loud on // genuine incoherence (repairIndex heals), silent on absent/coherent, // benign-behind refreshes at the next flush. Never blocks open. - await step( - 'verify-entity-tree-stamp', - 'comparing the entity tree\'s stamped generation and rollups against the store', - () => this.verifyEntityTreeStamp() - ) + await this.verifyEntityTreeStamp() // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — @@ -1356,11 +1203,7 @@ export class Brainy implements BrainyInterface { // them from the canonical records and then re-stamps the marker AFTER the // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / // absent marker, so the next open idempotently re-rebuilds). - this._brainFormat = await step( - 'read-brain-format', - 'reading the on-disk format marker that decides whether the derived indexes are stale', - () => readBrainFormat(this.storage) - ) + this._brainFormat = await readBrainFormat(this.storage) this._indexEpochStale = this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH @@ -1371,19 +1214,9 @@ export class Brainy implements BrainyInterface { // upgrade verifies + stamps; retained on failure. No-op for a reader, for // non-filesystem storage, or for a brain with no persisted data. if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { - await step( - 'pre-upgrade-backup', - 'snapshotting the brain directory before a one-time format rebuild (migrationBackup)', - () => this.createMigrationBackupIfNeeded() - ) + await this.createMigrationBackupIfNeeded() } - // PHASE 2 of 5 — "generation-store open+fold": GenerationStore - // construction+open (crash-recovery replay/rollback fold), the - // derived-family registration, the fact-scan seam, the entity-tree - // stamp check, the brain-format handshake, and the pre-upgrade backup. - markPhase('generation-store-open-fold') - // Provider: embeddings (reassign embedder if plugin provides one) const embeddingProvider = this.pluginRegistry.getProvider('embeddings') if (embeddingProvider) { @@ -1538,44 +1371,13 @@ export class Brainy implements BrainyInterface { `[Brainy] Rebuilding indexes after crash recovery rolled back ` + `${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)` ) - // SELF-REBUILD DEFERENCE, same law as the open gate: a provider that - // is already rebuilding itself from canonical is doing exactly this - // work. Kicking a second rebuild on top of it is redundant at best. - // Safe by ordering: the crash-recovery fold ran in the generation - // store's open, BEFORE any provider was constructed, so a provider - // rebuilding now is reading the repaired canonical records. - const kick = async (leg: string, provider: { rebuild: () => Promise }) => { - const rebuilding = assessProviderRebuild(provider) - if (rebuilding) { - prodLog.narrate( - `[Brainy] crash-recovery rebuild: the ${leg} provider is already ` + - `${describeRebuildProgress(rebuilding)} from canonical — not kicking a second one.` - ) - return - } - await provider.rebuild() - } await Promise.all([ - kick('metadata', this.metadataIndex), - kick('vector', this.index as unknown as { rebuild: () => Promise }), - kick('graph', this.graphIndex) + this.metadataIndex.rebuild(), + this.index.rebuild(), + this.graphIndex.rebuild() ]) } - // METADATA WATERMARK CATCHUP: the JS metadata index computed its - // three-way watermark verdict inside metadataIndex.init() above, - // against the generation store's now-FINAL committed generation (the - // crash-recovery fold above — the durable-at-ack replay of acked - // writes whose canonical bytes hadn't reached disk — has already run, - // and any rolled-back-transaction rebuild just above already brought - // every index current, so the verdict is consumed here whether or not - // that rebuild ran). Consumed BEFORE the rebuild gate below and BEFORE - // this open serves any read — the cure for the class of bug where - // canonical get()/counts recover a crash-window write but find() - // keeps serving the metadata index's pre-crash state (the index - // flushes only periodically, not per-commit). - await this.consumeMetadataWatermarkVerdict(generationOpenResult.rolledBackGenerations > 0) - // 8.0 versioned-provider replay-gap check: a provider whose persisted // index generation is behind the storage layer's committed generation // replays the gap itself (post-commit applier contract) — surface the @@ -1642,38 +1444,12 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } - // LEG C (zero-norm/unvector-door law): migrate a legacy zero-norm VFS - // root BEFORE the vector-leg open gate below ever compares the - // canonical vectored-noun count against the vector index's size — see - // migrateLegacyZeroNormVfsRootIfNeeded's JSDoc for why this is a safe - // O(1) exception to "nothing at open may scale with brain size", and - // why it must run here rather than waiting on VirtualFileSystem's own - // (VFS-instance-gated) lazy migration. - await this.migrateLegacyZeroNormVfsRootIfNeeded() - - // Rebuild indexes if needed for existing data. Runs to completion before - // init() returns — there is no more first-query lazy path, so the flag - // below (kept for getIndexStatus() API compatibility) simply flips true - // once this open-time step has run. - await step( - 'rebuild-indexes-if-needed', - 'the derived-index gate: each family\'s readiness verdict, and any build it asks for', - () => this.rebuildIndexesIfNeeded() - ) - this.lazyRebuildCompleted = true + // Rebuild indexes if needed for existing data + await this.rebuildIndexesIfNeeded() // Check for pending data migrations await this.checkMigrations() - // PHASE 3 of 5 — "index init+gate": provider wiring (embeddings, - // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph - // index construction, the eager cold-load, id-resolver + connections- - // codec wiring, crash-recovery index rebuild, the replay-gap check, - // legacy VFS blob adoption, blob-history backfill, the legacy - // zero-norm VFS root migration, and the rebuildIndexesIfNeeded() gate - // + migration check. - markPhase('index-init-gate') - // Register shutdown hooks for graceful count flushing (once globally) if (!Brainy.shutdownHooksRegisteredGlobally) { this.registerShutdownHooks() @@ -1732,11 +1508,7 @@ export class Brainy implements BrainyInterface { // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) - await step( - 'vfs.init', - 'creating or adopting the VFS root and wiring the path resolver', - () => this._vfs!.init() - ) + await this._vfs.init() this._vfsInitialized = true // Mark VFS as fully initialized // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the @@ -1760,11 +1532,7 @@ export class Brainy implements BrainyInterface { const storedArtifact = await this.storage .readRawObject(LOG_AUTHORITY_PATH) .catch(() => null) - const authority = await step( - 'read-log-authority', - 'reading the stored storage-authority artifact', - () => readLogAuthority(this.storage) - ) + const authority = await readLogAuthority(this.storage) this._logAuthority = authority if (authority.authority === 'log') { this.generationStore.setLogDurability('at-ack') @@ -1775,12 +1543,7 @@ export class Brainy implements BrainyInterface { this.generationStore.getFactLog() !== null ) { try { - await step( - 'adopt-log-authority', - 'the adoption oracle: verifying the log against canonical before flipping this ' + - 'brain to durable-at-ack, and backfilling any curable divergence', - () => this.adoptLogAuthority() - ) + await this.adoptLogAuthority() prodLog.info( '[Brainy] storage authority adopted at open: generation log ' + '(fleet default; oracle green; durable-at-ack enabled)' @@ -1820,16 +1583,8 @@ export class Brainy implements BrainyInterface { // this is where it lands. if (!this.isReadOnly) { try { - await step( - 'bridge-pending-embed-sidecars', - 'migrating any pre-log deferred-embed marker files into the generation log', - () => this.bridgeLegacyPendingEmbedSidecars() - ) - await step( - 'recover-pending-embeds', - 'folding the generation log\'s deferred-embed markers back into the pending set', - () => this.recoverPendingEmbedsFromLog() - ) + await this.bridgeLegacyPendingEmbedSidecars() + await this.recoverPendingEmbedsFromLog() if (this._pendingEmbedIds.size > 0) { prodLog.info( `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + @@ -1846,40 +1601,15 @@ export class Brainy implements BrainyInterface { } } - // PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob - // storage init, the provider-summary log, flipping `initialized`, - // the migration-lock wait, VFS construction+init, flipping generation - // stamping active, the log-authority adopt/oracle check, and - // pending-embed crash recovery. - markPhase('vfs-bootstrap') - - // Eager embedding initialization — BACKGROUND WARM (open-path fix). + // Eager embedding initialization. // - // Adaptive default (8.0): the WASM embedding engine eagerly WARMS + // Adaptive default (8.0): the WASM embedding engine eagerly initializes // during init() WHENEVER it is the active embedder — i.e. no native // 'embeddings' provider has taken over — and the instance is a writer // (not reader-mode) outside of unit tests. The WASM module (≈93MB with - // the embedded model) takes 90-140s to compile on throttled CPUs. - // - // Historically this AWAITED `embeddingManager.init()` INLINE, so every - // writer's open() blocked on the compile — N concurrent opens all - // queued on the ONE process-global singleton (an ~80x contention - // multiplier measured in a production restart storm: 90,017ms busy vs - // 1,117ms quiet). The engine only needs to be ready before the FIRST - // REAL embed() call, not before init() returns, so this now only - // STARTS the warm and moves on — init() never waits for it. - // - // No double-await needed for correctness: `this.embed()` (~line 15420) - // delegates to `this.embedder`, which for the default engine is - // `embeddingManager.getEmbeddingFunction()` → `embeddingManager.embed()` - // (src/embeddings/EmbeddingManager.ts). That method calls `await - // this.init()` FIRST, and `init()` itself serializes every concurrent - // caller onto ONE shared `globalInitPromise` — so the first real - // embed() automatically waits for whichever finishes first: this - // background warm (if still running) or a fresh init() (if the warm - // hasn't reached this code yet, e.g. `eagerEmbeddings: false`). - // Verified by reading both call sites; `_embeddingWarmPromise` below - // is stored for observability only, never re-awaited by embed(). + // the embedded model) takes 90-140s to compile on throttled CPUs; paying + // that during boot rather than on the first embed()-driven call is the + // right default for the overwhelmingly common single-process server. // // Skipped automatically when: // - a native 'embeddings' provider is registered (it owns embeddings; @@ -1887,8 +1617,8 @@ export class Brainy implements BrainyInterface { // - reader-mode (readers don't embed — they query existing vectors), // - unit-test mode (tests must stay fast and use the mock embedder). // - // `eagerEmbeddings: false` keeps meaning "no warm at all" — fully lazy, - // the first embed() call pays the full cost inline, same as before. + // `eagerEmbeddings: false` is the explicit override to force lazy init + // (first-embed) even when this instance is the active embedder. const isUnitTestMode = isDeterministicEmbedMode() const eager = this.config.eagerEmbeddings ?? true if ( @@ -1897,45 +1627,9 @@ export class Brainy implements BrainyInterface { this.config.mode !== 'reader' && !isUnitTestMode ) { - const warmStart = Date.now() - console.log('Background embedding-engine warm started (init() does not wait for it)...') - this._embeddingWarmPromise = embeddingManager - .init() - .then(() => { - prodLog.info( - `[Brainy] background embedding-engine warm complete in ${Date.now() - warmStart}ms` - ) - }) - .catch((err) => { - // Loud, never silent: a warm that fails to compile must be - // heard NOW, not discovered as a mystery latency spike on - // whichever request happens to trigger the first real embed(). - // That first embed() call still retries init() itself (the - // singleton promise contract above) and surfaces its own typed - // error to its caller — this is the immediate, background echo. - prodLog.warn( - `[Brainy] background embedding-engine warm FAILED: ` + - `${(err as Error).message} — the first embed() call will retry ` + - `initialization and surface the error there` - ) - }) - } - - // PHASE 5 of 5 — "embedding-warm-started": just the synchronous cost - // of kicking off the background warm above (the warm's own compile - // time is NOT included — that's the whole point of backgrounding it). - markPhase('embedding-warm-started') - { - const totalOpenMs = Date.now() - initStart - if (totalOpenMs > 2000) { - const phaseList = Object.entries(phaseTimingsMs) - .map(([name, ms]) => `${name}=${ms}ms`) - .join(', ') - prodLog.narrate( - `[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` + - `phase breakdown above to find which one to investigate first` - ) - } + console.log('Eager embedding initialization enabled...') + await embeddingManager.init() + console.log('Embedding engine ready') } // Integration Hub initialization @@ -1988,15 +1682,7 @@ export class Brainy implements BrainyInterface { if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { throw error } - // Wrap with the original as `cause` so the originating frame (a plugin's - // own file:line, e.g. a provider boot failure) survives to the caller's - // log — a plain string interpolation discards both stack and cause. - const message = error instanceof Error ? error.message : String(error) - throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error }) - } finally { - // The open is over — succeeded or failed. Stop the heartbeat here so a - // failed init never leaves a timer narrating a phase nobody is running. - clearInterval(openHeartbeat) + throw new Error(`Failed to initialize Brainy: ${error}`) } } @@ -2014,112 +1700,76 @@ export class Brainy implements BrainyInterface { * NOTE: Registers globally (once for all instances) to avoid MaxListenersExceededWarning */ private registerShutdownHooks(): void { - /** - * The signal-path shutdown. THREE LAWS, each written by a production - * shutdown that looked clean and wasn't: - * - * 1. PER-INSTANCE ISOLATION. This used to be one `try` around a loop over - * every open brain: the first instance whose flush rejected aborted the - * loop, so every remaining brain kept its writer lock and its unwritten - * markers — and the process still exited 0. A pool of brains failed in - * a batch, not one at a time. - * 2. THE MARKER IS PART OF SHUTDOWN. Flushing the indexes without closing - * the generation store leaves the clean-shutdown marker unwritten, so - * the NEXT open reads the store as crashed and folds the whole - * generation log — measured in tens of seconds on a real store, paid on - * every restart, after a shutdown the operator saw exit 0. - * 3. THE LOCK IS ALWAYS GIVEN UP. In a `finally`, per instance: a process - * on its way out holds nothing. - */ const flushOnShutdown = async () => { console.log('Shutdown signal received - flushing pending data...') - let flushedCount = 0 - let failedCount = 0 - // Snapshot: close() splices Brainy.instances while we iterate. - for (const instance of [...Brainy.instances]) { - if (!instance.initialized) continue - try { - // Flush all buffered data (parallel across components, this brain only). - await Promise.all([ - (async () => { - if (instance.storage && typeof instance.storage.flushCounts === 'function') { - await instance.storage.flushCounts() - } - })(), - (async () => { - if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { - await instance.metadataIndex.flush() - } - })(), - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { - await instance.graphIndex.flush() - } - })(), - (async () => { - if (instance.index && typeof instance.index.flush === 'function') { - await instance.index.flush() - } - })() - ]) - - // Close the generation store: persists the counter, advances the - // fold checkpoint, and stamps the clean-shutdown marker LAST — the - // one step that decides whether the next open adopts or folds. Law 2. - if (instance.generationStore && !instance.isReadOnly) { - await instance.generationStore.close() - } - - // Close components to stop timers that would prevent clean process exit - await Promise.all([ - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { - await instance.graphIndex.close() - } - })(), - (async () => { - const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && typeof index.close === 'function') { - await index.close() - } - })(), - (async () => { - const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && typeof metadataIndex.close === 'function') { - await metadataIndex.close() - } - })() - ]) - flushedCount++ - } catch (error) { - failedCount++ - console.error('Failed to flush one Brainy instance on shutdown:', error) - } finally { - // Law 3 — the lock and the watcher go regardless. - try { - if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { - instance.storage.stopFlushRequestWatcher() - } - } catch (error) { - console.error('Failed to stop the flush-request watcher on shutdown:', error) - } - try { - if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { - await instance.storage.releaseWriterLock() - } - } catch (error) { - console.error('Failed to release the writer lock on shutdown:', error) + try { + let flushedCount = 0 + for (const instance of Brainy.instances) { + if (instance.initialized) { + // Flush all buffered data, then close to release resources (timers, handles) + await Promise.all([ + (async () => { + if (instance.storage && typeof instance.storage.flushCounts === 'function') { + await instance.storage.flushCounts() + } + })(), + (async () => { + if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { + await instance.metadataIndex.flush() + } + })(), + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { + await instance.graphIndex.flush() + } + })(), + (async () => { + if (instance.index && typeof instance.index.flush === 'function') { + await instance.index.flush() + } + })() + ]) + // Close components to stop timers that would prevent clean process exit + await Promise.all([ + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { + await instance.graphIndex.close() + } + })(), + (async () => { + const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks + if (index && typeof index.close === 'function') { + await index.close() + } + })(), + (async () => { + const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks + if (metadataIndex && typeof metadataIndex.close === 'function') { + await metadataIndex.close() + } + })(), + // Release the writer lock so a successor process can take over. + // No-op for readers and for backends without locking. + (async () => { + if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { + await instance.storage.releaseWriterLock() + } + })(), + // Stop the flush-request watcher to release its interval timer. + (async () => { + if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { + instance.storage.stopFlushRequestWatcher() + } + })(), + ]) + flushedCount++ } } - } - if (flushedCount > 0) { - console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) - } - if (failedCount > 0) { - console.error( - `${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` + - `their writer locks were released, but their next open will run crash recovery.` - ) + if (flushedCount > 0) { + console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) + } + } catch (error) { + console.error('Failed to flush on shutdown:', error) } } @@ -2127,32 +1777,13 @@ export class Brainy implements BrainyInterface { // kept as statics so the last live instance's close() can deregister them // — the signal handles they hold are ref'd and would otherwise keep the // process alive forever after every brain is closed. - /** - * Exit the process ONLY when Brainy is the sole handler for this signal. - * - * Registering a signal listener suppresses Node's default terminate - * behaviour, so a library that attaches one must either exit or be sure - * someone else will. Brainy attaching one AND exiting was the wrong half - * of that choice for every host application with its own graceful - * shutdown: both handlers run concurrently, and whichever finishes first - * wins — a library flush finishing before an application's close() - * terminated that close mid-flight, at exit code 0, with locks and - * markers unwritten. When the host has its own handler (listener count - * above our own), the host owns the exit; Brainy only makes its data - * durable and steps aside. - */ - const exitIfSoleShutdownOwner = (signal: 'SIGTERM' | 'SIGINT'): void => { - if (process.listenerCount(signal) <= 1) { - process.exit(0) - } - } Brainy.sigtermListener = async () => { await flushOnShutdown() - exitIfSoleShutdownOwner('SIGTERM') + process.exit(0) } Brainy.sigintListener = async () => { await flushOnShutdown() - exitIfSoleShutdownOwner('SIGINT') + process.exit(0) } Brainy.beforeExitListener = async () => { // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- @@ -2619,17 +2250,6 @@ export class Brainy implements BrainyInterface { [{ type: 'embed.landed', id, vector: newVector }], 'system:embed-landing' ) - // Vectored-noun ledger: the landing commit above carries a vector - // write with NO accompanying metadata operation, so the - // saveNounMetadata(..., hasVector) seam never fires for it — the - // narrow storage hook is the only seam left. `oldVector.length===0` - // (already known for free from the pre-embed read above) proves this - // is a GENUINE first landing, not a re-embed of an already-vectored - // row (e.g. a deferred update() on a row that already had a real - // vector) — the latter must never double-count. - if (oldVector.length === 0) { - await this.storage.noteVectorLanded?.(id) - } this.clearPendingEmbed(id) } catch (err) { prodLog.warn( @@ -2776,12 +2396,6 @@ export class Brainy implements BrainyInterface { * engine's own cadence (callers never call flush() in hot paths). */ private noteWriteForPersistence(): void { - // THE DIRTY WITNESS. Set on every committed write — both commit paths - // (single-op and transaction) end here, and the deferred-embed worker - // lands its vectors through the single-op path — BEFORE the policy check, - // so a `'manual'` consumer's explicit flush() is never skipped either. - // Cleared by a flush that actually runs; see flush(). - this._dirtySinceLastFlush = true const cfg = this.config.persistence if (this.isReadOnly || cfg?.policy === 'manual') return this._persistDirtyWrites++ @@ -3150,43 +2764,13 @@ export class Brainy implements BrainyInterface { // vector shape, is structurally impossible). The background worker // embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - let vector = deferringEmbed + const vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) - // THE ZERO-NORM LAW (canonical write side): a zero-norm vector is not a - // vector — it never crosses an engine boundary (the engine pair's seam - // law). This engine's own cosine distance treats an all-zero vector - // safely (a zero-norm operand always scores MAXIMUM distance — see - // isZeroNormVector's JSDoc), but a downstream engine serving squared- - // euclidean distance cannot tell it apart from a legitimate origin - // point — a false attractor that silently darkened 150+ rows in a - // production deployment. The index belt (AddToVectorIndexOperation) - // already refuses to INDEX a zero-norm vector, but until now the - // CANONICAL write still persisted it and the vectored-noun ledger - // counted it — so a near-empty store whose only vectored row was - // zero-norm read "canonical vectored > 0, index size 0" and threw a - // not-ready error at open. Normalize HERE, before the dimension pin, - // the vectored-ledger flag (`SaveNounMetadataOperation`'s `hasVector`), - // and the index ops below ever see it, so it persists as the sanctioned - // "unvectored" `[]` shape instead — the canonical write still succeeds. - if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { - prodLog.warn( - `[Brainy] add(): entity ${id} was given an explicit all-zero vector — ` + - `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - vector = [] - } - // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). - // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose - // vector is the "unvectored" empty-array shape carries no dimension - // information, deferred or not — an explicit `vector: []` (e.g. the VFS - // root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must - // never pin `this.dimensions` to 0, which would poison every subsequent - // real embed's dimension check for the life of the store. - if (!deferringEmbed && vector.length > 0) { + if (!deferringEmbed) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -3284,11 +2868,8 @@ export class Brainy implements BrainyInterface { const runInsert: TransactionFunction = async (tx) => { // Operation 1: Save metadata FIRST (TypeAwareStorage caching) // isNew=true: skip pre-read for rollback (entity doesn't exist yet) - // hasVector: the vectored-noun ledger counts this insert iff its - // vector is real/non-empty (never true for a deferred embed, whose - // stub `vector` is `[]` — it counts later, at landing). tx.addOperation( - new SaveNounMetadataOperation(this.storage, id, storageMetadata, true, vector.length > 0) + new SaveNounMetadataOperation(this.storage, id, storageMetadata, true) ) // Operation 2: Save vector data @@ -3302,15 +2883,10 @@ export class Brainy implements BrainyInterface { }, true) ) - // Operation 3: Add to HNSW index (after entity saved). Gated on - // `vector.length > 0`, not `!deferringEmbed`: a deferred embed has - // nothing to index yet (the worker's atomic update inserts the real - // vector later), and an explicit `vector: []` insert (the VFS root's - // zero-norm fix — permanently unvectored plumbing, never embedded) - // is exactly the same "nothing to index yet" shape. The zero-norm - // BELT (a real all-zero vector, non-empty) is enforced inside - // AddToVectorIndexOperation itself — see its JSDoc. - if (vector.length > 0) { + // Operation 3: Add to HNSW index (after entity saved). A deferred + // embed has nothing to index yet — the worker's atomic update + // inserts the real vector. + if (!deferringEmbed) { tx.addOperation( new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration) ) @@ -3868,70 +3444,25 @@ export class Brainy implements BrainyInterface { // new `data`); otherwise new `data` re-embeds; otherwise the existing // vector is kept. Any vector change re-indexes HNSW below. let vector = existing.vector - // 'data' is a real new value whenever it's not null/undefined — an - // empty string ('') is legitimate content (e.g. truncating a file to - // empty via overwrite), matching validateUpdateParams's absent-vs-empty - // distinction. Using `Boolean(params.data)` here would treat '' as "no - // new data", silently skipping BOTH the deferred marker and the eager - // re-embed below — a stale vector left behind with no path to ever - // correct itself (a quiet loss, not the deferred-but-eventually- - // correct flicker the deferEmbedding contract promises). - const rawHasNewData = params.data !== undefined && params.data !== null - // NO RE-EMBED ON UNCHANGED DATA: a write carrying the row's CURRENT data - // is not a data change — no re-embed, no deferred landing, no vector - // rewrite. A host heartbeat re-writing an unchanged row every few - // seconds fed a live index-row loop on a production store (each - // "change" landed a vector); the amplifier dies here regardless of how - // often the host writes. - const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) - const hasNewData = rawHasNewData && !dataUnchanged - - // THE ZERO-NORM LAW (canonical write side) — see add()'s matching - // comment: an explicit REAL all-zero vector is not a vector. Normalize - // to the sanctioned "unvectored" `[]` shape BEFORE the dimension - // check, the unvector-door decision below, and the index ops ever see - // it — a local copy; `params.vector` itself is never mutated. - let explicitVector = params.vector - if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { - prodLog.warn( - `[Brainy] update(): entity ${params.id} was given an explicit all-zero vector — ` + - `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - explicitVector = [] - } - - // THE SANCTIONED UNVECTOR DOOR: `explicitVector` at length 0 (an - // explicit `vector: []`, or a real all-zero vector just normalized - // above) is an instruction to remove the vector NOW — never "please - // embed". `validateUpdateParams` already refuses combining it with - // `deferEmbedding: true` (an empty array is truthy, so that guard - // fires unconditionally on any explicit `vector`). Idempotent on an - // already-unvectored row: the ledger decrement near the end of this - // method is gated on the PRIOR vector actually having been real. - const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 - // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. const deferringEmbed = - params.deferEmbedding === true && hasNewData && !explicitVector - if (explicitVector) { - // A length-0 explicit vector (the unvector door) carries no - // dimension information — exempt from the check, mirroring add()'s - // own `vector.length > 0` gate on the dimension pin. - if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { + params.deferEmbedding === true && Boolean(params.data) && !params.vector + if (params.vector) { + if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` ) } - vector = explicitVector - } else if (hasNewData && !deferringEmbed) { + vector = params.vector + } else if (params.data && !deferringEmbed) { vector = await this.embed(params.data) } // A deferred data change does NOT reindex now (the vector is unchanged; // the worker's atomic swap carries the real reindex later). const needsReindexing = Boolean( - (hasNewData && !deferringEmbed) || params.type || explicitVector + (params.data && !deferringEmbed) || params.type || params.vector ) // Always update the noun with new metadata @@ -4025,22 +3556,6 @@ export class Brainy implements BrainyInterface { ? [this.enqueuePendingEmbed(params.id)] : undefined - // Leg D — the unvector door clears a PENDING deferred-embed marker: - // without this, the worker would later embed this row's current data - // and silently re-vector it, defeating the caller's explicit "remove - // the vector now" instruction. The clear rides THIS SAME commit fact - // (an `embed.landed` record with an empty vector — the recovery fold - // disarms a pending marker on ANY `embed.landed` for the id, - // regardless of the vector it carries), so a crash between the write - // and the in-memory clear below still recovers disarmed. Mutually - // exclusive with `embedMarkers` above: `deferringEmbed` requires an - // ABSENT `explicitVector`, so the two branches never both apply. - const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) - const commitRecords: FactMarkerRecord[] | undefined = - embedMarkers ?? (clearsPendingEmbed - ? [{ type: 'embed.landed', id: params.id, vector: [] }] - : undefined) - // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -4129,33 +3644,7 @@ export class Brainy implements BrainyInterface { } } ] - : undefined, commitRecords) - - // Leg D continued — the in-memory pending-embed clear runs only AFTER - // the commit above actually succeeded (an aborted update must not - // disarm a marker whose durable `embed.landed` twin was never - // written). - if (clearsPendingEmbed) { - this.clearPendingEmbed(params.id) - prodLog.warn( - `[Brainy] update(): entity ${params.id} had a pending deferred embed — ` + - `the unvector door cleared it ('vector: []' is an explicit instruction, ` + - `never "please embed").` - ) - } - - // Leg D — vectored-ledger decrement for the sanctioned unvector door. - // update()'s own metadata write goes through UpdateNounMetadataOperation - // (isNew=false), so the saveNounMetadata(..., hasVector) seam never - // fires here — noteVectorUnlanded is the ONLY seam, the same - // sanctioned hook unvectorNounForRootMigration() uses. Gated on the - // PRIOR vector having actually been real (non-empty, non-zero-norm): - // an already-unvectored row's second call is a true no-op — no - // decrement, matching the ledger-exactness law (never double-count, - // never drift negative). - if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { - await this.storage.noteVectorUnlanded?.(params.id) - } + : undefined, embedMarkers) // Aggregation hook (outside transaction — derived data). `existing` is // the full get() view — every reserved field top-level — and must be @@ -4172,117 +3661,6 @@ export class Brainy implements BrainyInterface { if (deferringEmbed) this.kickEmbedWorker() } - /** - * @description Build the metadata-index retraction operation for one id - * (noun or verb) — the null-metadata-safe closure shared by every removal - * leg that reaches the metadata index with a possibly-missed pre-read: - * `remove()`'s own noun leg, its verb-cascade retractions, `unrelate()`, - * and their `transact()`/`planTx*` mirrors (both callers add the returned - * operation to their own batch — `tx.addOperation()` for a single-op - * transaction, `plan.operations.push()` for a planned `transact()` batch). - * THE NULL-METADATA SKIP IS CLOSED (a posting-leak class): - * - metadata present → the ordinary, provider-agnostic - * `RemoveFromMetadataIndexOperation` (exact per-field retraction). - * - metadata absent (a torn pre-read, or the row was already gone) → - * a provider exposing `removeEntityById` (the id-keyed contract) gets - * exact per-entity retraction via its reverse record; the JS index - * gets `removeFromIndex(id)` — safe id-keyed cleanup (deleted bitmap + - * id mapper; field statistics reconcile at the next rebuild/repairIndex), - * narrated; a native provider WITHOUT the contract is never called - * metadata-omitted (that path walks its value space) — the skip is - * tracked in the degraded set instead, narrated, so `repairIndex()` - * reconciles it (and this method returns `null` — no operation to add). - * Silence is the only thing outlawed. - * @param id - The noun/verb id being retracted. - * @param metadata - The pre-read metadata/entity structure, or falsy when - * the read missed. - * @param context - Narration prefix identifying the caller/id, e.g. - * `remove(${id})` or `remove(${entityId}) cascade unrelate ${verbId}`. - * @returns The operation to add to the caller's batch, or `null` when - * nothing could be done (already narrated + tracked as degraded). - */ - /** - * @description A JSON-safe view of a record bound for the metadata-index - * crossing. The seam's metadata is JSON-safe BY CONTRACT (a native provider - * serializes it; u64 ints as Number corrupt above 2^53) — but - * {@link resolveVerbEndpointInts} MIRRORS the resolved endpoint ints onto - * the verb object itself as BigInt (`verb.sourceInt`/`targetInt`), so a - * verb object reused as index metadata carried BigInts into - * JSON.stringify, which throws, aborting the whole transaction (found by - * the first joint pair gate). Endpoint ints ride their OWN op params on the - * graph legs — the metadata crossing drops every BigInt-valued top-level - * key instead of guessing at a lossy numeric encoding. - * @param metadata - The candidate index-metadata record. - * @returns The same object when already JSON-safe, else a shallow copy - * without the BigInt-valued keys. - */ - private static jsonSafeIndexMetadata(metadata: unknown): unknown { - if (metadata === null || typeof metadata !== 'object') return metadata - const rec = metadata as Record - let hasBigint = false - for (const k in rec) { - if (typeof rec[k] === 'bigint') { hasBigint = true; break } - } - if (!hasBigint) return metadata - const out: Record = {} - for (const k in rec) { - if (typeof rec[k] !== 'bigint') out[k] = rec[k] - } - return out - } - - private metadataIndexRetractionOp( - id: string, - metadata: unknown, - context: string - ): Operation | null { - if (metadata) { - return new RemoveFromMetadataIndexOperation( - this.metadataIndex, id, Brainy.jsonSafeIndexMetadata(metadata), this.indexWriteGeneration - ) - } - const prov = this.metadataIndex as unknown as { - removeEntityById?: (id: string) => Promise - removeFromIndex?: (id: string, metadata?: unknown, generation?: bigint) => Promise - } - if (typeof prov.removeEntityById === 'function') { - const g = this.indexWriteGeneration - return { - name: 'RemoveEntityByIdTombstone', - execute: async () => { - await prov.removeEntityById!(id) - return async () => { - // Undo of an id-keyed tombstone on an absent row: nothing to - // restore (the row had no readable metadata to re-post). - void g - } - } - } - } else if (this.metadataIndex instanceof MetadataIndexManager) { - const gv = this.indexWriteGeneration - prodLog.warn( - `[Brainy] ${context}: no metadata at delete — id-keyed index cleanup ran ` + - `(deleted bitmap + id mapper); field statistics reconcile at the next rebuild/repairIndex.` - ) - return { - name: 'IdKeyedIndexCleanup', - execute: async () => { - await prov.removeFromIndex!(id, undefined, typeof gv === 'function' ? gv() : gv) - return async () => {} - } - } - } else { - this._indexDegradedIds.add(id) - prodLog.warn( - `[Brainy] ${context}: no metadata at delete and this provider has no id-keyed ` + - `removal — its postings for this id are NOT tombstoned yet (tracked as degraded; ` + - `repairIndex() reconciles). Never calling a metadata-omitted native removal: that ` + - `path walks the store's value space.` - ) - return null - } - } - /** * Remove an entity and all its relationships * @@ -4347,11 +3725,61 @@ export class Brainy implements BrainyInterface { ) } - // Operation 2: Remove from metadata index (null-metadata-safe — see - // metadataIndexRetractionOp's JSDoc for the full closure). - { - const retractionOp = this.metadataIndexRetractionOp(id, metadata, `remove(${id})`) - if (retractionOp) tx.addOperation(retractionOp) + // Operation 2: Remove from metadata index. THE NULL-METADATA SKIP IS + // CLOSED (a posting-leak class, confirmed at this site): when the + // pre-read missed, the leg no longer silently skips — + // - a provider exposing removeEntityById (the id-keyed contract) + // gets it: exact per-entity retraction via its reverse record; + // - the JS index gets removeFromIndex(id) — safe id-keyed cleanup + // (deleted bitmap + id mapper; field stats reconcile at rebuild); + // - a NATIVE provider WITHOUT the contract is never called + // metadata-omitted (that path walks its value space) — the skip + // happens, but NARRATED and tracked in the degraded set so + // repairIndex reconciles it. Silence is the only thing outlawed. + if (metadata) { + tx.addOperation( + new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration) + ) + } else { + const prov = this.metadataIndex as unknown as { + removeEntityById?: (id: string) => Promise + removeFromIndex?: (id: string, metadata?: unknown, generation?: bigint) => Promise + } + if (typeof prov.removeEntityById === 'function') { + const g = this.indexWriteGeneration + tx.addOperation({ + name: 'RemoveEntityByIdTombstone', + execute: async () => { + await prov.removeEntityById!(id) + return async () => { + // Undo of an id-keyed tombstone on an absent row: nothing + // to restore (the row had no readable metadata to re-post). + void g + } + } + }) + } else if (this.metadataIndex instanceof MetadataIndexManager) { + const gv = this.indexWriteGeneration + tx.addOperation({ + name: 'IdKeyedIndexCleanup', + execute: async () => { + await prov.removeFromIndex!(id, undefined, typeof gv === 'function' ? gv() : gv) + return async () => {} + } + }) + prodLog.warn( + `[Brainy] remove(${id}): no metadata at delete — id-keyed index cleanup ran ` + + `(deleted bitmap + id mapper); field statistics reconcile at the next rebuild/repairIndex.` + ) + } else { + this._indexDegradedIds.add(id) + prodLog.warn( + `[Brainy] remove(${id}): no metadata at delete and this provider has no id-keyed ` + + `removal — its postings for this id are NOT tombstoned yet (tracked as degraded; ` + + `repairIndex() reconciles). Never calling a metadata-omitted native removal: that ` + + `path walks the store's value space.` + ) + } } // Operation 3: Delete noun (full removal). The pre-read metadata rides @@ -4369,21 +3797,6 @@ export class Brainy implements BrainyInterface { tx.addOperation( new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) ) - // Retract the cascaded relation's metadata-index row too — the - // live mirror of what a rebuild would derive for this (now-gone) - // edge (mirrors the noun leg above). The whole hydrated verb - // (system fields top-level + the custom bag under `metadata`, - // same shape `extractIndexableFields` reads for any entity-record - // frame) is the before-image — every entry in `allVerbs` was - // already successfully hydrated by the reads above, so this is - // never metadata-omitted in practice, but the closure stays - // defensive rather than assuming. - { - const cascadeRetractionOp = this.metadataIndexRetractionOp( - verb.id, verb, `remove(${id}) cascade unrelate ${verb.id}` - ) - if (cascadeRetractionOp) tx.addOperation(cascadeRetractionOp) - } // Delete verb metadata tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) @@ -4563,9 +3976,8 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. A CHECK only — it never builds; throws a typed - // NotReady error if a provider's health report says it isn't serving. - this.ensureIndexesLoaded(['graph']) + // provider for 15 minutes. Fast path after the latch is one boolean. + await this.ensureIndexesLoaded() const entityInt = this.graphEntityInt(uuid) if (entityInt === undefined) return [] const neighborInts = await this.graphIndex.getNeighbors(entityInt, options) @@ -4593,72 +4005,72 @@ export class Brainy implements BrainyInterface { /** * @description Verify that the graph adjacency is actually LIVE before a graph read trusts * its result. A native graph index can load its relationship COUNT (manifest) on a cold open - * but NOT its source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even - * though edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would - * serve that `[]` as if it were truth. + * of a LARGE brain (≥10k nouns, which skips the eager index rebuild) but NOT its + * source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even though + * edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would serve + * that `[]` as if it were truth. * - * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the dark-rebuild - * failure mode this contract retires (open() alone owns building; see - * {@link rebuildIndexesIfNeeded}). Two detection strategies, in order of honesty: - * - **Preferred:** {@link assessProviderHealth} — the provider's named `healthReport()` when - * exposed, else its sync `isReady()`. Not serving → THROW {@link GraphIndexNotReadyError} - * naming the reasons, immediately — no rebuild attempt. - * - **Fallback (providers with neither signal):** a READ-ONLY GLOBAL known-edge sample (a real + * Two detection strategies, in order of honesty: + * - **Preferred (8.0 contract):** the provider exposes a sync `isReady()` that is true ONLY + * when the edges are loaded. `false` → hydrate the id-mapper (a native int adjacency + * resolves endpoints through it), rebuild from storage, and re-check `isReady()`; if it is + * still `false`, throw {@link GraphIndexNotReadyError} rather than returning `[]`. + * - **Fallback (providers without `isReady()`):** a GLOBAL known-edge sample (a real * persisted verb's `sourceId`, which by definition HAS an outgoing edge) — NOT any queried * anchor, because brainy cannot cheaply tell "adjacency unloaded" from "this node is - * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, THROW — - * the probe refuses loudly; it does not self-heal. + * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, the + * adjacency did not load: rebuild and re-probe; if even that fails, throw. * - * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing to - * verify). - * @throws {GraphIndexNotReadyError} when the index is not serving, or claims edges but cannot - * serve a known persisted edge. + * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing + * to verify), or `'rebuilt'` when a cold-unloaded adjacency was just healed from storage — + * in which case callers that observed an empty result must RE-RUN their collection. + * @throws {GraphIndexNotReadyError} when the index claims edges but cannot serve a known + * persisted edge (or stays not-ready) even after a rebuild. */ - private async verifyGraphAdjacencyLive(): Promise<'live'> { + private async verifyGraphAdjacencyLive(): Promise<'live' | 'rebuilt'> { if (this._graphAdjacencyVerified) return 'live' // Coordinated migration LOCK (#18): while the graph provider owns a locked - // rebuild-from-canonical, brainy must NOT judge it here — the provider owns - // its index until it verifies-and-swaps. The data-plane lock - // (awaitMigrationLock in ensureInitialized) already makes callers wait, so - // this is normally unreachable mid-migration; the guard is defensive. It + // rebuild-from-canonical, brainy must NOT fire its own graphIndex.rebuild() + // on a read — that would race the provider's in-place rebuild. The data-plane + // lock (awaitMigrationLock in ensureInitialized) already makes callers wait, + // so this is normally unreachable mid-migration; the guard is defensive. It // deliberately does NOT set `_graphAdjacencyVerified`, so the real verify runs // once the migration clears. if (this.providerIsMigrating(this.graphIndex)) return 'live' - // Re-entrancy: a fallback probe below calls getNeighbors(), which does not - // re-enter this guard, but the short-circuit is kept defensively cheap. + // Re-entrancy: rebuild() can trigger reads (neighbors/related) that call back into this + // guard. While a verify is in flight, short-circuit so we cannot recurse into rebuild(). if (this._graphAdjacencyVerifying) return 'live' this._graphAdjacencyVerifying = true try { - // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── - const assessment = assessProviderHealth(this.graphIndex) - if (assessment.via === 'health-report' || assessment.via === 'is-ready') { - if (assessment.readiness === 'ready') { + const gi = this.graphIndex as GraphAdjacencyIndex & { isReady?: () => boolean } + + // ── Strategy 1: honest isReady() signal (cortex >= 2.7.8 / 3.0) ────────── + if (typeof gi.isReady === 'function') { + if (gi.isReady()) { this._graphAdjacencyVerified = true return 'live' } - // A provider that is REBUILDING ITSELF gets a refusal that says so, - // with its own progress: open deliberately did not wait for it (see - // rebuildIndexesIfNeeded), so this door is temporarily closed and will - // open on its own. Anything else is a broken index needing a repair. - const rebuilding = assessProviderRebuild(this.graphIndex) - if (rebuilding) { - throw new GraphIndexNotReadyError( - `Graph adjacency index is ${describeRebuildProgress(rebuilding)} and is not serving ` + - `yet. find({ connected }), neighbors() and related() refuse rather than serve an ` + - `empty result. The brain is open and every other family is serving; this door opens ` + - `by itself when the provider reports serving — no action is needed.` + // Not ready: the edges did not load on open. Hydrate the id-mapper, then rebuild. + if (!this.config.silent) { + console.warn( + `[Brainy] Graph adjacency reports not-ready (isReady() === false) — the persisted ` + + `adjacency did not load on open. Rebuilding from storage…` ) } + await this.hydrateIdMapperForGraphRebuild() + await this.graphIndex.rebuild() + if (gi.isReady()) { + this._graphAdjacencyVerified = true + return 'rebuilt' + } throw new GraphIndexNotReadyError( - `Graph adjacency index is not serving (via ${assessment.via}): ` + - `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` + - `related() refuse rather than serve an empty result — rebuild via ` + - `repairIndex({ rebuild: ['graph'] }) or reopen the brain.` + `Graph adjacency index reports not-ready even after a rebuild — the persisted ` + + `adjacency could not be loaded. find({ connected }), neighbors() and related() ` + + `cannot be served reliably for this brain.` ) } - // ── Strategy 2: known-edge-sample probe (providers with neither signal) ─ - // READ-ONLY — refuses loudly on failure; never calls rebuild(). + // ── Strategy 2: known-edge-sample probe (providers without isReady()) ──── const claimed = await this.graphIndex.size() if (!claimed || claimed <= 0) return 'live' // no edges claimed — nothing to verify @@ -4674,9 +4086,10 @@ export class Brainy implements BrainyInterface { // the sample is not one of this brain's own edges — e.g. a shared on-disk store reused // across instances surfaces a foreign verb whose UUID this brain's resident mapper never // interned. We cannot prove a cold-unloaded adjacency from such a sample, so treat it as - // INCONCLUSIVE: mark verified and return 'live' rather than throwing. (The honest cold-load - // signal for native providers is Strategy 1, checked above; the JS baseline keeps its mapper - // resident, so its OWN edges always resolve.) + // INCONCLUSIVE: mark verified and return 'live' rather than rebuilding/throwing. (The honest + // cold-load signal for native providers is isReady(), checked above; the JS baseline keeps + // its mapper resident, so its OWN edges always resolve — the targeted 7.x failure mode, + // "mapper loaded but adjacency empty", still resolves the source and is detected below.) const sourceInt = this.graphEntityInt(verb.sourceId) if (sourceInt === undefined) { this._graphAdjacencyVerified = true @@ -4684,23 +4097,37 @@ export class Brainy implements BrainyInterface { } // Ask the adjacency for ONE neighbor of the (mapped) known-edge source. - const hasNeighbor = (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 - if (hasNeighbor) { + const probeKnownSource = async (): Promise => + (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 + + if (await probeKnownSource()) { this._graphAdjacencyVerified = true return 'live' // adjacency is live — the common case } // INCONSISTENT: the index reports edges but a KNOWN-mapped persisted edge's source has none → - // the adjacency did not load. Refuse loudly — never rebuild from a read. + // the adjacency did not load on open. Hydrate the mapper and rebuild from storage. + if (!this.config.silent) { + console.warn( + `[Brainy] Graph adjacency reports ${claimed} relationship(s) but a persisted edge ` + + `resolves to none — the persisted adjacency did not load on open. Rebuilding from storage…` + ) + } + await this.hydrateIdMapperForGraphRebuild() + await this.graphIndex.rebuild() + + if (await probeKnownSource()) { + this._graphAdjacencyVerified = true + return 'rebuilt' + } throw new GraphIndexNotReadyError( - `Graph adjacency index reports ${claimed} relationship(s) but a persisted edge's source ` + - `resolves to none — the persisted adjacency did not load. find({ connected }), ` + - `neighbors() and related() refuse rather than serve an empty result — rebuild via ` + - `repairIndex({ rebuild: ['graph'] }) or reopen the brain.` + `Graph adjacency index reports ${claimed} relationship(s) but returns no edges even ` + + `after a rebuild — the persisted adjacency could not be loaded. find({ connected }), ` + + `neighbors() and related() cannot be served reliably for this brain.` ) } catch (err) { if (err instanceof GraphIndexNotReadyError) throw err - // A transient probe failure must not break the actual query NOR be + // A transient probe/rebuild failure must not break the actual query NOR be // masked as "no data". Allow a re-check on the next graph read and fall through. this._graphAdjacencyVerified = false if (!this.config.silent) { @@ -4717,60 +4144,27 @@ export class Brainy implements BrainyInterface { * On a cold open a native metadata provider can report data yet not serve its * `where` postings, so `find({ where })` silently returns `[]` — the exact * failure a downstream deployment reported (cold reads blanking filtered pages - * after every restart). - * - * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the - * dark-rebuild failure mode this contract retires (open() alone owns - * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies: - * - **Preferred:** {@link assessProviderHealth} — the provider's named - * `healthReport()` when exposed, else its sync `isReady()`. Not serving → - * THROW {@link MetadataIndexNotReadyError} naming the reasons, immediately. - * - **Fallback (providers with neither signal):** a READ-ONLY known-value - * probe, run on the first FILTERED `find()`: take a KNOWN persisted entity - * + one of its plain field values and ask the index to resolve it. If the - * index does not return the known id, THROW — the probe refuses loudly; - * it does not self-heal. Inconclusive cases (empty store, no plain field - * to probe, a shared store surfacing a foreign entity) are treated as - * live — never a false throw. A migrating provider is skipped (it owns - * its locked rebuild). - * @returns `'live'` when the index serves. + * after every restart). This one-shot guard, run on the first FILTERED `find()`, + * closes that: it takes a KNOWN persisted entity + one of its plain field values + * and asks the index to resolve it. If the index returns the known id the field + * postings are live (the common case, and the ONLY cost on a warm brain — one + * O(1) probe). If it does not, the postings did not load: brainy rebuilds the + * index from the canonical records and re-probes; if it STILL cannot serve the + * known value it throws a loud {@link MetadataIndexNotReadyError} rather than + * let a silent `[]` stand. Inconclusive cases (empty store, no plain field to + * probe, a shared store surfacing a foreign entity) are treated as live — never + * a false rebuild. A migrating provider is skipped (it owns its locked rebuild). + * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. */ - private async verifyMetadataLive(): Promise<'live'> { + private async verifyMetadataLive(): Promise<'live' | 'rebuilt'> { if (this._metadataVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild — do // not race it. Defensive; the data-plane lock already gates callers upstream. if (this.providerIsMigrating(this.metadataIndex)) return 'live' - // Re-entrancy: the fallback probe below calls filterIdsBelted(), which - // re-enters ensureIndexesLoaded() (a cheap CHECK) but not this guard. + // Re-entrancy: rebuild() can trigger reads that call back into this guard. if (this._metadataVerifying) return 'live' this._metadataVerifying = true try { - // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── - const assessment = assessProviderHealth(this.metadataIndex) - if (assessment.via === 'health-report' || assessment.via === 'is-ready') { - if (assessment.readiness === 'ready') { - this._metadataVerified = true - return 'live' - } - const rebuilding = assessProviderRebuild(this.metadataIndex) - if (rebuilding) { - throw new MetadataIndexNotReadyError( - `Metadata field index is ${describeRebuildProgress(rebuilding)} and is not serving ` + - `yet. find({ where }) and other filtered reads refuse rather than serve an empty ` + - `result. The brain is open and every other family is serving; this door opens by ` + - `itself when the provider reports serving — no action is needed.` - ) - } - throw new MetadataIndexNotReadyError( - `Metadata field index is not serving (via ${assessment.via}): ` + - `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` + - `reads refuse rather than serve an empty result — rebuild via ` + - `repairIndex({ rebuild: ['metadata'] }) or reopen the brain.` - ) - } - - // ── Strategy 2: known-value probe (providers with neither signal) ────── - // READ-ONLY — refuses loudly on failure; never calls rebuild(). // A KNOWN persisted entity + one plain field to probe. Sample a few so a // system-only entity (e.g. the VFS root) doesn't make every open inconclusive. const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) @@ -4792,7 +4186,7 @@ export class Brainy implements BrainyInterface { return ids.includes(p.id) } catch { // FIELD_NOT_INDEXED for a field a persisted entity actually holds is - // itself the cold/broken signal — treat as not-serving. + // itself the cold/broken signal — treat as not-serving (→ rebuild). return false } } @@ -4802,15 +4196,26 @@ export class Brainy implements BrainyInterface { return 'live' // field postings are live — the common case } + if (!this.config.silent) { + console.warn( + `[Brainy] Metadata field index returns no match for a known persisted value of ` + + `'${p.field}' — the field postings did not load on open. Rebuilding from storage…` + ) + } + await this.metadataIndex.rebuild() + + if (await probeServes()) { + this._metadataVerified = true + return 'rebuilt' + } throw new MetadataIndexNotReadyError( - `Metadata field index cannot serve a known persisted value of '${p.field}' — the field ` + - `postings did not load. find({ where }) and other filtered reads refuse rather than ` + - `serve an empty result — rebuild via repairIndex({ rebuild: ['metadata'] }) or reopen ` + - `the brain.` + `Metadata field index cannot serve a known persisted value of '${p.field}' even after ` + + `a rebuild — find({ where }) and other filtered reads cannot be served reliably for ` + + `this brain (a silent empty result would misrepresent existing data).` ) } catch (err) { if (err instanceof MetadataIndexNotReadyError) throw err - // A transient probe failure must not break the query NOR mask as + // A transient probe/rebuild failure must not break the query NOR mask as // "no data". Allow a re-check on the next filtered read and fall through. this._metadataVerified = false if (!this.config.silent) { @@ -4856,61 +4261,57 @@ export class Brainy implements BrainyInterface { * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded * its serving structure (the mmap/DiskANN graph) — so a pure semantic * `find({ query })` silently returns `[]`. A pure semantic query has - * `hasFilterCriteria === false`, so the metadata guard never fires; this - * guard closes that gap. Run one-shot on the first vector/proximity search. - * - * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the - * dark-rebuild failure mode this contract retires (open() alone owns - * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies: - * - **Preferred:** {@link assessProviderHealth} — the provider's named - * `healthReport()` when exposed, else its sync `isReady()`. Not serving → - * THROW {@link VectorIndexNotReadyError} naming the reasons, immediately. - * - **Fallback (providers with neither signal):** a READ-ONLY KNOWN - * persisted vector (sampled + hydrated) is searched against the index; if - * it does not self-match, THROW — the probe refuses loudly; it does not - * self-heal. + * `hasFilterCriteria === false`, so the metadata guard never fires; this guard + * closes that gap. Run one-shot on the first vector/proximity search: + * - **Preferred (honest signal):** the provider exposes `isReady()`. `false` + * → rebuild from storage, re-check; if still `false`, throw + * {@link VectorIndexNotReadyError} rather than serving `[]`. + * - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled + + * hydrated) is searched against the index; if it does not self-match, the + * serving structure did not load — rebuild + re-probe, else throw. * Inconclusive cases (empty store, no probeable vector, `size()===0` — where - * the JS baseline is built at open) are treated as live: never a false - * throw. A migrating provider is skipped (it owns its locked rebuild). - * @returns `'live'` when the index serves. + * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as + * live: never a false rebuild. A migrating provider is skipped (it owns its + * locked rebuild). + * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. */ - private async verifyVectorLive(): Promise<'live'> { + private async verifyVectorLive(): Promise<'live' | 'rebuilt'> { if (this._vectorVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild. if (this.providerIsMigrating(this.index)) return 'live' - // Re-entrancy: the fallback probe below calls index.search(), which does - // not re-enter this guard, but the short-circuit is kept defensively cheap. + // Re-entrancy: rebuild() can trigger reads that call back into this guard. if (this._vectorVerifying) return 'live' this._vectorVerifying = true try { - // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── - const assessment = assessProviderHealth(this.index) - if (assessment.via === 'health-report' || assessment.via === 'is-ready') { - if (assessment.readiness === 'ready') { + // ── Strategy 1: honest isReady() signal (native provider) ────────────── + const readiness = assessIndexReadiness(this.index) + if (readiness !== 'unknown') { + if (readiness === 'ready') { this._vectorVerified = true return 'live' } - const rebuilding = assessProviderRebuild(this.index) - if (rebuilding) { - throw new VectorIndexNotReadyError( - `Vector index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + - `Semantic find({ query }) and proximity search refuse rather than serve an empty ` + - `result. The brain is open and every other family is serving; this door opens by ` + - `itself when the provider reports serving — no action is needed.` + // Not ready: the serving structure did not load on open. Rebuild. + if (!this.config.silent) { + console.warn( + `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` + + `vector index did not load on open. Rebuilding from storage…` ) } + await this.index.rebuild() + if (assessIndexReadiness(this.index) === 'ready') { + this._vectorVerified = true + return 'rebuilt' + } throw new VectorIndexNotReadyError( - `Vector index is not serving (via ${assessment.via}): ` + - `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` + - `proximity search refuse rather than serve an empty result — rebuild via ` + - `repairIndex({ rebuild: ['vector'] }) or reopen the brain.` + `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` + + `proximity search cannot be served reliably for this brain (a silent empty result ` + + `would misrepresent existing data).` ) } - // ── Strategy 2: known-vector probe (providers with neither signal) ───── - // READ-ONLY — refuses loudly on failure; never calls rebuild(). + // ── Strategy 2: known-vector probe (providers without isReady()) ─────── const claimed = this.index.size() - if (!claimed || claimed <= 0) return 'live' // JS cold path is built at open + if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job const probe = await this.pickVectorProbe() if (!probe) { @@ -4920,30 +4321,44 @@ export class Brainy implements BrainyInterface { } const p = probe - // The failure mode we guard is the SILENT EMPTY result: a cold index that - // loaded its COUNT but not its serving structure returns `[]` for a - // known-present vector, while a warm index returns at least one hit. We - // check for a NON-EMPTY result, NOT an exact self-match — HNSW is - // approximate and `get()` may return a re-hydrated/normalized vector, so - // demanding the exact self as top-1 would false-positive on a perfectly - // healthy index (and wrongly throw). - const hits = await this.index.search(p.vector, 1) + const probeServes = async (): Promise => { + // The failure mode we guard is the SILENT EMPTY result: a cold index that + // loaded its COUNT but not its serving structure returns `[]` for a + // known-present vector, while a warm index returns at least one hit. We + // check for a NON-EMPTY result, NOT an exact self-match — HNSW is + // approximate and `get()` may return a re-hydrated/normalized vector, so + // demanding the exact self as top-1 would false-positive on a perfectly + // healthy index (and wrongly rebuild → throw). + const hits = await this.index.search(p.vector, 1) + return hits.length > 0 + } void p.id // probe keyed on the vector; id retained for diagnostics only - if (hits.length > 0) { + if (await probeServes()) { this._vectorVerified = true return 'live' // serving structure is live — the common case } + if (!this.config.silent) { + console.warn( + `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` + + `returns no results — the serving structure did not load on open. Rebuilding…` + ) + } + await this.index.rebuild() + + if (await probeServes()) { + this._vectorVerified = true + return 'rebuilt' + } throw new VectorIndexNotReadyError( `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` + - `results — the serving structure did not load. Semantic find({ query }) refuses rather ` + - `than serve an empty result — rebuild via repairIndex({ rebuild: ['vector'] }) or ` + - `reopen the brain.` + `results even after a rebuild — semantic find({ query }) cannot be served reliably ` + + `for this brain (a silent empty result would misrepresent existing data).` ) } catch (err) { if (err instanceof VectorIndexNotReadyError) throw err - // A transient probe failure must not break the query NOR mask as + // A transient probe/rebuild failure must not break the query NOR mask as // "no data". Allow a re-check on the next vector read and fall through. this._vectorVerified = false if (!this.config.silent) { @@ -5248,16 +4663,6 @@ export class Brainy implements BrainyInterface { ) ) - // Operation 3b: Add the verb's metadata-index row, in the SAME - // commit as the graph leg — the live mirror of what rebuild()'s - // verb walk already derives (ADR-007 A4: one mechanism, never a - // second hand-rolled shape). `verbMetadata` is the exact raw stored - // record `SaveVerbMetadataOperation` above just persisted — the same - // shape `storage.getVerbMetadata()`/rebuild() read back. - tx.addOperation( - new AddToMetadataIndexOperation(this.metadataIndex, id, verbMetadata, this.indexWriteGeneration) - ) - // Create bidirectional if requested if (params.bidirectional && reverseId) { const reverseVerb: GraphVerb = { @@ -5295,13 +4700,6 @@ export class Brainy implements BrainyInterface { (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) - - // Operation 6b: Add the reverse edge's metadata-index row (same - // stored shape as the primary edge — SaveVerbMetadataOperation - // above persists the same `verbMetadata` object for both). - tx.addOperation( - new AddToMetadataIndexOperation(this.metadataIndex, reverseId, verbMetadata, this.indexWriteGeneration) - ) } }, undefined, @@ -5384,15 +4782,6 @@ export class Brainy implements BrainyInterface { ) } - // Operation 1b: Retract the verb's metadata-index row — the live - // mirror of remove()'s cascade leg (null-metadata-safe; see - // metadataIndexRetractionOp's JSDoc). Nothing to retract when the - // pre-read found no verb (already gone / never existed). - if (verb) { - const retractionOp = this.metadataIndexRetractionOp(id, verb, `unrelate(${id})`) - if (retractionOp) tx.addOperation(retractionOp) - } - // Operation 2: Delete verb metadata (which also deletes vector) tx.addOperation( new DeleteVerbMetadataOperation(this.storage, id) @@ -5536,23 +4925,6 @@ export class Brainy implements BrainyInterface { new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) ) - // Re-post the verb's metadata-index row — remove the old shape, add - // the new one, same commit (the plain pair; there is no update-op - // capability for the metadata leg yet — see the GRAPH leg's - // typeChanged branch just below for the capability this ISN'T: - // that's the graph adjacency's own remove+add, keyed on the verb - // TYPE changing; the metadata row updates on EVERY updateRelation() - // call, since metadata/subtype/weight/etc. can all change without a - // type change). `existing` is the pre-update hydrated verb (already - // read above); `updatedMetadata` is the raw stored record just - // persisted — the same shape relate()/rebuild() use to add. - tx.addOperation( - new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, Brainy.jsonSafeIndexMetadata(existing), this.indexWriteGeneration) - ) - tx.addOperation( - new AddToMetadataIndexOperation(this.metadataIndex, params.id, updatedMetadata, this.indexWriteGeneration) - ) - // If the verb type changed, re-index in graph adjacency so traversal-by-type // stays consistent. The id is preserved across the swap. if (typeChanged && reindexInts) { @@ -7187,10 +6559,14 @@ export class Brainy implements BrainyInterface { // loader and cold-read probes below already defer to a migrating provider. await this.ensureInitialized({ needs: [] }) - // READ-SURFACE READINESS GATE (see filterIdsBelted): a CHECK only — it - // never builds. open() already brought every provider to serving before - // init() returned; this throws a typed NotReady error if one isn't. - this.ensureIndexesLoaded(['metadata']) + // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) + // This is a production-safe, concurrency-controlled lazy load + await this.ensureIndexesLoaded() + + // One-shot cold-open self-heal: an O(1) probe of the metadata index (when the + // provider offers one) repairs an already-poisoned index on first read — the + // metadata counterpart of the graph cold-load guard. No-op for the JS index. + await this.ensureMetadataConsistencyProbed() // Loudly flag a degraded derived index (failed init rebuild, or an // adopt-forward degraded commit) so a partial result is never mistaken for @@ -7202,13 +6578,6 @@ export class Brainy implements BrainyInterface { let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query - // The vector and graph legs gate only the finds that consult them. - const consultsVector = Boolean( - (params.query && params.query.trim() !== '') || params.vector || params.near - ) - if (consultsVector) this.ensureIndexesLoaded(['vector']) - if (params.connected) this.ensureIndexesLoaded(['graph']) - // Id normalization (8.0): resolve the graph-traversal anchor id(s) so a // caller may constrain by natural key. Each maps to the canonical UUID // add() stored; real UUIDs pass through. Done once here so every downstream @@ -8557,11 +7926,6 @@ export class Brainy implements BrainyInterface { */ async clear(): Promise { await this.ensureInitialized() - // A clear mutates durable state without going through a commit path, so - // it must set the dirty witness itself — otherwise a `clear()` followed by - // `flush()` would find the brain "clean" and skip the entity-tree stamp, - // leaving a stamp that describes the population this call just removed. - this._dirtySinceLastFlush = true // Clear storage await this.storage.clear() @@ -9547,15 +8911,6 @@ export class Brainy implements BrainyInterface { hook() } - // Leg D — vectored-ledger decrements for this batch's unvector-door - // updates (see planTxUpdate's matching comment), applied after the - // commit point and properly awaited (unlike `postCommit`'s synchronous - // fire-and-forget hooks) — each is the same sanctioned hook - // unvectorNounForRootMigration() uses. - for (const id of plan.vectorUnlands) { - await this.storage.noteVectorUnlanded?.(id) - } - // Change feed: the batch's events share its single committed generation. // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) @@ -10638,18 +9993,6 @@ export class Brainy implements BrainyInterface { for (const id of nounIds) { const noun = await snapshotStorage.getNoun(id) if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) { - // THE ZERO-NORM LAW: a direct provider-write seam (this materializer - // inserts one-by-one, bypassing AddToVectorIndexOperation's own - // belt) — apply the same refusal here rather than handing a false - // attractor to the ephemeral reader's index. - if (isZeroNormVector(noun.vector)) { - prodLog.warn( - `[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` + - `entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` + - `engine boundary (the materialized record is unaffected)` - ) - continue - } await reader.index.addItem({ id: noun.id, vector: noun.vector }) } } @@ -10799,8 +10142,7 @@ export class Brainy implements BrainyInterface { casUpdates: [], createdNouns: new Set(), changeEvents: [], - markerRecords: [], - vectorUnlands: [] + markerRecords: [] } for (const op of ops) { @@ -10922,26 +10264,10 @@ export class Brainy implements BrainyInterface { // marker-less committed row would be a silently missing vector, which is // the disallowed direction). The background worker embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - let vector = deferringEmbed + const vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) - - // THE ZERO-NORM LAW — see the single-add() insert path's matching - // comment (a zero-norm vector is not a vector; never crosses an engine - // boundary). Normalized here BEFORE the dimension pin and the - // vectored-ledger `hasVector` flag below ever see it. - if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { - prodLog.warn( - `[Brainy] transact add: entity ${id} was given an explicit all-zero vector — ` + - `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - vector = [] - } - - // Gated on `vector.length > 0` — see the single-add() insert path's - // matching comment: an explicit `vector: []` carries no dimension - // information either, deferred or not. - if (!deferringEmbed && vector.length > 0) { + if (!deferringEmbed) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -11019,16 +10345,11 @@ export class Brainy implements BrainyInterface { plan.postCommit.push(() => this.kickEmbedWorker()) } plan.operations.push( - // hasVector: see the single-add() insert path's comment — never true - // for a deferred embed (stub vector `[]`; counted later at landing). - new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0), + new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), - // Gated on `vector.length > 0` — see the single-add() insert path's - // matching comment: an explicit `vector: []` has nothing to index - // either, deferred or not. - ...(vector.length > 0 - ? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)] - : []), + ...(deferringEmbed + ? [] + : [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]), new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration) ) plan.touchedNouns.push(id) @@ -11099,71 +10420,18 @@ export class Brainy implements BrainyInterface { // Resolve the updated vector — mirror of update(): an explicit `vector` // always wins, new `data` re-embeds, otherwise the existing vector is // kept. Any vector change re-indexes HNSW below. - // 'data' is present whenever it's not null/undefined — '' is real - // content (see the identical hasNewData in update()); a plain truthy - // check would silently skip re-embedding an emptied value and leave a - // stale vector with no path to ever correct itself. - const rawHasNewData = params.data !== undefined && params.data !== null - // No re-embed on unchanged data — the transact() mirror of update()'s rule. - const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) - const hasNewData = rawHasNewData && !dataUnchanged let vector = existing.vector - - // THE ZERO-NORM LAW + THE SANCTIONED UNVECTOR DOOR — transact() mirror - // of update()'s matching block: an explicit REAL all-zero vector - // normalizes to `[]` (never crosses an engine boundary), and an - // explicit `vector: []` (post-normalization) is the sanctioned unvector - // instruction, exempt from the dimension check. `validateUpdateParams` - // already refuses combining it with `deferEmbedding: true`. - let explicitVector = params.vector - if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { - prodLog.warn( - `[Brainy] transact update: entity ${params.id} was given an explicit all-zero ` + - `vector — a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - explicitVector = [] - } - const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 - - if (explicitVector) { - if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { + if (params.vector) { + if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` ) } - vector = explicitVector - } else if (hasNewData) { + vector = params.vector + } else if (params.data) { vector = await this.embed(params.data) } - const needsReindexing = Boolean(hasNewData || params.type || explicitVector) - - // Leg D — the unvector door clears a PENDING deferred-embed marker (see - // update()'s matching comment for the full rationale): the durable - // clear (an `embed.landed` record, empty vector) rides the batch's ONE - // commit fact via `plan.markerRecords`; the in-memory clear is deferred - // to `plan.postCommit` so an aborted batch never disarms a marker whose - // durable twin was never written. - const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) - if (clearsPendingEmbed) { - plan.markerRecords.push({ type: 'embed.landed', id: params.id, vector: [] }) - plan.postCommit.push(() => { - this.clearPendingEmbed(params.id) - prodLog.warn( - `[Brainy] transact update: entity ${params.id} had a pending deferred embed — ` + - `the unvector door cleared it ('vector: []' is an explicit instruction, ` + - `never "please embed").` - ) - }) - } - - // Leg D — vectored-ledger decrement for the sanctioned unvector door, - // deferred to `plan.vectorUnlands` (consumed with a proper `await` in - // `transact()`, AFTER the commit succeeds — see its matching comment). - // Gated on the PRIOR vector having actually been real (non-empty, - // non-zero-norm): idempotent on an already-unvectored row. - if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { - plan.vectorUnlands.push(params.id) - } + const needsReindexing = Boolean(params.data || params.type || params.vector) const newMetadata = params.merge !== false @@ -11361,15 +10629,6 @@ export class Brainy implements BrainyInterface { new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration), new DeleteVerbMetadataOperation(this.storage, verb.id) ) - // Retract the cascaded relation's metadata-index row too — the - // transact() mirror of remove()'s single-op cascade leg - // (null-metadata-safe; see metadataIndexRetractionOp's JSDoc). - { - const cascadeRetractionOp = this.metadataIndexRetractionOp( - verb.id, verb, `transact remove(${id}) cascade unrelate ${verb.id}` - ) - if (cascadeRetractionOp) plan.operations.push(cascadeRetractionOp) - } plan.touchedVerbs.push(verb.id) state.verbs.delete(verb.id) state.removedVerbs.add(verb.id) @@ -11536,10 +10795,7 @@ export class Brainy implements BrainyInterface { // id mapper to assign an int for an entity that did not exist yet. new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) - ), - // The transact() mirror of relate()'s metadata-index leg — same - // commit as the graph leg, same raw stored shape. - new AddToMetadataIndexOperation(this.metadataIndex, id, verbMetadata, this.indexWriteGeneration) + ) ) plan.touchedVerbs.push(id) state.verbs.set(id, verb) @@ -11581,8 +10837,7 @@ export class Brainy implements BrainyInterface { new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata), new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) - ), - new AddToMetadataIndexOperation(this.metadataIndex, reverseId, verbMetadata, this.indexWriteGeneration) + ) ) plan.touchedVerbs.push(reverseId) state.verbs.set(reverseId, reverseVerb) @@ -11627,12 +10882,6 @@ export class Brainy implements BrainyInterface { // may have been created earlier in this same batch (forward refs). new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration) ) - // The transact() mirror of unrelate()'s metadata-index leg - // (null-metadata-safe; see metadataIndexRetractionOp's JSDoc — a - // present `verb` here is never metadata-omitted, but the closure - // stays defensive rather than assuming). - const retractionOp = this.metadataIndexRetractionOp(id, verb, `transact unrelate(${id})`) - if (retractionOp) plan.operations.push(retractionOp) } plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id)) plan.touchedVerbs.push(id) @@ -12341,31 +11590,6 @@ export class Brainy implements BrainyInterface { } } - /** - * @description Stamp every projection's watermark with the store's - * current committed generation — the door BOTH {@link flush} and {@link - * close} open right before persisting, so EITHER path leaves a stamped, - * `'adopt'`-verdicting artifact on disk (stamp-after-data still holds - * inside each owner: this only hands the generation over — the owner's - * OWN flush is what durably writes the stamp, LAST). Before this method - * existed, `close()` had its own separate flush fan-out that never - * stamped, so a `close()` without a preceding explicit `flush()` left - * every projection unstamped — a real, closed store that legitimately - * verdicts `'rescan'` on its very next open (not a bug in the verdict, - * a gap in `close()`'s persistence completeness that this closes). - * No `committedGeneration` capability, or a replacement provider that - * doesn't carry the stamp method (a native pair swaps these managers) = - * no stamp = the owner's verdict machinery treats the artifact as - * legacy — never a flush/close crash either way. - */ - private stampProjectionWatermarks(): void { - const wmGen = this.storage?.committedGeneration?.() ?? null - if (wmGen === null) return - ;(this.metadataIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - ;(this.index as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - ;(this.graphIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - } - /** * Flush all indexes and caches to persistent storage * CRITICAL FIX: Ensures data survives server restarts @@ -12396,27 +11620,6 @@ export class Brainy implements BrainyInterface { return } - // A CLEAN BRAIN FLUSHES NOTHING, AND SAYS NOTHING. No write has been - // committed since the last flush, so every step below would re-persist - // state identical to what is already on disk — provider flushes, the - // watermark stamps, the generation counter, the entity-tree stamp — and - // print two lines announcing it. The witness is set by every committed - // write (see noteWriteForPersistence) and cleared here; a write landing - // DURING this flush sets it again, so it is never lost — the next flush - // does that write's work. This makes an unexplained flush FREE; it does - // not explain one (see _dirtySinceLastFlush). - if (!this._dirtySinceLastFlush) { - return - } - this._dirtySinceLastFlush = false - // An explicit flush IS a flush: tell the cadence so, or the very next - // write sees "30s since the last flush" (the cadence only counted its - // own) and kicks a background flush that has nothing left to do, and the - // idle timer fires two seconds later over writes this flush already - // persisted. - this._persistLastFlushAt = Date.now() - this._persistDirtyWrites = 0 - console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() @@ -12426,8 +11629,22 @@ export class Brainy implements BrainyInterface { await this.generationStore.flushPendingSingleOps() // Flush all components in parallel for performance - // Watermark stamps ride every flush fan-out — see stampProjectionWatermarks(). - this.stampProjectionWatermarks() + // Watermark stamps ride every flush fan-out: stamp each projection with + // the committed generation BEFORE its flush persists (stamp-after-data + // holds inside each owner — the stamp is its LAST write; here we only + // hand the generation over). No committedGeneration capability = no + // stamp = the owner's verdict machinery treats the artifact as legacy. + { + const wmGen = this.storage?.committedGeneration?.() ?? null + if (wmGen !== null) { + // ALL THREE optional-chained: a replacement provider (the native + // pair swaps these managers) may not carry the stamp method — a + // missing stamp is a verdict-side rescan, never a flush crash. + ;(this.metadataIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) + ;(this.index as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) + ;(this.graphIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) + } + } await Promise.all([ // 1. Flush storage adapter counts (entity/verb counts by type) (async () => { @@ -12497,18 +11714,6 @@ export class Brainy implements BrainyInterface { * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. - * - * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is - * the ALLOCATED counter — a number a write in flight has claimed and may - * never commit. Stamping it made the stamp's generation label a claim about - * counts it was not taken at, and every crash inside a write window then - * produced a spurious verdict at the next open: either `sourceGeneration N - * is ahead of the log head N-1` (the allocated generation died with the - * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the - * recovery fold folded facts the stamp's counts predate). MEASURED on the - * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on - * a coherent store raised one of those two verdicts, each of them naming - * `repairIndex()` — a whole-store recount — as the cure for nothing. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return @@ -12519,7 +11724,7 @@ export class Brainy implements BrainyInterface { ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', - sourceGeneration: this.generationStore.committedGeneration(), + sourceGeneration: this.generationStore.generation(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { @@ -12532,24 +11737,16 @@ export class Brainy implements BrainyInterface { /** * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the store's COMMITTED generation and - * the stamped rollup invariants against the live counters. Verdicts: + * compare `sourceGeneration` against the log head and the stamped rollup + * invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) → silent. * - `behind` → benign for the tree (it is written BY the commit; only the * stamp is stale — a crash landed between commit and flush). Refreshed at * the next flush. - * - `torn` → a TORN GENERATION-LOG TAIL, handled by - * {@link demoteTornEntityTreeStamp}: terminal, never a wait. * - `incoherent` → LOUD: the tree or its counters diverged from what was * stamped — `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. - * - * THE COMPARISON IS AGAINST `committedGeneration()`, matching what - * {@link stampEntityTree} writes and what every other open-time watermark in - * this class already reasons about (the fact-scan capability, the metadata / - * graph / HNSW watermark verdicts). Comparing against the allocated counter - * was the one place that disagreed, and disagreeing was the whole defect. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null @@ -12566,16 +11763,11 @@ export class Brainy implements BrainyInterface { this.storage.getNounCount(), this.storage.getVerbCount() ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), { + const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { nounCount, verbCount }) - if (verdict.state === 'torn') { - await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, { - nounCount, - verbCount - }) - } else if (verdict.state === 'incoherent') { + if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state — run ` + @@ -12589,92 +11781,6 @@ export class Brainy implements BrainyInterface { } } - /** - * @description THE TERMINAL VERDICT for a torn generation-log tail. - * - * A stamp whose `sourceGeneration` sits ABOVE the store's committed - * watermark witnesses a generation that is not in the log: the stamp's fsync - * outlived the tail's. By the time this runs, log-authority recovery has - * already folded every intact fact above the manifest and advanced the - * watermark to cover them — so if the stamp is STILL ahead, the generation - * it names is not merely late, it is GONE. There is nothing to wait for. - * - * That is the whole point of this method. A field report of this class - * (single-process store, abrupt termination mid-fold) described a reopen - * that narrated the tear and then held 100% CPU with zero log growth for - * eight minutes before an operator wiped the directory. A recovery that - * cannot say what it is waiting for has no business spinning; the honest - * answer here is a verdict, taken now, at O(1) cost. - * - * WHAT THE VERDICT DOES — the stamped surface is UNUSABLE, so it is - * discarded rather than believed: the stamped counts describe a generation - * that never became durable, and comparing them against live counters can - * only produce noise. The tree itself is not in question (it IS canonical — - * every commit writes it, and the fold re-applied every after-image the log - * still holds), so the demotion is a re-derivation of this family's verified - * surface at the generation the store can actually show: - * - * - WRITER open → re-stamp at `committedGeneration()` from the live - * counters — exactly what the next flush would write, taken now so the - * tear cannot re-narrate on every subsequent open. Both count sets are - * logged so an operator can see whether anything really moved. - * - READER open → a reader cannot re-stamp. Narrate the same terminal - * verdict with the named cure and carry on serving; a read-only inspector - * is never locked out of a store, and never left waiting either. - * - * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any - * external progress signal — the two counter reads and one stamp write are - * the entire cost, and none of them scales with the store. - */ - private async demoteTornEntityTreeStamp( - stamp: FamilyStamp, - stampSource: number, - head: number, - observed: { nounCount: number; verbCount: number } - ): Promise { - const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {} - const detail = - `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` + - `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` + - `generation is ${head} after crash recovery — the stamp's fsync outlived the log tail's, ` + - `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` + - `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.` - - if (this.isReadOnly) { - prodLog.warn( - `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` + - `family stays UNVERIFIED for this session (reads are unaffected — the canonical tree ` + - `is the truth this stamp only describes). Cure: open the store with a writer, or run ` + - `brain.repairIndex() there, to recount from canonical and re-stamp.` - ) - return - } - - const startedAt = Date.now() - try { - await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { - family: 'entity-tree', - sourceGeneration: head, - members: { - mode: 'rollup', - invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount } - } - }) - prodLog.warn( - `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` + - `from the live counters in ${Date.now() - startedAt}ms — terminal, not a wait. If the ` + - `observed counts above look wrong for your data, run brain.repairIndex() to recount ` + - `from canonical.` - ) - } catch (error) { - prodLog.warn( - `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) — the tear will ` + - `narrate again at the next open, which is the honest outcome; the store still serves ` + - `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.` - ) - } - } - /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. @@ -12745,10 +11851,10 @@ export class Brainy implements BrainyInterface { } /** - * Get index loading status (diagnostic) + * Get index loading status (Diagnostic for lazy loading) * - * Returns detailed information about index population state. Useful for - * debugging empty query results or performance troubleshooting. + * Returns detailed information about index population and lazy loading state. + * Useful for debugging empty query results or performance troubleshooting. * * @example * ```typescript @@ -12757,7 +11863,7 @@ export class Brainy implements BrainyInterface { * console.log(`Metadata Index: ${status.metadataIndex.entries} entries`) * console.log(`Graph Index: ${status.graphIndex.relationships} relationships`) * console.log(`Pending embeds: ${status.projections.semantic.pendingEmbeds}`) - * console.log(`Index build completed at open: ${status.lazyRebuildCompleted}`) + * console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`) * ``` */ @@ -12776,9 +11882,8 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. A CHECK only — it never builds; throws a typed - // NotReady error if a provider's health report says it isn't serving. - this.ensureIndexesLoaded(['metadata']) + // provider for 15 minutes. Fast path after the latch is one boolean. + await this.ensureIndexesLoaded() try { return await this.metadataIndex.getIdsForFilter(filter, opts) } catch (err) { @@ -12790,10 +11895,6 @@ export class Brainy implements BrainyInterface { async getIndexStatus(): Promise<{ initialized: boolean - /** `true` once open()'s index-build-if-needed step has run. Named for API - * compatibility with the retired first-query lazy-build path; a needed - * rebuild now always runs at open, never deferred to a read, so this is - * simply `initialized`'s index-build counterpart. */ lazyRebuildCompleted: boolean /** Deferred embeds not yet landed (MT5) — the eventual-vector-index backlog. */ pendingEmbeds: number @@ -12999,49 +12100,21 @@ export class Brainy implements BrainyInterface { const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() - // 1. Index size parity. HNSW must hold one node per VECTORED noun — the - // vectored-noun ledger (`getCanonicalCounts().vectors.all`), NOT the raw - // metadata-entry count: every store's VFS root is PERMANENTLY unvectored - // (`vector: []` by design — a zero-norm/empty vector never crosses into - // the index, see AddToVectorIndexOperation/JsHnswVectorIndex.rebuild()'s - // matching belts), and a not-yet-landed deferred embed is unvectored - // too. Comparing against total entries counted the always-unvectored - // root as a permanent 1-node "drift" on every VFS-having store — a false - // warn on an otherwise perfectly healthy handoff. `vectors.all` is - // already the documented coverage denominator for exactly this - // comparison (see `CanonicalCounts.vectors`'s JSDoc). Falls back to the - // metadata-entry count when the ledger is unavailable or suspect (a - // storage adapter without the optional hook, or an unrecounted store) — - // never worse than the prior behavior in that case. - const vectorLedgerForParity = await this.storage.getCanonicalCounts?.() - const vectorParityTarget = - vectorLedgerForParity && !vectorLedgerForParity.suspect - ? vectorLedgerForParity.vectors.all - : metadataStats.totalEntries - if (hnswSize === vectorParityTarget) { + // 1. Index size parity. HNSW must hold at least one node per indexed entity. + if (hnswSize === metadataStats.totalEntries) { checks.push({ name: 'index-parity', status: 'pass', - message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) agree.`, - details: { - hnswSize, - vectoredNouns: vectorParityTarget, - metadataEntries: metadataStats.totalEntries, - graphRelationships: graphSize - } + message: `HNSW (${hnswSize}) and metadata index (${metadataStats.totalEntries}) agree.`, + details: { hnswSize, metadataEntries: metadataStats.totalEntries, graphRelationships: graphSize } }) } else { - const drift = Math.abs(hnswSize - vectorParityTarget) + const drift = Math.abs(hnswSize - metadataStats.totalEntries) checks.push({ name: 'index-parity', - status: drift > Math.max(10, vectorParityTarget * 0.01) ? 'fail' : 'warn', - message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, - details: { - hnswSize, - vectoredNouns: vectorParityTarget, - metadataEntries: metadataStats.totalEntries, - drift - } + status: drift > Math.max(10, metadataStats.totalEntries * 0.01) ? 'fail' : 'warn', + message: `HNSW (${hnswSize}) and metadata (${metadataStats.totalEntries}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, + details: { hnswSize, metadataEntries: metadataStats.totalEntries, drift } }) } @@ -15300,9 +14373,8 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. A CHECK only — it never builds; throws a typed - // NotReady error if a provider's health report says it isn't serving. - this.ensureIndexesLoaded(['graph']) + // provider for 15 minutes. Fast path after the latch is one boolean. + await this.ensureIndexesLoaded() // 8.0 BigInt boundary: unmapped node → no relations. const nodeInt = this.graphEntityInt(nodeId) if (nodeInt === undefined) return [] @@ -15911,13 +14983,16 @@ export class Brainy implements BrainyInterface { // Cold-load guard: an empty connected set is suspicious. The native adjacency can report // size()>0 (or isReady()===false) on a cold open yet have loaded NO source→target edges — so - // traversal would silently return [] as if it were truth. Re-verify against the health-report/ - // isReady() authority (or, for older providers, a READ-ONLY GLOBAL known-edge sample — NOT the - // queried anchor, which may be genuinely edgeless): a dead adjacency throws - // GraphIndexNotReadyError here rather than serving the empty set as fact — verifyGraphAdjacencyLive - // never rebuilds, so a genuinely edgeless anchor simply verifies 'live' and the empty result stands. + // traversal silently returns []. Re-verify against the honest isReady() signal (or, for older + // providers, a GLOBAL known-edge sample — NOT the queried anchor, which may be genuinely + // edgeless). If the adjacency was dead and a rebuild healed it, re-collect; if it stays dead, + // verifyGraphAdjacencyLive() throws GraphIndexNotReadyError. A genuinely edgeless anchor + // verifies 'live' and the empty result stands — no spurious rebuild/throw. if (connectedIds.size === 0) { - await this.verifyGraphAdjacencyLive() + const verdict = await this.verifyGraphAdjacencyLive() + if (verdict === 'rebuilt') { + await populate() + } } // Filter existing results to only connected entities @@ -16643,160 +15718,6 @@ export class Brainy implements BrainyInterface { return embeddingManager.isInitialized() } - /** - * Whether the process-global WASM embedding engine (all-MiniLM-L6-v2, - * fixed 384-dim output, ≈93MB with the bundled model, 90-140s cold compile - * on throttled CPUs) is this instance's active embedder — `false` when a - * plugin has replaced it via the `'embeddings'` provider key. A native - * provider has no such cold-start cost and may use a different output - * dimension, so it is never worth avoiding. - * - * Used by init-path bootstrap writes (the VFS root — see - * `VirtualFileSystem.doInitializeRoot()`) to decide whether embedding a - * value during `init()` risks paying the WASM engine's cold compile. - * - * @returns true when the default WASM engine is active (no native - * `'embeddings'` provider registered). - */ - usesDefaultWasmEmbedder(): boolean { - return !this.pluginRegistry.hasProvider('embeddings') - } - - /** - * @description LEG C of the zero-norm/unvector-door law — migrate a - * legacy zero-norm VFS root BEFORE the vector-leg open gate - * ({@link rebuildIndexesIfNeeded}'s `vectorCoverageGap` check) ever - * compares the canonical vectored-noun count against the vector index's - * size. A pre-fix store may have persisted the VFS root (the fixed - * all-zeros UUID) with a REAL all-zero placeholder vector — lawful inside - * brainy (`cosineDistance` treats a zero-norm operand as MAXIMUM distance, - * see {@link isZeroNormVector}'s JSDoc) but never indexed (the index belt - * refuses to insert a zero-norm vector) and never meant to cross an - * engine boundary. Left unmigrated, the canonical ledger still counts it - * as vectored while the vector index correctly holds nothing for it — a - * near-empty store whose ONLY vectored row is this zero-norm root reads - * "canonical vectored 1, index size 0" and throws - * `VectorIndexNotReadyError` at open, going DARK instead of serving. - * - * THE LIFECYCLE LAW: nothing at open may scale with brain size. This step - * is safe under that law BECAUSE the VFS root lives at a FIXED, - * well-known id (`00000000-0000-0000-0000-000000000000` — mirrors - * `VirtualFileSystem.VFS_ROOT_ID`; kept as a literal here, the same - * convention as the other reserved-root literals in this file and in - * `db/factLog.ts`/`db/portableGraph.ts` — `brainy.ts` cannot import - * `VirtualFileSystem.ts`, which itself imports `Brainy`) — this is ONE - * direct canonical read by id (`storage.getNoun`, the same O(1) - * fixed-path lookup {@link unvectorNounForRootMigration} itself uses - * internally), NEVER a listing or a walk over `entities/nouns/**`. An - * absent root (a store that has never used the VFS) is a no-op, no error. - * - * Runs UNCONDITIONALLY at every open, independent of whether a - * `VirtualFileSystem` is ever constructed this session — the vector-leg - * gate this fixes runs during Brainy's OWN init, before any - * `VirtualFileSystem` instance exists to run its own lazy migration at - * `doInitializeRoot()` (kept in place as the second line of defense for a - * VFS actually opened this session — belt AND suspenders, never either - * alone). - */ - private async migrateLegacyZeroNormVfsRootIfNeeded(): Promise { - const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' - // TORN-TOLERANT: a torn root record is a recovery-walk healer's job - // (see tests/integration/recovery-walk-tolerance.test.ts — an init-time - // walk that meets a torn record narrates+counts, via the adapter's own - // loud floor at the read site, and heals PAST it; the open itself must - // still succeed), not this O(1) migration check's. Skip this open's - // migration attempt rather than aborting init(): this leg is a - // defensive EXTRA (the index belt + VirtualFileSystem's own - // doInitializeRoot() migration still stand as the other lines of - // defense), and it retries harmlessly at a later open once the root - // heals. - let root: HNSWNounWithMetadata | null - try { - root = await this.storage.getNoun(VFS_ROOT_ID) - } catch (err) { - if ((err as { code?: string }).code !== 'TORN_RECORD') throw err - prodLog.warn( - `[Brainy] open(): the VFS root's record is TORN — skipping the zero-norm root ` + - `migration check this open (the recovery walk is the healer; this migration ` + - `retries harmlessly once the root heals).` - ) - return - } - if (!root || !Array.isArray(root.vector) || root.vector.length === 0) return - if (!isZeroNormVector(root.vector)) return - const migrated = await this.unvectorNounForRootMigration(VFS_ROOT_ID) - if (migrated) { - prodLog.warn( - `[Brainy] open(): migrated the VFS root's legacy all-zero placeholder vector to ` + - `the unvectored shape (zero-norm vectors never cross an engine boundary) — run ` + - `before the vector-leg open gate compares canonical-vectored-count against the ` + - `vector index, so a near-empty store never reads a false coverage gap.` - ) - } - } - - /** - * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's - * persisted vector from a real (non-empty) vector to the "unvectored" - * empty-array shape: the vector record is rewritten to `[]`, the row is - * removed from the vector index (if present), and the vectored-noun - * ledger (`getCanonicalCounts().vectors.all`) is decremented through the - * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the - * coverage ledger never silently drifts. - * - * Exists SOLELY for the VFS root zero-norm migration, called from two - * sites that detect the same legacy shape (a persisted root whose vector - * is the legacy all-zero placeholder): {@link migrateLegacyZeroNormVfsRootIfNeeded} - * (this brain's own init sequence, BEFORE the vector-leg open gate — Leg - * C of the zero-norm/unvector-door law) and - * `VirtualFileSystem.doInitializeRoot()` (the second line of defense, for - * a VFS actually constructed this session). This is NOT the general- - * purpose unvector API — ordinary application data uses the sanctioned - * unvector DOOR instead (`update({ id, vector: [] })` / the same op inside - * `transact()`), which decrements the ledger and clears any pending - * deferred-embed marker inline; it does not call this method. Never call - * this outside a VFS root migration. - * - * Idempotent: a noun already unvectored (`vector.length === 0`) or absent - * is a no-op — safe to call on every `init()`. - * - * @param id - The canonical noun id to migrate. - * @returns `true` if a migration write happened, `false` if the noun was - * already unvectored (or absent) — a no-op. - */ - async unvectorNounForRootMigration(id: string): Promise { - const noun = await this.storage.getNoun(id) - if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false - - await this.persistSingleOp({ nouns: [id] }, async (tx) => { - // Rewrite the vector leg to the unvectored shape. Placeholder adjacency - // (mirrors update()'s own SaveNounOperation staging) — the op preserves - // stored graph state when `connections.size === 0`. - tx.addOperation( - new SaveNounOperation(this.storage, { - id, - vector: [], - connections: new Map(), - level: 0 - }) - ) - // Remove from the vector index — safe even if the row was never - // actually indexed (RemoveFromVectorIndexOperation's removeItem is a - // no-op when the id is absent). - tx.addOperation( - new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration) - ) - }) - - // Vectored-noun ledger: this migration carries a vector write with no - // accompanying metadata operation (metadata is untouched), so the - // saveNounMetadata(..., hasVector) seam never fires for it — mirrors the - // deferred-embed LANDING path's use of the narrow storage hook, in - // reverse. - await this.storage.noteVectorUnlanded?.(id) - return true - } - /** * Setup embedder */ @@ -16877,19 +15798,8 @@ export class Brainy implements BrainyInterface { if (legacyEntityPaths.length === 0) { // Already flat (root entities, no head-branch entities) → stamp the marker // so future opens short-circuit. A genuinely empty/fresh dir gets no marker. - // "Are there any entities?" is answered by ONE directory read, not by a - // recursive listing of every file in the tree: this runs on the open path - // of every store that does not yet carry the marker (a restore, a store - // built by an older release), and on a large store that listing walks the - // whole canonical tree to learn a boolean. - const oneLevel = ( - probe as unknown as { listRawPrefixes?: (prefix: string) => Promise } - ).listRawPrefixes - const hasRootEntities = - typeof oneLevel === 'function' - ? (await oneLevel.call(probe, 'entities')).length > 0 - : (await probe.listRawObjects('entities')).length > 0 - if (hasRootEntities) { + const rootEntities = await probe.listRawObjects('entities') + if (rootEntities.length > 0) { await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, @@ -17318,112 +16228,111 @@ export class Brainy implements BrainyInterface { } /** - * @description THE READ GATE. Every read choke point (getNeighborUuids, - * find, filterIdsBelted, getTypedNeighbors) calls this before touching a - * derived index. It is a CHECK, never a build: it asks each of the three - * providers (vector, metadata, graph) for its named health verdict via - * {@link assessProviderHealth} — the provider's own sync, O(1) - * `healthReport()` when exposed, else the `isReady()` / size-heuristic - * fallback — and either lets the read proceed or throws the matching typed - * `*NotReadyError` naming the provider and its failing reasons. It NEVER - * triggers a rebuild and NEVER walks the store: a needed rebuild is - * entirely open()'s job (see {@link rebuildIndexesIfNeeded}), which runs to - * completion before `init()` returns — so by the time any read reaches - * this gate, a healthy provider is already built. A migrating provider is - * deferred to exactly as before (it owns its own in-place rebuild). + * Ensure indexes are loaded (Production-scale lazy loading) * - * A report with something worth telling an operator (a failing invariant, - * whether serving or not, or a named `unledgered` family) narrates via - * `prodLog.warn` ONCE per (provider, `report.generation`) — never once per - * read — before any throw decision is made. + * Called by query methods (find, search, get, etc.) when disableAutoRebuild is true. + * Handles concurrent queries safely - multiple calls wait for same rebuild. + * + * Performance: + * - First query: Triggers rebuild (~50-200ms for 1K-10K entities) + * - Concurrent queries: Wait for same rebuild (no duplicate work) + * - Subsequent queries: Instant (0ms check, indexes already loaded) + * + * Production scale: + * - 1K entities: ~50ms + * - 10K entities: ~200ms + * - 100K entities: ~2s (streaming pagination) + * - 1M+ entities: Uses chunked lazy loading (per-type on demand) */ - /** - * @description Whether two entity `data` payloads are the same content — - * the "no re-embed on unchanged data" comparison. Primitives compare by - * value; objects compare structurally with key order normalized. - * @param a - The incoming data. - * @param b - The stored data. - * @returns `true` when the content is identical. - */ - private static sameEntityData(a: unknown, b: unknown): boolean { - if (a === b) return true - if (a === null || b === null || typeof a !== typeof b) return false - if (typeof a !== 'object') return false - const stable = (v: unknown): string => - JSON.stringify(v, (_k, val) => - val && typeof val === 'object' && !Array.isArray(val) - ? Object.keys(val as Record).sort().reduce((o, k) => { - ;(o as Record)[k] = (val as Record)[k] - return o - }, {} as Record) - : val - ) - try { return stable(a) === stable(b) } catch { return false } - } - - private ensureIndexesLoaded( - families: ReadonlyArray<'vector' | 'metadata' | 'graph'> = ['vector', 'metadata', 'graph'] - ): void { - // PER-FAMILY SCOPE. This gate used to refuse on ANY provider's not-ready - // verdict at every read choke point — so a pure metadata find({where}) - // was refused because the VECTOR leg was not serving; a production - // deployment's badge reads returned 500s for exactly that reason on the - // pair's first adoption. A read may only be refused by the family it - // actually consults: metadata reads by the metadata leg (+ graph for a - // `connected` filter), vector search by the vector leg, traversal by the - // graph leg. Callers name what they need. - const all: ReadonlyArray BrainyError]> = [ - ['vector', this.index, VectorIndexNotReadyError], - ['metadata', this.metadataIndex, MetadataIndexNotReadyError], - ['graph', this.graphIndex, GraphIndexNotReadyError] - ] - const providers = all.filter(([name]) => families.includes(name)) - - for (const [name, provider, ErrorClass] of providers) { - // Migration LOCK (#18) deference: a migrating provider owns its own - // in-place rebuild — brainy must not judge (or race) it here. - if (this.providerIsMigrating(provider)) continue - - const assessment = assessProviderHealth(provider) - - if (assessment.reasons.length > 0 && assessment.report != null) { - const generation = assessment.report.generation - // Dedupe by CONTENT, not by the provider's generation counter — see - // _lastNarratedHealth. The generation is still REPORTED (an operator - // wants to know which generation produced the verdict); it just no - // longer decides whether the line is worth saying. - const line = - `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + - assessment.reasons.join('; ') - const key = `${assessment.report.provider}\u0000${assessment.reasons.join('; ')}` - if (this._lastNarratedHealth.get(provider) !== key) { - this._lastNarratedHealth.set(provider, key) - prodLog.warn(line) - } - } - - if (assessment.readiness === 'not-ready') { - // A provider REBUILDING ITSELF gets a refusal that says so, with its - // own progress: open deliberately did not wait for it, this door is - // temporarily closed, and it opens by itself. Distinct from a broken - // index, which needs an operator. - const rebuilding = assessProviderRebuild(provider) - if (rebuilding) { - throw new ErrorClass( - `${name} index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + - `Reads of this family refuse rather than serve an empty result. The brain is open ` + - `and every other family is serving; this door opens by itself when the provider ` + - `reports serving — no action is needed.` - ) - } - throw new ErrorClass( - `${name} index is not serving (via ${assessment.via}): ` + - `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` + - `empty result — open() builds the derived indexes; a read never does. Rebuild via ` + - `repairIndex({ rebuild: ['${name}'] }) or reopen the brain.` - ) - } + private async ensureIndexesLoaded(): Promise { + // Fast path: If rebuild already completed, return immediately (0ms) + if (this.lazyRebuildCompleted) { + return } + + // If indexes already populated AND honestly serving, mark complete and skip. + // Honest gate: when a provider exposes isReady(), that REPLACES the size()>0 + // proxy (a native index can report a non-zero size while its serving structure + // is not loaded — the silent-empty cold-load class). A not-ready provider falls + // through so the rebuild path can load it; verifyVectorLive() is the query-time + // backstop either way. Providers without isReady() keep the size() heuristic + // (the JS index's size()>0 genuinely means loaded). + // + // ALL THREE providers vote (fleet-adoption find, SELF-ENGINE-PAIR-STANDARD): + // this gate used to assess ONLY the vector index, so a not-ready native + // METADATA provider (its strand report) never blocked the completion latch + // — under disableAutoRebuild the promised lazy first-query rebuild never + // fired and every find() silently returned [] on a populated store. A + // not-ready report from ANY provider now falls through to the rebuild. + const vectorReadiness = assessIndexReadiness(this.index) + const metadataReadiness = assessIndexReadiness(this.metadataIndex) + const graphReadiness = assessIndexReadiness(this.graphIndex) + const anyProviderNotReady = + vectorReadiness === 'not-ready' || + metadataReadiness === 'not-ready' || + graphReadiness === 'not-ready' + if ( + !anyProviderNotReady && + (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) + ) { + this.lazyRebuildCompleted = true + return + } + + // Migration LOCK (#18) deference: while the vector provider runs its one-time + // 7.x → 8.0 rebuild-from-canonical, a first query must NOT trigger brainy's + // force-rebuild — the provider owns that index. Normally unreachable here: the + // data-plane lock (awaitMigrationLock) makes the caller wait upstream, so a + // query only reaches this point once the migration has cleared. Defensive + // (no `lazyRebuildCompleted` latch) so the check re-runs: once the provider + // clears the lock, `index.size() > 0` above ends the lazy path normally. + if (this.providerIsMigrating(this.index)) { + return + } + + // Concurrency control: If rebuild is in progress, wait for it + if (this.lazyRebuildInProgress && this.lazyRebuildPromise) { + await this.lazyRebuildPromise + return + } + + // Check if lazy rebuild is needed + // Only needed if: disableAutoRebuild=true AND indexes are empty AND storage has data + if (!this.config.disableAutoRebuild) { + // Auto-rebuild is enabled, indexes should already be loaded + return + } + + // Check if storage has data (fast check with limit=1) + const entities = await this.storage.getNouns({ pagination: { limit: 1 } }) + const hasData = (entities.totalCount && entities.totalCount > 0) || entities.items.length > 0 + + if (!hasData) { + // Storage is empty, no rebuild needed + this.lazyRebuildCompleted = true + return + } + + // Start lazy rebuild (with mutex to prevent concurrent rebuilds). + // ALWAYS narrated (prodLog, never the silent-suppressible console): a + // read that triggers an index build must be visible to the operator — + // fifteen silent minutes of a production blackout taught this line. + prodLog.warn( + `[Brainy] first read on this instance is building the derived indexes ` + + `(deferred at open by disableAutoRebuild) — reads WAIT and then serve; ` + + `nothing serves empty. Bounded by store size; progress under [MetadataIndex]/[GraphIndex].` + ) + this.lazyRebuildInProgress = true + this.lazyRebuildPromise = this.rebuildIndexesIfNeeded(true) + .then(() => { + this.lazyRebuildCompleted = true + }) + .finally(() => { + this.lazyRebuildInProgress = false + this.lazyRebuildPromise = null + }) + + await this.lazyRebuildPromise } /** @@ -17483,186 +16392,7 @@ export class Brainy implements BrainyInterface { } /** - * @description Consume the JS metadata index's watermark verdict (see - * {@link MetadataIndexManager.watermarkVerdict}) at open — the coordinator - * half of the catchup wiring; {@link MetadataIndexManager.applyWatermarkCatchup} - * is the mechanism half. Feature-detected to the JS manager only: a native - * metadata-index provider consumes the same verdict door in its own train - * (this method never touches the native-provider wrapper contract). - * - * Ordering: called from `performInit()` immediately after - * `metadataIndex.init()` has computed the verdict against the generation - * store's now-FINAL committed generation, and BEFORE `rebuildIndexesIfNeeded()` - * (the open-time rebuild gate) or any read serves — so a caller can never - * observe the pre-catchup state. - * - * @param alreadyRebuilt - `true` when crash recovery just rebuilt every - * index from canonical (rolled-back uncommitted transactions) — the - * verdict's prescribed action is redundant with what already ran (a - * fresh canonical walk supersedes any catchup fold or rescan), so it is - * skipped, narrated, rather than duplicating the work. - */ - private async consumeMetadataWatermarkVerdict(alreadyRebuilt: boolean): Promise { - if (!(this.metadataIndex instanceof MetadataIndexManager)) return - const verdict = this.metadataIndex.watermarkVerdict() - if (verdict === null || verdict === 'adopt') return - - if (alreadyRebuilt) { - prodLog.info( - `[Brainy] metadata index watermark verdict '${verdict}' at open — skipped: crash ` + - `recovery already rebuilt every index from canonical this open.` - ) - return - } - - const window = this.metadataIndex.watermarkGap() - // A genuine first boot (no persisted artifact at all) verdicts 'rescan' - // too — same as a real unverifiable artifact — but it is routine, not - // alarming: narrate it at info level instead of warn (mirrors the - // manager's own internal distinction in loadWatermarkVerdict()). - const firstBoot = verdict === 'rescan' && !this.metadataIndex.watermarkArtifactPresent() - const preNarrate = firstBoot ? prodLog.info.bind(prodLog) : prodLog.warn.bind(prodLog) - preNarrate( - verdict === 'catchup' && window - ? `[Brainy] metadata index watermark verdict: CATCHUP — folding generations ` + - `(${window.from}, ${window.to}] from the fact log before this open serves reads.` - : firstBoot - ? `[Brainy] metadata index watermark verdict: rescan (no persisted artifact — first ` + - `boot; the rebuild below is a trivial no-op walk).` - : `[Brainy] metadata index watermark verdict: RESCAN — the persisted artifact is ` + - `unverifiable (unstamped, or ahead of the store's committed generation); ` + - `forcing a full rebuild from canonical at open.` - ) - - const scan = window - ? this.scanFacts({ fromGeneration: window.from + 1, toGeneration: window.to }) - : null - const result = await this.metadataIndex.applyWatermarkCatchup(scan) - - if (result.action === 'rescan') { - const postNarrate = firstBoot ? prodLog.debug.bind(prodLog) : prodLog.warn.bind(prodLog) - postNarrate( - `[Brainy] metadata index catchup demoted to a full rebuild` + - `${result.reason ? ` — ${result.reason}` : ''}.` - ) - } else if (result.action === 'caught-up') { - prodLog.warn( - `[Brainy] metadata index catchup complete: ${result.factsApplied} fact(s) folded ` + - `(${result.nounsApplied} noun op(s), ${result.verbsApplied} verb op(s)) — index now ` + - `reflects generation ${result.window?.to}.` - ) - } - } - - /** - * @description B3 Deliverable 3 — THE ONLINE METADATA REBUILD. - * `repairIndex()`'s ceremony door for the `'metadata'` family routes here - * instead of calling `MetadataIndexManager.rebuild()` directly: build a - * FRESH replacement manager BESIDE the live one (same storage, same - * idMapper — identity is shared, never a second mapper), walk canonical - * into it while every live write during the build ALSO mirrors there - * (`MetadataIndexManager.beginShadow`), fold the generation window the - * walk may have read stale, then atomically swap this brain's reference — - * `this.metadataIndex` points at the OLD manager for the ENTIRE build, so - * every read in progress (and every read that starts before the swap - * line executes) keeps serving its full, unbuilt-adjacent population; - * nothing ever observes a half-built index. - * - * PERSISTENCE CHOICE (named per the B3 brief): the JS manager's persisted - * keys (field-index chunks, column-store segments, the watermark stamp, - * the id-mapper record) are GLOBAL per storage — not namespaced per - * manager instance — so two managers cannot safely persist independently - * mid-build (a segment-number race, a stamp race, an id-mapper reload - * that would discard the live manager's not-yet-flushed assignments — - * see `MetadataIndexManager.initForShadowBuild`'s JSDoc for the id-mapper - * hazard specifically). This build therefore PERSISTS ONLY AT SWAP: the - * shadow builds entirely in memory (`rebuild({ inMemoryOnly: true })` + - * a fact-log fold — neither touches storage) and flushes exactly once, - * after the swap, as the sole owner of the shared keys. - * - * FALLBACK: a store with no fact log (or a non-JS/native metadata - * provider — its own train owns its online-rebuild strategy) cannot - * safely bound "what landed during the walk"; this method falls back to - * the ORIGINAL blocking clear-then-walk `rebuild()`, narrated. - */ - private async rebuildMetadataIndexOnline(): Promise { - if (!(this.metadataIndex instanceof MetadataIndexManager)) { - // A registered provider (e.g. a native accelerator) may replace - // `this.metadataIndex` with a non-MetadataIndexManager object at - // runtime even though the field's declared type is the JS class — - // the cast mirrors the same reach-in used elsewhere in this file - // (e.g. checkHealth()'s `metadataProvider` locals) for exactly this. - const provider = this.metadataIndex as unknown as MetadataIndexProvider - await provider.rebuild() - return - } - - const committedAtStart = this.storage.committedGeneration?.() ?? null - const factLogAvailable = committedAtStart !== null && this.scanFacts() !== null - if (!factLogAvailable) { - prodLog.warn( - `[Brainy] repairIndex(): metadata rebuild — no fact log on this store, build-beside ` + - `is unavailable; falling back to the blocking rebuild (reads may serve a ` + - `partially-built index for its duration).` - ) - await this.metadataIndex.rebuild() - return - } - - prodLog.warn( - `[Brainy] repairIndex(): metadata rebuild — building a fresh replacement index BESIDE ` + - `the live one (reads keep serving the current index throughout); swapping in ` + - `atomically once it is caught up.` - ) - const startedAt = Date.now() - const oldManager = this.metadataIndex - const shadow = new MetadataIndexManager(this.storage, {}, { - entityIdMapper: oldManager.getIdMapper() - }) - - oldManager.beginShadow(shadow) - let committedAtSwap: number - try { - await shadow.buildBeside(committedAtStart!) - // Capture the true final generation right before the swap — a - // synchronous read, no `await` between here and the reference - // assignment below, so nothing can land ungoverned in the gap: the - // shadow has been live-mirroring every write since beginShadow() - // above, and this generation is the floor a FUTURE open's watermark - // verdict will trust once stamped. - committedAtSwap = this.storage.committedGeneration?.() ?? committedAtStart! - } catch (err) { - oldManager.endShadow() - prodLog.error( - `[Brainy] repairIndex(): online metadata rebuild FAILED during the walk/fold — the ` + - `live index is UNCHANGED (never swapped); reads keep serving the current ` + - `(pre-rebuild) metadata index. Error: ${(err as Error).message}` - ) - throw err - } - - oldManager.endShadow() - this.metadataIndex = shadow - - // NOW persist — the shadow is the SOLE owner of the shared storage keys - // (nothing references `oldManager` any more; it never flushes again). - shadow.stampWatermark(committedAtSwap) - await shadow.flush() - - prodLog.warn( - `[Brainy] repairIndex(): online metadata rebuild complete in ${Date.now() - startedAt}ms — ` + - `swapped in a fresh index reflecting generation ${committedAtSwap}, zero read downtime.` - ) - } - - /** - * @description Rebuild indexes from persisted data if needed — THE OPEN-TIME - * BUILD. Called once per open (init calls it; `repairIndex()`'s - * write-quarantine lift calls it forced). Runs to completion BEFORE `init()` - * returns: a needed rebuild is NEVER deferred to a read (there is no more - * first-query lazy path — see {@link ensureIndexesLoaded}, which is a - * read-time CHECK only). `disableAutoRebuild` no longer defers index - * construction to the first query; see its JSDoc in `brainy.types.ts`. + * Rebuild indexes from persisted data if needed (LAZY LOADING) * * FIXES FOR CRITICAL BUGS: * - Bug #1: GraphAdjacencyIndex rebuild never called ✅ FIXED @@ -17672,24 +16402,34 @@ export class Brainy implements BrainyInterface { * * Production-grade rebuild with: * - Handles BILLIONS of entities via streaming pagination - * - A provider's named {@link HealthReport} (when it exposes one) decides - * per-leg need; `isReady()` / a size heuristic decides otherwise — no - * dataset-size threshold gates whether the rebuild runs at open. + * - Smart threshold-based decisions (auto-rebuild < 1000 items) + * - Lazy loading on first query (when disableAutoRebuild: true) * - Progress reporting for large datasets * - Parallel index rebuilds for performance * - Robust error recovery (continues on partial failures) + * - Concurrency-safe (multiple queries wait for same rebuild) * - * @param force - Force the rebuild path to run even when no leg reports a need (used by `repairIndex()`'s write-quarantine lift). + * @param force - Force rebuild even if disableAutoRebuild is true (for lazy loading) */ private async rebuildIndexesIfNeeded(force = false): Promise { try { + // Check if auto-rebuild is explicitly disabled (ONLY during init, not for lazy loading) + // force=true means this is a lazy rebuild triggered by first query + if (this.config.disableAutoRebuild === true && !force) { + if (!this.config.silent) { + console.log('⚡ Auto-rebuild explicitly disabled via config') + console.log('💡 Indexes will build automatically on first query (lazy loading)') + } + return + } + // No instant fast-path here: the honest per-leg readiness checks below - // are all O(1) (one bounded storage sample + each provider's health - // report / size()/isReady()), and this method runs exactly once per - // open. The removed shortcut keyed off `this.index.size() > 0`, a - // dishonest proxy — it skipped the metadata and graph checks whenever - // the vector happened to be warm, and it never fired on a real cold - // process (the JS vector size is 0 until it loads). + // are all O(1) (one bounded storage sample + each provider's size()/ + // isReady()), and this method runs exactly once per open (init calls it; + // the lazy path passes force=true). The removed shortcut keyed off + // `this.index.size() > 0`, a dishonest proxy — it skipped the metadata + // and graph checks whenever the vector happened to be warm, and it never + // fired on a real cold process (the JS vector size is 0 until it loads). // BUG #2 FIX: Don't trust counts - check actual storage instead // Counts can be lost/corrupted in container restarts @@ -17708,23 +16448,30 @@ export class Brainy implements BrainyInterface { return } + // Intelligent decision: Auto-rebuild based on dataset size + // Production scale: Handles billions via streaming pagination + const AUTO_REBUILD_THRESHOLD = 10000 // Auto-rebuild if < 10K items (increased from 1K) + // Check if indexes need rebuilding const metadataStats = await this.metadataIndex.getStats() const hnswIndexSize = this.index.size() - // Readiness contract: a provider's named {@link HealthReport} (when - // exposed) is the authority — `serving === false` needs the rebuild, - // full stop. Absent a health report, fall back to `isReady()` (an - // mmap/disk-native index legitimately reports 0 resident entries while - // fully durable on disk, so rebuilding it from canonical on every boot - // would be the 48-seconds-per-restart class a production deployment - // hit); absent BOTH, keep the per-leg empty-heuristic passed in. - const legNeedsRebuild = (provider: unknown, emptyFallback: boolean): boolean => { - const assessment = assessProviderHealth(provider) - if (assessment.via === 'health-report') return assessment.readiness !== 'ready' - if (assessment.via === 'is-ready') return assessment.readiness === 'not-ready' - return emptyFallback + // Readiness contract: when a provider exposes isReady(), that honest + // signal REPLACES the size/count heuristic below — an mmap/disk-native + // index legitimately reports 0 resident entries while fully durable on + // disk, and rebuilding it from canonical re-reads every entity file on + // every boot (the 48-seconds-per-restart class a production deployment + // hit). The signal is honest in BOTH directions: a provider whose + // durable state failed to load returns false and gets its rebuild even + // when size() > 0 (the silent-empty cold-load failure). Providers + // without isReady() keep the exact prior empty-heuristics. + const providerReady = (leg: unknown): boolean | undefined => { + const candidate = leg as { isReady?: () => boolean } + return typeof candidate.isReady === 'function' ? candidate.isReady() : undefined } + const metadataReady = providerReady(this.metadataIndex) + const vectorReady = providerReady(this.index) + const graphReady = providerReady(this.graphIndex) // Epoch-drift trigger: a format-version change makes EVERY derived index // suspect even when each is non-empty, so it forces a rebuild of all @@ -17738,43 +16485,15 @@ export class Brainy implements BrainyInterface { // by awaitMigrationLock meanwhile (nothing serves from a half-built index). // Gated per-index, so a non-migrating sibling still rebuilds when it needs // to; a migrating provider is skipped even under epoch-drift or size()===0. - // SELF-REBUILD DEFERENCE (the sibling of the migration lock, and the - // reason a production open took 641 seconds): a provider that reports - // `rebuildInProgress()` is ALREADY rebuilding its own index. Brainy must - // neither start a second rebuild nor WAIT for the provider's — init() - // returns, every other family serves, and that family's own doors refuse - // by name (carrying this progress) until the provider reports serving. - // A provider without the hook behaves exactly as before. - const metadataRebuilding = assessProviderRebuild(this.metadataIndex) - const vectorRebuilding = assessProviderRebuild(this.index) - const graphRebuilding = assessProviderRebuild(this.graphIndex) - for (const [leg, progress] of [ - ['metadata', metadataRebuilding], - ['vector', vectorRebuilding], - ['graph', graphRebuilding] - ] as const) { - if (progress) { - prodLog.narrate( - `[Brainy] open(): the ${leg} provider is ${describeRebuildProgress(progress)} — ` + - `open does NOT wait for it. The brain opens now, every other family serves, and ` + - `${leg} reads refuse by name until the provider reports itself serving.` - ) - } - } - - const metadataMigrating = - this.providerIsMigrating(this.metadataIndex) || metadataRebuilding !== null - const vectorMigrating = this.providerIsMigrating(this.index) || vectorRebuilding !== null - const graphMigrating = this.providerIsMigrating(this.graphIndex) || graphRebuilding !== null - // The epoch stamp certifies EVERY derived index, so it must not advance - // while any family is still being built — by a migration lock or by the - // provider itself. + const metadataMigrating = this.providerIsMigrating(this.metadataIndex) + const vectorMigrating = this.providerIsMigrating(this.index) + const graphMigrating = this.providerIsMigrating(this.graphIndex) const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its - // index (skip) → epoch drift forces a rebuild → the health-report/ - // isReady() authority decides → otherwise a per-leg fallback. The - // fallbacks differ by leg because "empty" means different things: + // index (skip) → epoch drift forces a rebuild → an exposed isReady() + // decides → otherwise a per-leg fallback. The fallbacks differ by leg + // because "empty" means different things: // - METADATA: past the empty-store early-return, entities exist, so the // id-mapper SHOULD have loaded entries — totalEntries===0 is a real // load-failure signal, so rebuild (self-heal from canonical). @@ -17785,132 +16504,62 @@ export class Brainy implements BrainyInterface { // against canonical) inside storage.getGraphIndex() BEFORE this gate, // so it is already authoritative here; re-deriving would be spurious // (a full O(E) verb scan on every open of an edgeless brain). It - // therefore rebuilds only on epoch drift or a native !isReady()/ - // not-serving report. (verifyGraphAdjacencyLive is the query-time - // backstop — it refuses loudly, it never rebuilds.) + // therefore rebuilds only on epoch drift or a native !isReady(). + // (verifyGraphAdjacencyLive is the query-time backstop.) const shouldRebuildMetadata = !metadataMigrating && - (epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0)) - - // VECTOR LEG — the two-engine gate's last red: a migrated 7.x-era store - // can hold canonical vectored nouns with NO derived vector index built. - // `legNeedsRebuild`'s size-heuristic fallback (below) only fires off - // `hnswIndexSize === 0`, and its health-report branch trusts a - // provider's own `serving` verdict verbatim — but a provider's health - // report can legitimately say `serving: true` while vector coverage is - // honestly UNLEDGERED on ITS side too (an unledgered invariant never - // flips serving), so neither signal alone can tell "genuinely empty" - // apart from "never built". The canonical vectored-noun ledger - // (`getCanonicalCounts().vectors.all` — Deliverable 1) is the - // denominator that CAN tell them apart, and is compared here: - // - a CONFIDENT (non-suspect) ledger `> 0` while the reported node - // count is 0 is a proven coverage gap — force the build regardless - // of what a health report claims; - // - a CONFIDENT ledger `=== 0` while the node count is 0 proves there - // is nothing to load (e.g. every noun's embed is still deferred) — - // skip the size-heuristic fallback's blunt "always rebuild when - // empty" trigger, which otherwise wastes a full canonical walk for - // zero benefit on every cold open of such a store; - // - an unavailable/suspect ledger changes nothing — loud errors never - // quiet losses, so a doubtful ledger must never suppress a rebuild - // the old heuristic would have run. - // The bare `isReady()` boolean (no report, no `unledgered` concept) is - // NOT overridden — that signal is what fixed the 48-seconds-per-restart - // regression pinned in tests/unit/cold-open-rebuild-gate.test.ts (a - // disk-native provider legitimately reporting 0 resident while durable - // on disk), and re-deriving it from a denominator the provider itself - // has no way to consult would reopen exactly that regression. - const vectorAssessment = assessProviderHealth(this.index) - const vectorLedger = await this.storage.getCanonicalCounts?.() - const vectorLedgerAll = vectorLedger?.vectors.all - const vectorLedgerConfident = vectorLedger !== undefined && !vectorLedger.suspect - const vectorHasCoverageProof = vectorLedgerConfident && (vectorLedgerAll as number) > 0 - const vectorConfirmedEmpty = vectorLedgerConfident && vectorLedgerAll === 0 - - let vectorNeedsRebuild: boolean - if (vectorAssessment.via === 'is-ready') { - // Bare isReady() stays authoritative and UNMODIFIED — see above. - vectorNeedsRebuild = vectorAssessment.readiness === 'not-ready' - } else if (vectorAssessment.via === 'health-report') { - vectorNeedsRebuild = - vectorAssessment.readiness !== 'ready' || - (hnswIndexSize === 0 && vectorHasCoverageProof) - } else { - // size-heuristic / no provider (the built-in JS engine's own posture). - vectorNeedsRebuild = hnswIndexSize === 0 && !vectorConfirmedEmpty - } - - const shouldRebuildVector = !vectorMigrating && (epochStale || vectorNeedsRebuild) - - // Narration (and the FAIL-TYPED backstop below) are scoped EXACTLY to - // the defect this gate closes: a provider whose OWN health report - // claims `serving: true` — an affirmative "I am ready" a caller would - // otherwise trust outright — while the canonical ledger proves vector - // coverage is missing. This is deliberately NARROWER than "any branch - // where the ledger contributed to the decision": - // - the bare isReady() branch is untouched, as above (never in scope); - // - the health-report branch's OWN `readiness !== 'ready'` case is - // already an ordinary, PRE-EXISTING rebuild trigger (the provider - // admits not-ready) — not a ledger override, so not a "gap"; - // - the size-heuristic/no-provider branch's rebuild-when-empty is the - // SAME blunt trigger the code always had (`hnswIndexSize === 0`) - // — the ledger only ever SUPPRESSES a rebuild there (the confirmed- - // empty case), it never forces one the old heuristic wouldn't - // already have run. Marking that branch a "gap" too made the - // FAIL-TYPED backstop fire on ordinary white-box tests that stub - // rebuild() as a no-op and pin `size()` at 0 to drive OTHER - // assertions (e.g. migration-deference's isMigrating() coverage) — - // those are not silent-empty defects, so they must open exactly as - // before (tests/unit/brainy/migration-deference.test.ts). - const vectorCoverageGap = + (epochStale || + (metadataReady !== undefined ? !metadataReady : metadataStats.totalEntries === 0)) + const shouldRebuildVector = !vectorMigrating && - vectorAssessment.via === 'health-report' && - vectorAssessment.readiness === 'ready' && - hnswIndexSize === 0 && - vectorHasCoverageProof - if (vectorCoverageGap) { - prodLog.warn( - `[Brainy] open(): vector index reports ${hnswIndexSize} node(s) but the canonical ` + - `ledger holds ${vectorLedgerAll} vectored noun(s) — the derived vector index is ` + - `missing or unbuilt on this store. Forcing the vector rebuild rather than serving ` + - `silent-empty search results.` - ) - } - + (epochStale || (vectorReady !== undefined ? !vectorReady : hnswIndexSize === 0)) const shouldRebuildGraph = !graphMigrating && - (epochStale || legNeedsRebuild(this.graphIndex, false)) + (epochStale || (graphReady !== undefined ? !graphReady : false)) const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph if (!needsRebuild && !force) { - // All indexes report current — durably loaded (health-report/isReady/ - // size), or owned by a background migration. No rebuild needed. + // All indexes report current — durably loaded (isReady/size), or owned + // by a background migration. No rebuild needed. return } - // Name exactly which legs rebuild — "all indexes" was a lie whenever - // the durable legs were skipped (e.g. only the JS vector index loads - // here on a warm reopen), and it misread as a whole-brain rebuild in - // consumer boot logs. - const rebuildingLegs = [ - shouldRebuildMetadata && 'metadata', - shouldRebuildVector && 'vector', - shouldRebuildGraph && 'graph' - ] - .filter(Boolean) - .join(' + ') + // Determine rebuild strategy + const isLazyRebuild = force && this.config.disableAutoRebuild === true + const isSmallDataset = totalCount < AUTO_REBUILD_THRESHOLD + const shouldRebuild = isLazyRebuild || isSmallDataset || this.config.disableAutoRebuild === false - // ALWAYS narrated (prodLog, never the silent-suppressible console): there - // is no more first-query lazy path — a rebuild that runs here BLOCKS - // open() regardless of dataset size or `disableAutoRebuild`, so an - // operator must see it in the boot log, not discover it as an - // unexplained slow open. - prodLog.warn( - `[Brainy] open() is building/rebuilding the ${rebuildingLegs || 'no'} index(es) from ` + - `${totalCount.toLocaleString()} stored entities — open blocks until the derived ` + - `indexes serve; reads never build.` - ) + if (!shouldRebuild) { + // Large dataset with auto-rebuild disabled: Wait for lazy loading + if (!this.config.silent) { + console.log(`⚡ Large dataset (${totalCount.toLocaleString()} items) - using lazy loading for optimal startup`) + console.log('💡 Indexes will build automatically on first query') + } + return + } + + // REBUILD: Either small dataset, forced rebuild, or explicit enable + const rebuildReason = isLazyRebuild + ? '🔄 Lazy loading triggered by first query' + : isSmallDataset + ? `🔄 Small dataset (${totalCount.toLocaleString()} items)` + : '🔄 Auto-rebuild explicitly enabled' + + if (!this.config.silent) { + // Name exactly which legs rebuild — "all indexes" was a lie whenever + // the durable legs were skipped (e.g. only the JS vector index loads + // here on a warm reopen), and it misread as a whole-brain rebuild in + // consumer boot logs. + const rebuildingLegs = [ + shouldRebuildMetadata && 'metadata', + shouldRebuildVector && 'vector', + shouldRebuildGraph && 'graph' + ] + .filter(Boolean) + .join(' + ') + console.log(`${rebuildReason} - loading/rebuilding ${rebuildingLegs || 'no'} index(es) from persisted data...`) + } // Before the graph rebuild, hydrate the entity id-mapper from the persisted // snapshot. A native int-keyed adjacency resolves every verb endpoint through @@ -17929,49 +16578,21 @@ export class Brainy implements BrainyInterface { // provider running its own background migration is skipped here (it owns // its index until it verifies-and-swaps). const rebuildStartTime = Date.now() - // The vector leg's build door, by contract with the native provider: a - // provider exposing fillFromCanonical() gets THAT call — idempotent, the - // provider's own init runs it first so this is the backstop — never a - // full rebuild() for a coverage gap. A PARTIAL shortfall deliberately - // triggers nothing here: that is repair()'s operator door. The JS index - // has no fill door and keeps its rebuild. - const vectorBuild = (): Promise => { - const fillDoor = (this.index as unknown as { fillFromCanonical?: () => Promise }) - .fillFromCanonical - if (vectorCoverageGap && typeof fillDoor === 'function') { - prodLog.warn( - `[Brainy] open(): vector coverage gap routes through the provider's ` + - `fillFromCanonical() (idempotent canonical fill), not a full rebuild.` - ) - return fillDoor.call(this.index) - } - return this.index.rebuild() - } await Promise.all([ shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(), - shouldRebuildVector ? vectorBuild() : Promise.resolve(), + shouldRebuildVector ? this.index.rebuild() : Promise.resolve(), shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve() ]) const rebuildDuration = Date.now() - rebuildStartTime const metadataCountAfter = (await this.metadataIndex.getStats()).totalEntries - const graphSizeAfter = await this.graphIndex.size() - - // Completion narration — ALWAYS via prodLog (see the pre-rebuild narration - // above for why): the operator who saw "open() is building…" needs the - // matching "…and it's done" line, with the numbers to confirm it worked. - prodLog.warn( - `[Brainy] open() finished building derived indexes in ${rebuildDuration}ms: ` + - `metadata=${metadataCountAfter} entries, vector=${this.index.size()} nodes, ` + - `graph=${graphSizeAfter} relationships.` - ) if (!this.config.silent) { console.log( `All indexes rebuilt in ${rebuildDuration}ms:\n` + ` - Metadata: ${metadataCountAfter} entries\n` + ` - HNSW Vector: ${this.index.size()} nodes\n` + - ` - Graph Adjacency: ${graphSizeAfter} relationships` + ` - Graph Adjacency: ${await this.graphIndex.size()} relationships` ) } @@ -17980,15 +16601,6 @@ export class Brainy implements BrainyInterface { // when the metadata provider holds the migration lock: a 0 count there // reflects its in-place rebuild in progress, not a missed rebuild, so // forcing a second rebuild would collide with the provider's own. - // THREE states, not two. `metadataMigrating` above is true for a - // provider holding the migration lock AND for one that reports it is - // rebuilding itself — a provider whose rebuild() returns once the - // rebuild is OWNED AND RUNNING (online, its doors refusing by name) - // legitimately reports 0 entries here, and calling that CRITICAL would - // print a false alarm and kick a redundant second rebuild on every - // first contact. The check's real class — a rebuild that ran to - // completion and produced nothing — is untouched: a provider reporting - // 0 entries with NO rebuild in progress still trips it. if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + @@ -17999,24 +16611,6 @@ export class Brainy implements BrainyInterface { console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`) } - // Vector coverage verification: the coverage-gap rebuild above (see - // `vectorCoverageGap`) MUST have actually restored the ledger's - // vectored nouns. A provider that STILL reports 0 nodes after its own - // rebuild() ran — no JS (or provider) fallback could build from what's - // on disk — cannot silently complete open(): search would then serve - // empty results with no signal, exactly the defect this gate closes. - // FAIL TYPED, pre-serve, rather than let a broken vector leg pass as a - // successful open. - if (vectorCoverageGap && this.index.size() === 0) { - throw new VectorIndexNotReadyError( - `open(): the canonical ledger holds ${vectorLedgerAll} vectored noun(s) but the vector ` + - `index still reports 0 node(s) after rebuild() — the derived vector index could not ` + - `be restored from canonical. Refusing to serve silent-empty search results; ` + - `investigate the vector provider/storage, or repairIndex({ rebuild: ['vector'] }) ` + - `after restoring the underlying data.` - ) - } - // 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes // have now rebuilt and verified, so they match this build's epoch — // re-stamp the marker LAST, only here. A crash anywhere above leaves the @@ -18404,6 +16998,49 @@ export class Brainy implements BrainyInterface { return result } + /** + * Run the optional metadata cold-open consistency probe at most once per brain. + * When the active provider exposes `probeConsistency()` (the native cross-bucket + * O(1) sampler), a `false` result triggers `detectAndRepairCorruption()` so an + * already-poisoned index self-heals on first read — the metadata counterpart of + * the 7.33.2 graph cold-load guard. Best-effort: a probe failure never breaks the + * read (the guard is reset so a transient failure retries). No-op for the JS index + * (it exposes no probe), and the full-scan `validateConsistency` stays the explicit + * deep diagnostic via `validateIndexConsistency()`. + */ + private async ensureMetadataConsistencyProbed(): Promise { + if (this._metadataConsistencyProbed) return + // Defer while the metadata provider runs its one-time in-place migration: + // probing (and self-healing via rebuild) an index the provider is mid-rebuild + // would collide with the provider that owns it. Mirrors the vector deference + // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next + // read runs the probe. (The family-scoped find() gate waits on the metadata + // family separately before any actual filter read.) + if (this.providerIsMigrating(this.metadataIndex)) return + this._metadataConsistencyProbed = true + const provider = this.metadataIndex as { + probeConsistency?: () => Promise + detectAndRepairCorruption?: () => Promise + } + if (typeof provider.probeConsistency !== 'function') return + try { + const healthy = await provider.probeConsistency() + if (!healthy && typeof provider.detectAndRepairCorruption === 'function') { + if (!this.config.silent) { + console.warn('[Brainy] metadata index failed the cold-open consistency probe — self-healing via rebuild.') + } + await provider.detectAndRepairCorruption() + } + } catch (error) { + // The self-heal is best-effort and must never break a read. Reset the guard + // so a transient probe failure is retried on the next read. + this._metadataConsistencyProbed = false + if (!this.config.silent) { + console.warn('[Brainy] metadata cold-open consistency probe failed (continuing):', error) + } + } + } + /** * Detect and repair corrupted metadata indexes. * @@ -18529,111 +17166,14 @@ export class Brainy implements BrainyInterface { ) } - /** - * @description The ceremony door for index repair. Bare `repairIndex()` is - * REPORT-DRIVEN, exactly as before: it prunes orphans, recomputes count - * rollups, reconciles VFS containment, and — for the three derived-index - * providers — consults each one's `validateInvariants()` and rebuilds only - * a family whose failing invariant asks for it (`heal: 'rebuild'`). - * - * `options.rebuild` is the EXPLICIT operator override: name one or more - * families (or `'all'`) to rebuild them UNCONDITIONALLY — no invariant is - * consulted, JS or native provider alike. Use it when an operator has - * independent reason to believe a family needs reconciling regardless of - * what its own self-report says (a report can only be as honest as the - * provider that produced it). A family named here is recorded as its own - * `provider:` row with `rebuilt: true` and - * `reason: 'explicit rebuild requested'`, and is SKIPPED by the normal - * invariant-driven pass (it was already rebuilt unconditionally — a second, - * report-driven pass over the same family would be redundant at best). - * - * NARRATION IS PART OF THE CONTRACT. A repair on a production store ran for - * more than thirty minutes at a full core with NOT ONE log line between its - * start and its end while the doors kept serving; the operator could tell it - * was alive only from `top`. Every phase now announces itself before it - * works, a heartbeat names the phase still running every five seconds, and - * each phase reports its own wall — carried in the receipt as - * `durationMs` per family, so nobody has to infer progress from CPU. - * - * @param options.rebuild - Family name(s) to unconditionally rebuild, or `'all'` for all three (`'metadata' | 'graph' | 'vector'`). - * @returns The full per-family receipt (see {@link RepairReport}); also narrated as it goes. - */ - async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { + async repairIndex(): Promise { await this.ensureInitialized() - // A repair recounts, prunes and rebuilds outside the commit paths; the - // dirty witness is set so a caller's flush after a repair does its normal - // work rather than finding the brain "clean". - this._dirtySinceLastFlush = true const startedAt = Date.now() const families: RepairFamilyReport[] = [] - - // THE REPAIR HEARTBEAT — the same law the open obeys: no stretch of work - // may be silent for more than REPAIR_HEARTBEAT_MS. Unref'd (it never holds - // a process open) and cleared in the `finally` below. - const REPAIR_HEARTBEAT_MS = 5_000 - let currentPhase = 'starting' - let currentPhaseCause = 'preparing the repair' - let phaseStartedAt = Date.now() - const heartbeat = setInterval(() => { - prodLog.narrate( - `[Brainy] repairIndex: still in "${currentPhase}" after ` + - `${Math.round((Date.now() - phaseStartedAt) / 1000)}s ` + - `(${Math.round((Date.now() - startedAt) / 1000)}s into the repair) — ${currentPhaseCause}` - ) - }, REPAIR_HEARTBEAT_MS) - if (typeof heartbeat.unref === 'function') heartbeat.unref() - - /** Announce a phase before it does any work, and start its clock. */ - const beginPhase = (name: string, cause: string): void => { - currentPhase = name - currentPhaseCause = cause - phaseStartedAt = Date.now() - prodLog.narrate(`[Brainy] repairIndex: "${name}" started — ${cause}`) + const record = (family: string, entry: Omit): void => { + families.push({ family, ...entry }) } - /** - * Close the current phase: stamp its wall into the receipt row and say - * what it did. Every family row carries its own `durationMs`. - */ - const record = (family: string, entry: Omit): void => { - const durationMs = Date.now() - phaseStartedAt - families.push({ family, ...entry, durationMs }) - prodLog.narrate( - `[Brainy] repairIndex: "${family}" finished in ${durationMs}ms — ` + - (entry.checked - ? `${entry.healed} heal(s)${entry.rebuilt ? ', rebuilt' : ''}` + - (entry.detail ? ` (${entry.detail})` : '') - : `skipped (${entry.skipped ?? entry.reason ?? 'no reason given'})`) - ) - phaseStartedAt = Date.now() - } - - try { - return await this.runRepairIndexPhases(options, families, record, beginPhase, startedAt) - } finally { - clearInterval(heartbeat) - } - } - - /** - * @description The phases of {@link repairIndex}, separated so its heartbeat - * can live in a `finally` around them. Not a public door — see `repairIndex` - * for the contract. - * @param options - As `repairIndex`. - * @param families - The receipt rows being accumulated. - * @param record - Closes a phase: stamps its wall and narrates its outcome. - * @param beginPhase - Announces a phase before it works. - * @param startedAt - When the repair began, for the closing line. - * @returns The full receipt. - */ - private async runRepairIndexPhases( - options: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' } | undefined, - families: RepairFamilyReport[], - record: (family: string, entry: Omit) => void, - beginPhase: (name: string, cause: string) => void, - startedAt: number - ): Promise { - // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete // defect: a delete that removed the metadata (content) leg but left the // vector leg + the entity directory (a "ghost"), or left an empty directory @@ -18647,10 +17187,6 @@ export class Brainy implements BrainyInterface { rebuildSubtypeCounts?: () => Promise } if (typeof pruner.pruneOrphanedEntities === 'function') { - beginPhase( - 'orphaned-containers', - 'walking every canonical id directory for ghost/scar containers left by a partial delete' - ) const orphans = await pruner.pruneOrphanedEntities() const pruned = orphans.nouns.length + orphans.verbs.length record('orphaned-containers', { @@ -18661,7 +17197,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (pruned > 0) { - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + `partial delete.` @@ -18677,10 +17213,6 @@ export class Brainy implements BrainyInterface { // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup // (scalar totals + per-type maps + type-statistics arrays) from one // canonical walk and persists them. - beginPhase( - 'count-rollups', - 'ONE canonical walk recomputing every counter rollup — scalar totals, per-type maps, type statistics' - ) await pruner.rebuildTypeCounts?.() await pruner.rebuildSubtypeCounts?.() record('count-rollups', { @@ -18703,10 +17235,6 @@ export class Brainy implements BrainyInterface { // concurrent writers. Canonical metadata.path is the truth; only VFS // containment edges are touched. Loud per repair. if (this._vfsInitialized && this._vfs) { - beginPhase( - 'vfs-containment', - 'reconciling VFS containment edges against canonical metadata.path' - ) const containment = await this._vfs.repairContainment() record('vfs-containment', { checked: true, @@ -18716,7 +17244,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (containment.removed + containment.restored > 0) { - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` ) @@ -18727,98 +17255,42 @@ export class Brainy implements BrainyInterface { record('vfs-containment', { checked: false, healed: 0, skipped: 'VFS not initialized' }) } - beginPhase( - 'metadata-corruption', - 'detect-and-repair pass over the metadata index' - ) await this.metadataIndex.detectAndRepairCorruption() record('metadata-corruption', { checked: true, healed: 0, detail: 'detect-and-repair pass ran (see its own narration for repairs)' }) // Lift a failed-rollback write-quarantine: force a full rebuild so the // derived indexes are provably reconciled with canonical, then clear the // flag so writes resume. if (this.storeInconsistency) { - beginPhase( - 'write-quarantine', - 'full derived-index rebuild to lift the quarantine set by a failed transaction rollback' - ) await this.rebuildIndexesIfNeeded(true) const cleared = this.storeInconsistency record('write-quarantine', { checked: true, healed: 1, detail: `lifted (${cleared.records.length} record(s) reconciled)` }) this.storeInconsistency = null - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` + `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` + `Writes are re-enabled.` ) } - // THE CEREMONY DOOR: an explicit `options.rebuild` names a family (or - // 'all') to rebuild UNCONDITIONALLY — no invariant consulted. Resolved - // here so the loop below can skip a family's normal report-driven pass - // once its unconditional rebuild has already run. - const explicitRebuildFamilies: ReadonlySet<'metadata' | 'vector' | 'graph'> = - options?.rebuild === 'all' - ? new Set<'metadata' | 'vector' | 'graph'>(['metadata', 'vector', 'graph']) - : new Set(options?.rebuild ?? []) - // Cross-layer repair: repairIndex must reconcile NATIVE derived // state from canonical, not just the JS metadata index. Consult each provider's // own validateInvariants() and rebuild any whose failing invariant asks for it // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption(). - const providerFamilies: ReadonlyArray = [ - ['metadata', this.metadataIndex], - ['vector', this.index], - ['graph', this.graphIndex] - ] - for (const [familyName, provider] of providerFamilies) { - if (explicitRebuildFamilies.has(familyName)) { - const p = provider as { rebuild?: () => Promise } | null - if (!p || typeof p.rebuild !== 'function') { - record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no rebuild() contract' }) - continue - } - beginPhase( - `provider:${familyName}`, - `explicit rebuild requested — rebuilding '${familyName}' unconditionally, no invariant consulted` - ) - // The metadata family routes through the online build-beside - // orchestrator (B3 D3) instead of the provider's own rebuild() — - // zero read downtime when a fact log is available, narrated - // fallback to the blocking rebuild() otherwise. - if (familyName === 'metadata') { - await this.rebuildMetadataIndexOnline() - } else { - await p.rebuild() - } - record(`provider:${familyName}`, { - checked: true, - healed: 1, - rebuilt: true, - reason: 'explicit rebuild requested' - }) - prodLog.narrate(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) - continue - } - + for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { const p = provider as { validateInvariants?: () => Promise rebuild?: () => Promise } | null if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') { - beginPhase(`provider:${familyName}`, 'checking the provider contract') - record(`provider:${familyName}`, { + record(`provider:${(provider as { constructor?: { name?: string } })?.constructor?.name ?? 'unknown'}`, { checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract' }) continue } - beginPhase( - `provider:${familyName}`, - `reading the '${familyName}' provider's own invariant report, then healing only what it asks for` - ) let report: ProviderInvariantReport try { report = await p.validateInvariants() } catch (err) { - record(`provider:${familyName}`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` }) + record(`provider:unknown`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` }) continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here } if (report.healthy) { @@ -18830,61 +17302,15 @@ export class Brainy implements BrainyInterface { checked: true, healed: 1, detail: `rebuilt from canonical (failing: ${report.invariants.filter((i) => !i.holds).map((i) => i.name).join(', ')})` }) - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + `requiring a rebuild — reconciling its derived state from canonical.` ) - // See the explicit-rebuild branch above: 'metadata' routes through - // the online build-beside orchestrator (B3 D3). - if (familyName === 'metadata') { - await this.rebuildMetadataIndexOnline() - } else { - await p.rebuild() - } - } else if ( - report.invariants.some((i) => !i.holds && i.heal === 'repair') && - typeof (provider as { repair?: () => Promise }).repair === 'function' - ) { - // INCREMENTAL HEAL ROUTING (ADR-008 D4): a failing verdict whose heal - // is 'repair' routes to the provider's own repair() — O(missing), - // re-posting exactly what its ledger names, never a store-sized - // rebuild. The return shape is the provider's own; the RE-READ of the - // report is what decides success (the acceptance meta-pin's law: run - // the named heal once, re-read, nothing may still fail the same way). - const failingRepairs = report.invariants - .filter((i) => !i.holds && i.heal === 'repair') - .map((i) => i.name) - prodLog.narrate( - `[Brainy] repairIndex(): provider '${report.provider}' asks for an incremental ` + - `repair (${failingRepairs.join(', ')}) — running its own repair().` - ) - await (provider as { repair: () => Promise }).repair() - let cleared = false - let after: ProviderInvariantReport | null = null - try { - after = await p.validateInvariants() - cleared = !after.invariants.some( - (i) => !i.holds && i.heal === 'repair' && failingRepairs.includes(i.name) - ) - } catch { - // The post-heal re-read failing is itself reportable, never a crash. - } - record(`provider:${report.provider}`, { - checked: true, - healed: cleared ? failingRepairs.length : 0, - detail: cleared - ? `incremental repair cleared: ${failingRepairs.join(', ')}` - : `repair() ran but the re-read still fails (${ - after - ? after.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ') - : 're-read threw' - }) — escalate to repairIndex({ rebuild: ['${familyName}'] })`, - reason: cleared ? undefined : 'repair did not converge' - }) + await p.rebuild() } else { record(`provider:${report.provider}`, { checked: true, healed: 0, - detail: `unhealthy without a routable verdict (failing: ${report.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')})` + detail: `unhealthy without a rebuild verdict (failing: ${report.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')})` }) } } @@ -18893,7 +17319,6 @@ export class Brainy implements BrainyInterface { // rebuild failure are now reconciled — clear the queryable degraded state // and re-arm the read-path warning. if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { - beginPhase('degraded-read-state', 'clearing degraded ids and re-arming the read-path warning') this._indexDegradedIds.clear() this._indexRebuildFailed = null this._degradedReadWarned = false @@ -18902,13 +17327,11 @@ export class Brainy implements BrainyInterface { const healedTotal = families.reduce((n, f) => n + f.healed, 0) const report: RepairReport = { families, healedTotal, durationMs: Date.now() - startedAt } - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex complete in ${report.durationMs}ms — ` + `${families.filter((f) => f.checked).length}/${families.length} families checked, ` + `${healedTotal} heal(s): ` + - families - .map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}@${f.durationMs ?? 0}ms`) - .join(', ') + families.map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}`).join(', ') ) return report } @@ -18957,15 +17380,8 @@ export class Brainy implements BrainyInterface { private static isPackageNotInstalledError(error: unknown, pkg: string): boolean { const code = (error as { code?: string })?.code const message = error instanceof Error ? error.message : String(error) - // The package name must TERMINATE where it ends: an unanchored prefix match - // read a missing platform-binary SIBLING package (e.g. "-linux-x64-gnu", - // exactly what a deploy replacing node_modules mid-restart leaves behind) as - // " is not installed" — and a present-but-broken accelerator silently - // degraded to the default JS engines. A production storm was hunted for a - // day because of that swallow. The name must be followed by a quote, - // whitespace, punctuation, or end-of-message — never a longer name's tail. - const escaped = pkg.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const namesPackage = new RegExp("(^|['\"\\s])" + escaped + "(?=$|['\"\\s.,)])").test(message) + const namesPackage = + message.includes(`'${pkg}'`) || message.includes(`"${pkg}"`) || message.includes(` ${pkg}`) const isResolutionFailure = code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' || @@ -19434,105 +17850,12 @@ export class Brainy implements BrainyInterface { } /** - * @description Close and clean up: flush every buffered component, stamp - * the durability markers, release resources, then give up the writer lock. + * Close and cleanup * - * TWO PARTS, AND THE SECOND IS UNCONDITIONAL. Everything that persists data - * runs in {@link closeDurableSteps}; the terminal releases — the flush-request - * watcher, the WRITER LOCK, the VFS timers, and the terminal `closed` flag — - * run whether those steps succeeded or not, in a `finally`. A close that - * threw halfway used to strand the writer lock on disk with this process's - * (soon dead) pid in it, so the next boot of every affected store announced - * `Overwriting stale writer lock … appears dead` after an orderly exit and - * an operator had to decide whether their database had crashed. A closed - * brain holds no lock — there is no failure for which the opposite is the - * safer answer. - * - * The original failure is never swallowed: it is narrated with what it costs - * the next open, then rethrown to the caller. - * @returns Nothing. - * @throws The first failure from the durable close steps, after the - * terminal releases have run. + * Now flushes HNSW dirty nodes before closing + * This ensures deferred persistence mode data is saved */ async close(): Promise { - let closeFailure: unknown = null - try { - await this.closeDurableSteps() - } catch (error) { - closeFailure = error - } - - // ---- TERMINAL RELEASES: always, even after a failure above ---- - - // Stop the cross-process flush-request watcher (no-op if never started). - try { - if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { - this.storage.stopFlushRequestWatcher() - } - } catch (error) { - console.warn('[Brainy] close: stopping the flush-request watcher failed:', error) - } - - // Release the writer lock. Runs after the metadata buffer drain in - // closeDurableSteps() — otherwise a pending write could land after a - // successor writer claimed the lock — and runs even if that drain threw: - // holding a lock from a process that is about to exit locks the store's - // next boot out of a clean verdict. - try { - if (this.storage && typeof this.storage.releaseWriterLock === 'function') { - await this.storage.releaseWriterLock() - } - } catch (error) { - console.warn('[Brainy] close: releasing the writer lock failed:', error) - } - - // Shut down the VFS: stops its background maintenance interval and the - // PathResolver's — both are ref'd timers that would keep the process - // alive after the last brain closes (consumer-reported hang). - try { - if (this._vfs) { - await this._vfs.close() - } - } catch (error) { - console.warn('[Brainy] close: VFS shutdown failed:', error) - } - - this.initialized = false - // close() is terminal: block lazy re-initialization on any subsequent - // operation (ensureInitialized() throws once this is set). Set even when - // the durable steps failed — a half-closed brain must not keep serving. - this.closed = true - - // Drop this instance from the global registry, and when it was the last - // one, deregister the global shutdown hooks — their ref'd signal handles - // would otherwise keep the process alive after every brain is closed. - const instanceIndex = Brainy.instances.indexOf(this) - if (instanceIndex !== -1) { - Brainy.instances.splice(instanceIndex, 1) - } - Brainy.deregisterShutdownHooksIfIdle() - - if (closeFailure !== null) { - console.error( - `[Brainy] close FAILED partway: ` + - `${closeFailure instanceof Error ? closeFailure.message : String(closeFailure)}\n` + - ` This brain is closed and holds no writer lock, but the clean-shutdown ` + - `marker may not have been written — the next open will run crash recovery ` + - `(a generation-log fold) and report its wall.` - ) - throw closeFailure - } - } - - /** - * @description The durable half of {@link close}: flush every component, - * persist the generation counter and its markers, close the components, - * deactivate plugins, drain the metadata write buffer. Separated from - * `close()` so the terminal releases there can run in a `finally` — see that - * method's contract. - * @returns Nothing. - */ - private async closeDurableSteps(): Promise { // Persistence cadence teardown: no background flush may fire after close // begins (close() runs its own final flush). if (this._persistIdleTimer) { @@ -19574,14 +17897,6 @@ export class Brainy implements BrainyInterface { } await this.autoCompactHistory() - // Watermark stamps ride this flush too — see stampProjectionWatermarks(). - // Read-only instances skip it (no writes, no committed-generation drift - // to certify; ensureInitialized()'s guard below never runs for them - // either, so this must not assume a writer's invariants). - if (!this.isReadOnly) { - this.stampProjectionWatermarks() - } - // Phase 1: Flush ALL components in parallel to persist buffered data // This is critical when cor native providers buffer data in Rust memory await Promise.all([ @@ -19683,6 +17998,38 @@ export class Brainy implements BrainyInterface { } } + // Stop the cross-process flush-request watcher (no-op if never started). + if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { + this.storage.stopFlushRequestWatcher() + } + + // Release the writer lock (no-op for readers and for backends that don't + // hold a lock). Must run after the metadata buffer drain — otherwise a + // pending write could land after a successor writer claimed the lock. + if (this.storage && typeof this.storage.releaseWriterLock === 'function') { + await this.storage.releaseWriterLock() + } + + // Shut down the VFS: stops its background maintenance interval and the + // PathResolver's — both are ref'd timers that would keep the process + // alive after the last brain closes (consumer-reported hang). + if (this._vfs) { + await this._vfs.close() + } + + this.initialized = false + // close() is terminal: block lazy re-initialization on any subsequent + // operation (ensureInitialized() throws once this is set). + this.closed = true + + // Drop this instance from the global registry, and when it was the last + // one, deregister the global shutdown hooks — their ref'd signal handles + // would otherwise keep the process alive after every brain is closed. + const instanceIndex = Brainy.instances.indexOf(this) + if (instanceIndex !== -1) { + Brainy.instances.splice(instanceIndex, 1) + } + Brainy.deregisterShutdownHooksIfIdle() } } diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 4b018e94..cf4a29e0 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -801,17 +801,6 @@ export interface DerivedFamilyDeclaration { export interface CanonicalCounts { nouns: { counted: number; all: number } verbs: { counted: number; all: number } - /** - * The count of canonical nouns holding a REAL (non-empty) vector — the - * coverage denominator a vector index's node-count ledger is measured - * against (`nodeCount === vectors.all` is the whole-store coverage - * verdict for the vector leg, the vector-side mirror of `nouns.all` for - * metadata/graph). A deferred-embed noun (`add({ deferEmbedding: true })`) - * counts only once its vector actually LANDS — its canonical record exists - * (counted in `nouns.all`) with an empty vector until then, so it is - * deliberately NOT counted here in the interim. - */ - vectors: { all: number } /** An unprovable delete has left the `all` scalars unverified since the last recount. */ suspect: boolean } @@ -830,68 +819,14 @@ export interface StorageAdapter { * Save noun metadata separately * @param id Noun ID * @param metadata Noun metadata - * @param hasVector - OPTIONAL vectored-noun ledger hint: `true` when this - * write is a FRESH insert (`isNew`) whose vector is a real, non-empty - * array — the caller already knows this for free (the insert's own - * `vector` local), so the increment rides the SAME isNew gate that - * already protects `totalNounCountAll` from double-counting on HNSW - * neighbor-link re-saves (`saveNoun_internal` re-runs on every link - * change; this metadata seam does not). Absent/`false` ⇒ no ledger - * action. A deferred-embed insert passes `false` (its vector lands - * later — see {@link StorageAdapter.noteVectorLanded}). */ - saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise + saveNounMetadata(id: string, metadata: NounMetadata): Promise /** * Delete noun metadata * @param id Noun ID - * @param priorRecord - OPTIONAL already-known metadata (the caller's - * pre-delete read) — see {@link StorageAdapter.deleteNoun}. - * @param hadVector - OPTIONAL vectored-noun ledger hint: `true`/`false` - * when the caller already knows (read as a side effect of ITS OWN delete - * flow — e.g. `remove()`'s pre-read for the vector-index removal — never - * a read added FOR this ledger), `undefined` when genuinely unknown. A - * known `true` decrements the vectored-noun ledger; a known `false` is a - * no-op (it was never counted); `undefined` marks the ledger SUSPECT - * rather than guessing — the delete path must never add a canonical read - * to answer this question. */ - deleteNounMetadata(id: string, priorRecord?: NounMetadata | null, hadVector?: boolean): Promise - - /** - * OPTIONAL narrow ledger hook: record that a canonical noun's vector just - * LANDED for the first time. Exists ONLY for the deferred-embedding - * lifecycle — the landing commit (`system:embed-landing`) carries a vector - * write with no accompanying metadata operation, so the normal - * `saveNounMetadata(..., hasVector)` seam never fires for it. Callers MUST - * call this only when the noun held NO real vector before this write (the - * deferred-embed worker already holds that fact for free, from its own - * pre-embed read — never an added read). A backend without vectored-noun - * tracking is a no-op via this method's absence (feature-detected). - * @param id - The noun whose vector just landed. - */ - noteVectorLanded?(id: string): Promise - - /** - * OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}: - * record that a canonical noun's vector was just REMOVED — rewritten from - * a real (non-empty) vector to the "unvectored" empty-array shape. Exists - * for the ONE sanctioned reverse migration this engine supports: the VFS - * root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and - * `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix - * store's all-zero placeholder root vector to `[]` and must decrement - * `vectors.all` through this hook so the coverage ledger never drifts. - * NOT a general-purpose "I removed a vector" callback — ordinary - * application data has no sanctioned path from vectored back to - * unvectored (`update()` refuses an empty vector as a dimension - * mismatch by design). Callers MUST call this only when the noun held a - * REAL vector immediately before this write (the caller already holds - * that fact for free, from its own pre-write read — never an added read). - * A backend without vectored-noun tracking is a no-op via this method's - * absence (feature-detected). - * @param id - The noun whose vector was just removed. - */ - noteVectorUnlanded?(id: string): Promise + deleteNounMetadata(id: string): Promise /** * Get noun with metadata combined @@ -940,11 +875,8 @@ export interface StorageAdapter { * REQUIRE re-reading the record being removed: when the internal read * returns `null` (replace race, or a ghost left by an earlier version) the * decrement falls back to this record instead of being silently skipped. - * @param hadVector OPTIONAL vectored-noun ledger hint — see - * {@link StorageAdapter.deleteNounMetadata}'s `hadVector` param, which - * this forwards to unchanged. */ - deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise + deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise /** * Save verb - Pure HNSW verb with core fields only diff --git a/src/db/errors.ts b/src/db/errors.ts index 3b4c1af6..e20488f8 100644 --- a/src/db/errors.ts +++ b/src/db/errors.ts @@ -28,7 +28,7 @@ * speculative `with()` overlay; the canonical storage walk only ever answers * "what is live right now." * - * All are exported from the package root (`@soulcraftlabs/brainy`). + * All are exported from the package root (`@soulcraft/brainy`). */ /** diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts index 2f01e935..98342884 100644 --- a/src/db/familyStamp.ts +++ b/src/db/familyStamp.ts @@ -12,11 +12,9 @@ * the verified surface is a small set of rollup invariants (entity/ * relationship counts) plus `sourceGeneration`. * - * `sourceGeneration` is the COMMITTED generation of the source-of-truth log - * this projection reflects — never the allocated counter, which names a - * generation that may never commit (see {@link StampVerdict.torn}) — so - * open-time coherence becomes a COMPARISON (stamp vs committed head), not a - * walk: + * `sourceGeneration` is the generation of the source-of-truth log this + * projection reflects — open-time coherence becomes a COMPARISON (stamp vs + * log head), not a walk: * * - equal + invariants hold → coherent, serve. * - behind → the projection missed the tail (crash between commit and stamp); @@ -26,9 +24,6 @@ * - invariants FAIL at equal generation → genuine incoherence: loud, and the * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * canonical walk) heals it. - * - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log - * tail's. TERMINAL, never a wait — the generation the stamp names does not - * exist to arrive. * * Stamps are JSON on purpose — every incident gets debugged by reading a * stamp in a terminal. @@ -75,12 +70,6 @@ export type StampVerdict = | { state: 'coherent' } | { state: 'absent' } // legacy store — first stamp writes at the next flush | { state: 'behind'; stampSource: number; head: number } - /** - * TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the - * store's committed watermark can no longer show. TERMINAL — there is no - * generation to wait for, so the open demotes (or refuses) and never spins. - */ - | { state: 'torn'; stampSource: number; head: number } | { state: 'incoherent'; failures: string[] } | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence @@ -129,15 +118,12 @@ export function verifyFamilyStamp( ): StampVerdict { if (stamp === null) return { state: 'absent' } if (stamp.sourceGeneration > head) { - // A stamp AHEAD of committed truth witnesses a generation the store can no - // longer show: the stamp's fsync survived a crash that the log tail did - // not. This is the TORN GENERATION-LOG TAIL — its own class, never folded - // in with `incoherent` (a count that drifted at a generation both sides - // agree on), because the two have opposite cures: incoherence is recounted, - // a tear is DEMOTED. It is also terminal by construction — there is no - // generation the open can wait for, because the one the stamp names is - // gone. - return { state: 'torn', stampSource: stamp.sourceGeneration, head } + // A stamp AHEAD of the log claims state that never committed — the + // projection was stamped against truth that a crash rolled back. + return { + state: 'incoherent', + failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] + } } if (stamp.sourceGeneration < head) { return { state: 'behind', stampSource: stamp.sourceGeneration, head } diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 91451281..0c14b60c 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,60 +147,6 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } - /** - * @description True when `meta` declares more generations than it holds - * frames — a segment sealed by a writer that folded across a hole. The - * manifest records `frames` at fold time, so this is an O(1) comparison - * against the declared span and needs no I/O. - */ - private isSparse(meta: SegmentMeta): boolean { - return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames - } - - /** - * @description The generations this tier ACTUALLY holds, as coalesced - * ascending intervals — not what the segments declare. - * - * Dense segments (every one a current writer produces) contribute their - * declared range with no I/O. A SPARSE segment — one sealed before the - * density law was enforced, whose declared range spans generations it has - * no frame for — has its real generation list read from its sidecar and - * contributed instead, with the discrepancy narrated once. - * - * This is what keeps a store that already carries the damage from wedging. - * `open()` seeds `committedRanges` from these intervals, so a hole is never - * re-admitted as a committed generation, and the auto-compaction pass that - * used to fail on every run with "packed history is damaged" simply never - * asks for the missing frame. - * - * @returns Ascending, non-overlapping `[first, last]` intervals. - */ - async actualRanges(): Promise> { - const out: Array<[number, number]> = [] - for (const meta of this.manifest.segments) { - if (!this.isSparse(meta)) { - out.push([meta.firstGeneration, meta.lastGeneration]) - continue - } - const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames - prodLog.warn( - `[GenerationSegments] sealed segment ${meta.file} declares generations ` + - `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + - `frame(s) — ${missing} generation(s) in that span were never folded into it. ` + - `Serving the frames it actually holds; the declared span is not treated as ` + - `committed history. (Written by a pre-density-law writer that folded across a ` + - `gap; the segment itself is intact and no record is lost.)` - ) - const idx = await this.sidecarFor(meta) - for (const [gen] of idx.generations) { - const last = out[out.length - 1] - if (last !== undefined && gen === last[1] + 1) last[1] = gen - else out.push([gen, gen]) - } - } - return out - } - /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -218,38 +164,6 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } - // THE DENSITY LAW, MADE MECHANICAL. - // - // A sealed segment declares a CONTIGUOUS range [firstGeneration, - // lastGeneration] and every reader treats that range as containment: - // `coveringSegment` is an interval test, `hasGeneration` returns true for - // anything inside it, and `open()` seeds committedRanges from it. So a - // segment folded from a SPARSE input silently claims generations it does - // not hold, and the first read of one of those holes throws - // "inside sealed segment ... but has no frame — packed history is damaged". - // - // That is exactly how the damage was produced. `repackHistory` skipped - // generations mid-batch — ones absent from committedRanges, ones still in - // the pending buffer, ones whose tx.json would not read — and handed the - // survivors here, where the range was computed from the first and last of - // them. Worse, the mis-declared range was then merged back into - // committedRanges at the next open, which is what turned a quiet hole into - // a repeating auto-compaction failure on every subsequent run. - // - // Callers now split at discontinuities; this refusal is what keeps any - // future caller from reintroducing the class. A refusal here loses - // nothing — the generations stay in the live tier, readable, and the next - // pass folds them correctly. - for (let i = 1; i < gens.length; i++) { - if (gens[i].generation !== gens[i - 1].generation + 1) { - throw new Error( - `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` + - `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + - `generation(s). A sealed segment declares a dense range, so folding a sparse ` + - `batch would claim generations it does not hold. Split the batch at the gap.` - ) - } - } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -450,37 +364,12 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // Inside the covering range but with no frame. Two very different causes, - // and conflating them is what made this class wedge every maintenance pass - // on the affected stores. - // - // (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than - // the span it declares. That segment was sealed by a writer that - // folded across a hole (the class this file's density law now bars). - // The segment is INTACT and nothing is lost; it simply never held this - // generation. Answering "not packed" is the honest answer, and it lets - // the caller's two-tier read decide what a genuinely absent generation - // means, instead of every compaction pass dying on a repeating throw. - // `actualRanges()` keeps such holes out of committedRanges at open, so - // in a healed store nobody asks this question in the first place. - // - // (2) A DENSE SEGMENT missing a frame it says it has — the manifest and - // the sidecar disagree about a segment that claims to be complete. - // That IS damage, and it stays loud. - if (this.isSparse(meta)) { - prodLog.warn( - `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + - `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + - `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + - `-generation span — it was sealed across a gap and never held this generation. ` + - `Reporting it as unpacked rather than as damage; no record is lost.` - ) - return null - } + // In the covering range but not present: the packed tier is dense by + // construction (fold packs every generation it is handed, including + // record-less ones) — absence inside a sealed range is damage. throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + - `span — the manifest and the sidecar disagree; packed history is damaged` + `range but has no frame — packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index da21dc61..bfb68959 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,35 +96,6 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' -/** - * @description Split an ascending list of fold candidates into maximal - * CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. - * - * A sealed segment declares one dense range `[firstGeneration, - * lastGeneration]`, and every reader treats that range as containment. So a - * batch with a hole in it must never become one segment: it would claim a - * generation it does not hold, and the first read of that hole reports the - * packed history as damaged. One run, one segment — the ranges then describe - * exactly what the segments contain. - * - * @param gens - Fold candidates, strictly ascending by generation. - * @returns One array per contiguous run, in ascending order. Empty in, empty out. - */ -export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { - const runs: FoldGeneration[][] = [] - let run: FoldGeneration[] = [] - for (const g of gens) { - const prev = run[run.length - 1] - if (prev !== undefined && g.generation !== prev.generation + 1) { - runs.push(run) - run = [] - } - run.push(g) - } - if (run.length > 0) runs.push(run) - return runs -} - /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -566,29 +537,12 @@ export class GenerationStore { this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) - // Discover existing generation record directories — BY DIRECTORY NAME. - // This used to call listRawObjects(), which recurses the whole - // `_generations/` tree and returns every file in every generation, to - // extract a set of integers the top-level directory names already spell. - // MEASURED on a real store with an 11 GB generation history: the phase - // this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with - // no fold to blame — this walk is what it was doing. An adapter without - // the one-level door falls back to the recursive listing, unchanged. + // Discover existing generation record directories. + const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) const seenGens = new Set() - const oneLevel = ( - this.storage as { listRawPrefixes?: (prefix: string) => Promise } - ).listRawPrefixes - if (typeof oneLevel === 'function') { - for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) { - const gen = Number(name) - if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen) - } - } else { - const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) - for (const p of recordPaths) { - const gen = parseGenerationFromPath(p) - if (gen !== null) seenGens.add(gen) - } + for (const p of recordPaths) { + const gen = parseGenerationFromPath(p) + if (gen !== null) seenGens.add(gen) } let rolledBack = 0 @@ -698,56 +652,21 @@ export class GenerationStore { : 'WHOLE-LOG fold' : 'above-manifest replay' let replayed = 0 - const foldStartedAt = Date.now() const replayFact = async (fact: CommitFact): Promise => { for (const op of fact.ops) { - let image: { metadata: unknown | null; vector: unknown | null } - if (op.record === null) { - // A genuine tombstone (both legs absent) — the fold removes - // both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw - // exact-restore contract. - image = { metadata: null, vector: null } - } else if ( - op.record.metadata !== null && - (op.record.vector === null || op.record.vector === undefined) - ) { - // PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's - // half): a metadata-only after-image must never DELETE an - // existing vector leg through the fold. `writeNounRaw`/ - // `writeVerbRaw` are exact-restore primitives — a `vector: - // null` there means "delete", which is exactly right for - // `rollBackUncommittedGeneration`'s before-image restore (a - // transaction abort legitimately un-writes a vector the failed - // transaction added). It is NOT right here: this fold replays - // AFTER-IMAGES, and re-applying an already-intact record must - // be byte-safe (this module's own invariant, see the log-authority - // comment above) — silently erasing a landed vector because one - // replayed fact's vector leg came back null is the exact defect - // that left metadata-counted, never-enumerated rows in a - // production store (confirmed root cause: the enumeration walk - // used to key on the vector leg, so a preserved-but-then-deleted - // vector made the row invisible while the ledger still counted - // it by metadata). A genuine "unvector" has its own sanctioned, - // ledger-correct path (`Brainy.unvectorNounForRootMigration`) — - // never this raw primitive, and never the fold. - const current = - op.kind === 'verb' - ? await this.storage.readVerbRaw(op.id) - : await this.storage.readNounRaw(op.id) - image = { metadata: op.record.metadata, vector: current.vector ?? null } - } else { - image = { metadata: op.record.metadata, vector: op.record.vector } - } + const image = + op.record === null + ? { metadata: null, vector: null } + : { metadata: op.record.metadata, vector: op.record.vector } if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image) this.noteCheckpointDirty(op.kind, op.id) } replayed++ if (replayed % 1000 === 0) { - prodLog.narrate( + prodLog.warn( `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + - `in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` + - `do not restart, the fold is finite` + `(at generation ${fact.generation}); do not restart, the fold is finite` ) } if (fact.generation > this.committed) { @@ -762,7 +681,7 @@ export class GenerationStore { } } if (uncleanOpen) { - prodLog.narrate( + prodLog.warn( `[GenerationStore] log-authority recovery: ${foldKind} beginning ` + `(unclean shutdown detected) — streaming replay, bounded memory, ` + `progress every 1000 facts. Do not restart the process; a restart ` + @@ -785,10 +704,9 @@ export class GenerationStore { } await this.storage.writeRawObject(MANIFEST_PATH, manifest) await this.storage.syncRawObjects([MANIFEST_PATH]) - prodLog.narrate( + prodLog.warn( `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` + - `${this.committed}) — an acked write is never lost` + `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost` ) } // A recovery fold re-applied (and the barrier below re-syncs) every @@ -813,15 +731,9 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law - // writer can declare a span wider than the frames it holds; seeding - // committedRanges from the declared span re-admits those holes as - // committed generations, and every later maintenance pass then asks for a - // frame that was never written. `actualRanges()` reads the real - // generation list from the sidecar for exactly those segments (and does - // no I/O for the dense ones, which is all of them on a healthy store). - const packedRanges = (await this.segments.actualRanges()) - .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) + const packedRanges = this.segments + .segments() + .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets — both ascending; @@ -3156,26 +3068,13 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three - // filters above punch holes in it: a generation missing from - // committedRanges never appears, one still in the pending buffer is - // skipped, and one whose tx.json will not read is skipped. A sealed - // segment declares a DENSE range, so folding across such a hole makes - // the segment claim a generation it does not hold; the next open - // merges that mis-declared range into committedRanges, and every - // subsequent auto-compaction pass then asks for the missing frame and - // fails with "packed history is damaged". Fold each contiguous RUN as - // its own segment instead — same bytes, honest ranges. - for (const run of contiguousRuns(foldInput)) { - if (deadline !== undefined && Date.now() >= deadline) break - await segments.fold(run) - segmentsCreated++ - // Segment + manifest durable → the live copies retire. - for (const g of run) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) - } - folded += run.length + await segments.fold(foldInput) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of foldInput) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) } + folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/src/db/types.ts b/src/db/types.ts index 56bdef11..363de086 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -450,21 +450,6 @@ export interface GenerationStorage { deleteRawObject(path: string): Promise /** List raw object paths under a prefix (normalized, `.gz`-stripped). */ listRawObjects(prefix: string): Promise - /** - * OPTIONAL: the IMMEDIATE child directory names under a prefix — one level, - * no recursion, no file paths. - * - * Why it exists: discovering which generations are on disk needs only the - * top-level directory NAMES under `_generations/`, but the only door for it - * was `listRawObjects`, which recurses the whole tree and returns every file - * in every generation. On a store with a long history that is a full walk of - * the entire generation log, paid on EVERY open, to learn a set of integers - * the directory names already spell out. - * - * An adapter without this door keeps working — the caller falls back to the - * recursive listing. - */ - listRawPrefixes?(prefix: string): Promise /** Remove every object under a prefix (and the directory itself on disk). */ removeRawPrefix(prefix: string): Promise /** Durability barrier: fsync the given object paths (no-op in memory). */ diff --git a/src/embeddings/wasm/modelLoader.ts b/src/embeddings/wasm/modelLoader.ts index b39d90ea..45ffc4d3 100644 --- a/src/embeddings/wasm/modelLoader.ts +++ b/src/embeddings/wasm/modelLoader.ts @@ -128,7 +128,7 @@ async function loadBunAssets(): Promise { } // Strategy 2: node_modules path relative to CWD (for installed packages) - const nmPath = './node_modules/@soulcraftlabs/brainy/assets/models/all-MiniLM-L6-v2' + const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2' pathsToTry.push([ `${nmPath}/model.safetensors`, `${nmPath}/tokenizer.json`, @@ -168,9 +168,9 @@ async function loadBunAssets(): Promise { // If all strategies fail, provide helpful error message throw new Error( 'Could not load model assets. For bun --compile, ensure model files are accessible:\n' + - ' Option 1: Keep node_modules/@soulcraftlabs/brainy/assets/ alongside your binary\n' + + ' Option 1: Keep node_modules/@soulcraft/brainy/assets/ alongside your binary\n' + ' Option 2: Copy assets/ folder to your working directory\n' + - ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraftlabs/brainy/assets/**/*"' + ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraft/brainy/assets/**/*"' ) } @@ -190,7 +190,7 @@ async function loadNodeAssets(): Promise { if (!fs.existsSync(assetsDir)) { throw new Error( `Model assets not found: ${assetsDir}\n` + - `Ensure @soulcraftlabs/brainy is installed correctly.` + `Ensure @soulcraft/brainy is installed correctly.` ) } diff --git a/src/errors/notFound.ts b/src/errors/notFound.ts index 628797a6..8eca9b2e 100644 --- a/src/errors/notFound.ts +++ b/src/errors/notFound.ts @@ -14,7 +14,7 @@ * - {@link RelationNotFoundError} — a referenced relationship (verb) does * not exist. * - * Both are exported from the package root (`@soulcraftlabs/brainy`). + * Both are exported from the package root (`@soulcraft/brainy`). */ /** diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index ebd3b90c..d002164e 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1052,17 +1052,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { */ private startAutoFlush(): void { this.flushTimer = setInterval(async () => { - // NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an - // index nobody has written to since the last flush has nothing to - // write, and calling into the trees (and their logging) on a cadence - // over a quiet store is exactly the idle cost this law exists to - // remove. - if ( - !this.lsmTreeVerbsBySource.hasPendingWrites() && - !this.lsmTreeVerbsByTarget.hasPendingWrites() - ) { - return - } await this.flush() }, this.config.flushInterval) // Background maintenance must never keep the host process alive — diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index b4f6052f..e19ec145 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -687,17 +687,6 @@ export class LSMTree { } } - /** - * @description Whether this tree holds anything a flush would write — - * the MemTable is non-empty. Synchronous and O(1), so a background cadence - * can ask before it does anything at all: the engine does no periodic work - * without a cause. - * @returns true when a flush would write; false when it would be a no-op. - */ - hasPendingWrites(): boolean { - return !this.memTable.isEmpty() - } - async close(): Promise { this.stopCompactionTimer() diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 8b9badc1..77e4f84d 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -10,7 +10,7 @@ import { Vector, VectorDocument } from '../coreTypes.js' -import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' +import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' @@ -64,34 +64,6 @@ export class HnswFlushError extends Error { } } -/** - * @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link - * JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0 - * vector is the sanctioned "unvectored" shape for a canonical noun record - * (class-J: a VFS-system row, a deferred embed not yet landed, or any other - * legitimately-vector-less row) — but it is NEVER a legal INDEX insert. The - * index itself has no concept of "unvectored"; deciding that a row is - * unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job - * (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem. - * A length-0 vector reaching this point is a caller bug: silently accepting - * it would pin `this.dimension = 0` on an empty index (poisoning every real - * insert thereafter with a dimension mismatch) or store a vector-less node - * that a distance calculation can never safely compare against. Loud errors, - * never quiet losses — this throws instead of either. - */ -export class EmptyVectorIndexError extends Error { - constructor(public readonly id: string, operation: 'addItem' | 'updateItem') { - super( - `${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` + - `sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` + - `insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` + - `themselves (unvectored = nothing to index, not an error at that layer); reaching ` + - `here with one is a caller bug.` - ) - this.name = 'EmptyVectorIndexError' - } -} - /** * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native @@ -608,15 +580,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { throw new Error('Vector is undefined or null') } - // THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc): - // an empty vector is the sanctioned "unvectored" shape at the canonical - // layer, never a legal index member. Refusing here — loudly, before the - // dimension pin below — means no future fill/rebuild/load path can ever - // poison `this.dimension` to 0 or park a vector-less node in the graph. - if (vector.length === 0) { - throw new EmptyVectorIndexError(id, 'addItem') - } - // Set dimension on first insert if (this.dimension === null) { this.dimension = vector.length @@ -991,13 +954,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return } - // Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an - // in-place relink must never rewrite an already-indexed node down to the - // unvectored shape or poison the pinned dimension. - if (vector.length === 0) { - throw new EmptyVectorIndexError(id, 'updateItem') - } - if (this.dimension === null) { this.dimension = vector.length } else if (vector.length !== this.dimension) { @@ -1599,15 +1555,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } const loaded = await this.storage.getNounVector(noun.id) - // `loaded` is a length-0 array (not null/undefined) for a canonical row - // that is legitimately unvectored — `![]` is FALSE (an empty array is - // truthy), so the bare `!loaded` check below would silently accept it - // as "found" and hand a dimension-0 vector to a distance calculation. - // A node only reaches this lazy-load path because it is a MEMBER of - // the live index (rebuild() now refuses to admit unvectored rows — see - // its JSDoc), so an empty vector here is never legitimate: treat it - // exactly like "not found", loudly. - if (!loaded || loaded.length === 0) { + if (!loaded) { throw new Error(`Vector not found for noun ${noun.id}`) } @@ -1817,56 +1765,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider { totalCount = result.totalCount || result.items.length - // UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical - // noun whose vector leg is `[]` (a VFS-root-style system row, a - // deferred embed not yet landed, or a best-effort fallback for an - // unreadable vector leg) is a normal, enumerable, countable row — it - // is simply not indexed. `storage.getVectorIndexData()` derives its - // {level, connections} answer straight from the noun's OWN record, so - // it returns non-null for every existing noun regardless of whether - // that noun ever actually reached `addItem()` — it cannot be used to - // decide indexability. `nounData.vector.length === 0` is the one - // truthful signal (mirrors the `noun.vector.length > 0` guards in - // {@link getVectorSafe} / {@link getVectorSync}): skip here, counted - // once in a summary line, never per-row spam. - let skippedUnvectored = 0 - // Process all nouns at once for (const nounData of result.items) { try { - if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) { - skippedUnvectored++ - continue - } - // THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm - // vector (a pre-10.4.2 row the canonical write has not yet - // normalized) must never enter the index either, mirroring the - // belt AddToVectorIndexOperation enforces on the live write path. - // Only the canonical vector is authoritative here — persisted - // HNSW graph metadata (level/connections) can outlive an unvector. - if (isZeroNormVector(nounData.vector)) { - prodLog.warn( - `[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` + - `zero-norm (a zero-norm vector is not a vector and never crosses an ` + - `engine boundary)` - ) - continue - } - - // Restore the pinned dimension from the first real vector this - // rebuild loads. `addItem`/`updateItem` only pin `this.dimension` - // on a LIVE insert — a fresh rebuild from storage never goes - // through either, so without this the pin stays `null` across a - // restart. A `null` pin means the very next insert (correct OR - // wrong length) silently BECOMES the new pin instead of being - // checked against the store's real dimension — the wrong-length - // case then fails much later and less clearly, inside a distance - // calculation against an already-loaded node, instead of here, - // immediately, with a named expected-vs-got mismatch. - if (this.dimension === null) { - this.dimension = nounData.vector.length - } - // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) @@ -1914,10 +1815,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider { options.onProgress(loadedCount, totalCount) } - prodLog.info( - `HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` + - (skippedUnvectored > 0 ? ` — ${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '') - ) + prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`) } // Step 5: CRITICAL - Recover entry point if missing) diff --git a/src/index.ts b/src/index.ts index edc21809..765c20b4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -184,7 +184,6 @@ export { // Export version utilities export { getBrainyVersion } from './utils/version.js' -export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js' // Export plugin system export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' @@ -270,10 +269,6 @@ export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.j export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js' -// The named, synchronous, O(1) health-report contract (the read gate's ONLY -// source of truth for "can I serve right now") — see HealthReport's -// derivation laws in plugin.ts. -export type { HealthReport, LedgerInvariantResult, InvariantSource } from './plugin.js' // Optional provider self-report of outstanding background maintenance work // (compaction, deferred writes, etc.) — the payload type for // brain.maintenanceDebt(). See the measure-only-what-you-track contract on @@ -390,10 +385,7 @@ import type { HNSWVerb, HNSWConfig, StorageAdapter, - DerivedFamilyDeclaration, - // The canonical count ledger a storage adapter maintains (counted + ALL-visibility - // scalars per family, the coverage-ledger denominators) — see StorageAdapter.getCanonicalCounts. - CanonicalCounts + DerivedFamilyDeclaration } from './coreTypes.js' // Export vector index implementation (the JS HNSW path) diff --git a/src/integrations/index.ts b/src/integrations/index.ts index 6a6734d9..757a9fe5 100644 --- a/src/integrations/index.ts +++ b/src/integrations/index.ts @@ -9,7 +9,7 @@ * * @example Enable integrations (recommended) * ```typescript - * import { Brainy } from '@soulcraftlabs/brainy' + * import { Brainy } from '@soulcraft/brainy' * * const brain = new Brainy({ integrations: true }) * await brain.init() diff --git a/src/mcp/README.md b/src/mcp/README.md index 092534a1..c69a3b24 100644 --- a/src/mcp/README.md +++ b/src/mcp/README.md @@ -41,7 +41,7 @@ The `BrainyMCPService` has been refactored to separate the core functionality fr ### In Any Environment (Browser, Node.js, Server) ```typescript -import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraftlabs/brainy' +import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy' // Create a Brainy instance const brainyData = new Brainy() @@ -81,7 +81,7 @@ const toolResponse = await toolset.handleRequest({ ### In Browser Environment (Core Functionality Only) ```typescript -import { Brainy, BrainyMCPService } from '@soulcraftlabs/brainy' +import { Brainy, BrainyMCPService } from '@soulcraft/brainy' // Create a Brainy instance const brainyData = new Brainy() diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 4f4339f4..c15447e7 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2025-09-29T10:10:00-07:00 + * Generated: 2026-07-02T21:43:26.976Z * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index 5b10116c..b5f3546b 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-06-29T10:04:19-07:00 + * Generated: 2026-02-09T16:59:48.867Z * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-06-29T10:04:19-07:00", + generatedAt: "2026-02-09T16:59:48.867Z", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/plugin.ts b/src/plugin.ts index b1aef8e0..947c86a5 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -9,7 +9,6 @@ * registered manually via `brain.use()` — there is no implicit detection. */ -import { prodLog } from './utils/logger.js' import type { StorageAdapter, Vector, @@ -22,7 +21,7 @@ import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js' // Re-export the provider contracts that already live closer to their // implementations so a plugin author (Cor) can import the *entire* -// provider surface from one stable entrypoint: `@soulcraftlabs/brainy/plugin`. +// provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`. export type { ColumnStoreProvider } from './indexes/columnStore/types.js' export type { AggregationProvider, @@ -41,7 +40,7 @@ export interface BrainyPlugin { name: string /** - * Optional semver range of `@soulcraftlabs/brainy` this plugin supports + * Optional semver range of `@soulcraft/brainy` this plugin supports * (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is * OUTSIDE the range, `init()` THROWS rather than silently falling back to the * default JS engine. This is the version-coupling guard for the native @@ -172,66 +171,6 @@ export interface ProviderInvariantReport { durationMs: number } -/** - * @description Where a {@link LedgerInvariantResult} verdict came from: - * - `'ledger'` — decided from an exact, durable ledger (a real count, not a sample). - * - `'deep'` — decided by a full/expensive scan (the `validateInvariants()` diagnostic path only). - * - `'unledgered'` — this family has no ledger yet; the verdict is UNKNOWN, never healthy and never broken. - */ -export type InvariantSource = 'ledger' | 'deep' | 'unledgered' - -/** - * @description One invariant verdict inside a {@link HealthReport}. Extends - * {@link InvariantResult} with the provenance of the verdict ({@link InvariantSource}) - * and, for a failing set-membership invariant, an exact count plus a capped sample - * of the diverging ids — a VERDICT, never a dump. `sample` MUST be capped at 16 ids; - * `count` is the exact number even when `sample` is truncated. - */ -export interface LedgerInvariantResult extends InvariantResult { - /** Provenance of this verdict — see {@link InvariantSource}. */ - source: InvariantSource - /** Exact count of diverging/missing items plus a capped (≤16 ids) sample. Present only on a failing set-membership invariant. */ - missing?: { count: number; sample: string[] } -} - -/** - * @description The NAMED, SYNCHRONOUS, O(1) health report a provider exposes via - * {@link MetadataIndexProvider.healthReport} / {@link GraphIndexProvider.healthReport} / - * {@link VectorIndexProvider.healthReport}. This is the read gate's ONLY source of - * truth for "can I serve right now" — it replaces sampled self-probes and the - * unnamed `isReady()` latch with an exact, ledger-derived verdict. - * - * Derivation laws (a provider MUST honor these; brainy's read gate assumes them): - * - `healthy` = every VERIFIED invariant in {@link invariants} holds. An invariant - * whose family is named in {@link unledgered} is NEVER counted toward `healthy` - * either way — it is unknown, not passing. - * - `serving` = no verified invariant in {@link invariants} FAILS with `heal: 'rebuild'`. - * A failure with `heal: 'repair'` or `heal: 'none'` is degraded-but-serving — - * `serving` stays `true`. Only a `'rebuild'`-grade failure makes `serving` `false`. - * - `validateInvariants()` remains the async DEEP diagnostic (full scans allowed, - * `source: 'deep'` results); `healthReport()` MUST be synchronous, O(1) from - * exact ledgers/counters, and MUST NOT throw for a well-formed provider — a - * provider that cannot produce a safe verdict reports it as a failing invariant, - * it does not throw (a throw is read by the gate as a CONTRACT VIOLATION, not as - * "unknown"). - */ -export interface HealthReport extends ProviderInvariantReport { - /** - * Monotonic per provider: bumps on every ledger mutation and every rebuild - * boundary. Consumers (the read gate's narration dedup, external callers) may - * cache a verdict per generation. - */ - generation: number - /** Each checked invariant, with provenance — see {@link LedgerInvariantResult}. */ - invariants: LedgerInvariantResult[] - /** - * Families with no ledger yet. NAMED here so an operator can see what is not - * yet tracked — NEVER counted as healthy (they are not verified) and NEVER - * counted as broken (there is nothing to fail). - */ - unledgered: string[] -} - /** * @description A provider's self-report of its own outstanding background * maintenance work (compaction, deferred writes, a build-new→verify→swap in @@ -327,20 +266,6 @@ export interface MetadataIndexProvider { */ validateInvariants?(): Promise - /** - * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this - * provider derives from its own exact ledgers — see {@link HealthReport} for - * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a - * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never - * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) - * reads THIS instead of `isReady()` / size heuristics: `serving` decides - * whether reads may proceed; a `false` refuses the read loudly rather than - * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the - * size heuristic (this train's JS built-in providers stay on that interim - * path). - */ - healthReport?(): HealthReport - /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -537,20 +462,6 @@ export interface GraphIndexProvider { */ validateInvariants?(): Promise - /** - * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this - * provider derives from its own exact ledgers — see {@link HealthReport} for - * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a - * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never - * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) - * reads THIS instead of `isReady()` / size heuristics: `serving` decides - * whether reads may proceed; a `false` refuses the read loudly rather than - * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the - * size heuristic (this train's JS built-in providers stay on that interim - * path). - */ - healthReport?(): HealthReport - /** * @description OPTIONAL eager cold-load. Called once during brain init — AFTER * the metadata provider's `init()` (so the id-mapper is hydrated; a native int @@ -1314,20 +1225,6 @@ export interface VectorIndexProvider { */ validateInvariants?(): Promise - /** - * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this - * provider derives from its own exact ledgers — see {@link HealthReport} for - * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a - * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never - * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) - * reads THIS instead of `isReady()` / size heuristics: `serving` decides - * whether reads may proceed; a `false` refuses the read loudly rather than - * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the - * size heuristic (this train's JS built-in providers stay on that interim - * path). - */ - healthReport?(): HealthReport - /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -1575,13 +1472,9 @@ export class PluginRegistry { this.activated.add(name) activated.push(name) } else { - // Documented graceful decline (activate() → false). Surface it on the - // ALWAYS-ON channel: `silent: true` patches console, and a declined - // accelerator warned into a patched console is a silent degrade to the - // default engines — the exact invisible-fallback class this registry - // exists to prevent (a production storm ran the WASM engine for 90s - // behind one suppressed warn). - prodLog.warn( + // Documented graceful decline (activate() → false). Surface it loudly so + // a silent degrade to the default engine never goes unnoticed. + console.warn( `[brainy] Plugin "${name}" declined activation (activate() returned false); ` + `the default engine is in use for its providers.` ) diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index cabe2e30..7c080b4c 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1041,43 +1041,16 @@ export abstract class BaseStorageAdapter implements StorageAdapter { */ protected totalNounCountAll = 0 protected totalVerbCountAll = 0 - /** - * The count of canonical nouns holding a REAL (non-empty) vector — the - * vector-side mirror of `totalNounCountAll` and the coverage denominator a - * vector index's node-count ledger is measured against. A deferred-embed - * noun (`add({ deferEmbedding: true })`) counts only once its vector - * LANDS (the `system:embed-landing` commit) — its canonical record exists - * (already counted in `totalNounCountAll`) with an empty vector until - * then. Maintained on the write path (a fresh insert whose vector is - * non-empty +1, a deferred embed's landing +1, a PROVEN delete of a - * vectored noun −1), persisted beside the other ALL scalars, recomputed by - * the sanctioned recount. Shares `allCountsSuspect` — no separate flag. - */ - protected totalVectoredNounCount = 0 /** * `true` when a delete could not prove whether the record existed (no * canonical read, no caller-provided prior) — the ALL scalar may be off by * the unprovable deletes since. Loud, persisted, and cleared only by the * sanctioned recount; a consumer reading the scalar as a ledger denominator - * must treat a suspect scalar as unverified, never as exact. Also covers - * `totalVectoredNounCount` — a delete whose vector-presence fact was - * unknowable marks this SAME flag rather than minting a second one. + * must treat a suspect scalar as unverified, never as exact. */ protected allCountsSuspect = false /** One narration per session for the suspect transition (never per delete). */ private allCountsSuspectNarrated = false - /** - * Which rule produced the ALL scalars currently in memory. `'identity-record'` - * means one counted entity per metadata content leg — the honest rule: a - * bare id-directory (a ghost or scar left by a partial-delete defect, no - * content leg) counts zero. Set by the one-time derivation and by the - * sanctioned recount, alongside `allCountsSuspect = false`; left `undefined` - * when a loaded counts.json carries the ALL scalars but no stamp — the - * legacy container-rule derivation, which forces `allCountsSuspect = true` - * at load instead. A filesystem concern: `MemoryStorage` has no counts.json - * and never sets this. - */ - protected allCountsDerivedBy?: 'identity-record' protected entityCounts: Map = new Map() // type -> count protected verbCounts: Map = new Map() // verb type -> count protected countCache: Map = new Map() @@ -1110,19 +1083,15 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * The canonical count ledger — O(1), no I/O. `counted` is the user-facing * scalar (public/internal tiers, what `getNounCount()` returns); `all` is * the ALL-visibility scalar every unfiltered storage walk is measured - * against (the coverage-ledger denominator for derived-index providers); - * `vectors.all` is the vectored-noun scalar — the coverage denominator for - * a vector index's node-count ledger specifically. - * `suspect` is `true` when an unprovable delete has made `all` (any - * family, including `vectors`) unverified since the last sanctioned - * recount (`rebuildTypeCounts`). - * @returns All scalars per family plus the suspect flag. + * against (the coverage-ledger denominator for derived-index providers). + * `suspect` is `true` when an unprovable delete has made `all` unverified + * since the last sanctioned recount (`rebuildTypeCounts`). + * @returns Both scalars per family plus the suspect flag. */ async getCanonicalCounts(): Promise { return { nouns: { counted: this.totalNounCount, all: this.totalNounCountAll }, verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll }, - vectors: { all: this.totalVectoredNounCount }, suspect: this.allCountsSuspect } } @@ -1134,7 +1103,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * @param family - Which family's delete was unprovable. * @param id - The id whose existence could not be established. */ - protected markAllCountsSuspect(family: 'noun' | 'verb' | 'noun-vector', id: string): void { + protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void { this.allCountsSuspect = true if (!this.allCountsSuspectNarrated) { this.allCountsSuspectNarrated = true @@ -1147,41 +1116,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } } - /** - * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorLanded}): - * record a deferred-embed noun's FIRST real vector landing. The caller - * (the deferred-embed worker) proves this is a genuine landing — not a - * re-embed of an already-vectored row — by observing its own pre-embed - * read's vector was empty, at no added storage cost. - * @param id - The noun whose vector just landed (retained for a future - * narration seam; the count itself needs no id-keyed state). - */ - async noteVectorLanded(id: string): Promise { - void id - this.totalVectoredNounCount++ - this.scheduleCountPersist().catch(() => { - // Ignore persist errors — the in-memory count is authoritative; a later op retries. - }) - } - - /** - * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}): - * the mirror of {@link noteVectorLanded} — record a noun's vector was just - * REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a - * caller that (incorrectly) fires this for a noun already unvectored would - * otherwise drive the ledger negative — clamped defensively, matching the - * delete path's `if (this.totalVectoredNounCount > 0)` guard. - * @param id - The noun whose vector was just removed (retained for a - * future narration seam; the count itself needs no id-keyed state). - */ - async noteVectorUnlanded(id: string): Promise { - void id - if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- - this.scheduleCountPersist().catch(() => { - // Ignore persist errors — the in-memory count is authoritative; a later op retries. - }) - } - /** * Increment count for entity type - O(1) operation. * Concurrency is handled by the process-global mutex diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 5ec1d88e..6d2d9b3c 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -14,13 +14,10 @@ import { StorageBatchConfig, SYSTEM_DIR, STATISTICS_KEY, - WriterLockInfo, - WriterCloseRecord + WriterLockInfo } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' -import { prodLog } from '../../utils/logger.js' -import { isZeroNormVector } from '../../utils/distance.js' import { TornRecordError, isUnparseablePayloadError, @@ -100,30 +97,7 @@ export class FileSystemStorage extends BaseStorage { // timer rewrites the lock every 10s so stale-lock detection can tell a dead // writer from a slow one. The constant name matches the file path used. private static readonly WRITER_LOCK_FILE = '_writer.lock' - /** - * The clean-close record at `locks/_writer.close` (see - * {@link WriterCloseRecord}). Written when the lock is released, consumed by - * the next claim, so an open can distinguish "the previous writer left" from - * "the previous writer died" without inferring either from a pid. - */ - private static readonly WRITER_CLOSE_FILE = '_writer.close' - /** - * How often the lock file's `lastHeartbeat` is rewritten. - * - * THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness - * is decided by PID LIVENESS alone (see isWriterLockStale) and the fence - * compares pid + hostname — no decision anywhere reads this timestamp. It - * exists so an operator inspecting a lock file, or reading the - * BRAINY_WRITER_LOCKED error, can judge liveness themselves. - * - * At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1 - * writes/s across a production process holding 21 idle brains, for a - * human-readable timestamp nothing computes with. At 60s an operator still - * sees a heartbeat inside the minute, at a sixth of the cost. With the - * clean-close record now recording orderly releases explicitly, the - * heartbeat carries even less weight than it did. - */ - private static readonly WRITER_HEARTBEAT_MS = 60_000 + private static readonly WRITER_HEARTBEAT_MS = 10_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo @@ -136,13 +110,6 @@ export class FileSystemStorage extends BaseStorage { */ private writerHeartbeatInFlight?: Promise - /** - * The in-flight background count-ledger derivation, if one was needed at - * open. See {@link scheduleCountLedgerDerivation} — awaited only by - * {@link whenCountLedgerSettled}, never by a read. - */ - private countLedgerDerivation?: Promise - // Flush-request RPC state. The writer polls `locks/_flush_requests/` for // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after // flushing. Inspectors call `requestFlushOverFilesystem` to drop a request @@ -151,16 +118,9 @@ export class FileSystemStorage extends BaseStorage { private static readonly FLUSH_REQUEST_DIR = '_flush_requests' private static readonly FLUSH_RESPONSE_DIR = '_flush_responses' private static readonly FLUSH_WATCH_INTERVAL_MS = 500 - /** - * The safety sweep behind the fs.watch: catches events an exotic filesystem - * dropped, and runs the stale-request GC. See startFlushRequestWatcher. - */ - private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000 private static readonly FLUSH_POLL_INTERVAL_MS = 100 private static readonly FLUSH_REQUEST_TTL_MS = 60_000 private flushWatcherInterval?: NodeJS.Timeout - /** The inotify-backed watch on the request directory, when the FS supports one. */ - private flushWatcher?: import('node:fs').FSWatcher private flushWatcherInFlight = false private flushWatcherOnRequest?: () => Promise @@ -279,54 +239,38 @@ export class FileSystemStorage extends BaseStorage { // Finish any restore interrupted by a crash (resume the staged swap, or // discard an uncommitted staging area) BEFORE counts/derived state load, - // so the rest of startup sees the completed store. ORDER-DEPENDENT: - // `swapStagedRestoreIn()` reads `fs.readdir(rootDir)` and then - // removes/renames rootDir's own TOP-LEVEL entries to place the staged - // copy — racing that against the directory-creation batch below (which - // also touches rootDir's children) could see a half-created directory - // mid-swap or a mkdir racing a concurrent rm/rename on the same path. - // Stays strictly sequential, never folded into the OPEN-PATH batch. + // so the rest of startup sees the completed store. await this.completeInterruptedRestore() - // OPEN-PATH FIX: the remaining bootstrap directories are mutually - // independent — each is its own subtree under rootDir, and - // `fs.mkdir(dir, { recursive: true })` creates every intermediate - // segment of ITS OWN path in one call, so it never depends on any - // sibling here existing first. Nothing between here and - // `initializeCounts()` reads any of them, so batching collapses what - // was up to 8 sequential mkdir round-trips (each a real syscall+await) - // into one wave — this is what serialized an N-writer restart storm on - // filesystem I/O it never structurally needed. `initializeCounts()` - // right after DOES depend on `systemDir` (which the batch creates), so - // it stays outside, awaited only once every directory has landed. - await Promise.all([ - // Create the nouns directory if it doesn't exist - this.ensureDirectoryExists(this.nounsDir), - // Create the verbs directory if it doesn't exist - this.ensureDirectoryExists(this.verbsDir), - // Create the metadata directory if it doesn't exist - this.ensureDirectoryExists(this.metadataDir), - // Create the noun metadata directory if it doesn't exist - this.ensureDirectoryExists(this.nounMetadataDir), - // Create the verb metadata directory if it doesn't exist - this.ensureDirectoryExists(this.verbMetadataDir), - // Create both directories for backward compatibility - this.ensureDirectoryExists(this.systemDir), - // Only create legacy directory if it exists (don't create new legacy - // dirs) — a read-then-maybe-write, but on its own subtree, so it's - // still independent of every other entry in this batch. - (async () => { - if (await this.directoryExists(this.indexDir)) { - await this.ensureDirectoryExists(this.indexDir) - } - })(), - // Create the locks directory if it doesn't exist - this.ensureDirectoryExists(this.lockDir), - // Create the binary blobs directory if it doesn't exist - this.ensureDirectoryExists(this.blobsDir) - ]) + // Create the nouns directory if it doesn't exist + await this.ensureDirectoryExists(this.nounsDir) - // Initialize count management — depends on systemDir, created above. + // Create the verbs directory if it doesn't exist + await this.ensureDirectoryExists(this.verbsDir) + + // Create the metadata directory if it doesn't exist + await this.ensureDirectoryExists(this.metadataDir) + + // Create the noun metadata directory if it doesn't exist + await this.ensureDirectoryExists(this.nounMetadataDir) + + // Create the verb metadata directory if it doesn't exist + await this.ensureDirectoryExists(this.verbMetadataDir) + + // Create both directories for backward compatibility + await this.ensureDirectoryExists(this.systemDir) + // Only create legacy directory if it exists (don't create new legacy dirs) + if (await this.directoryExists(this.indexDir)) { + await this.ensureDirectoryExists(this.indexDir) + } + + // Create the locks directory if it doesn't exist + await this.ensureDirectoryExists(this.lockDir) + + // Create the binary blobs directory if it doesn't exist + await this.ensureDirectoryExists(this.blobsDir) + + // Initialize count management this.countsFilePath = path.join(this.systemDir, 'counts.json') await this.initializeCounts() @@ -642,20 +586,6 @@ export class FileSystemStorage extends BaseStorage { * automatically. Returns the pruned container ids so the caller can recompute * counts. */ - /** - * @description Whether an id directory's file legs include the metadata - * CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single - * test that decides whether an `entities////` container is - * a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete - * defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune - * and {@link scanCanonicalEntities} so the two agree by construction — one - * counted entity per identity record, never per bare container. - * @param legs - File names in one `entities////` directory. - */ - private hasMetadataContentLeg(legs: string[]): boolean { - return legs.some((f) => f.startsWith('metadata.json')) - } - public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { await this.ensureInitialized() const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } @@ -695,7 +625,7 @@ export class FileSystemStorage extends BaseStorage { } // A live entity has its metadata content leg. No content leg → a // vector-only ghost or an empty scar → prune the whole container. - if (this.hasMetadataContentLeg(legs)) continue + if (legs.some((f) => f.startsWith('metadata.json'))) continue await fs.promises.rm(idAbs, { recursive: true, force: true }) pruned[kind].push(entry.name) console.warn( @@ -709,30 +639,6 @@ export class FileSystemStorage extends BaseStorage { return pruned } - /** - * @description The IMMEDIATE child directory names under a prefix — ONE - * `readdir`, no recursion, no file paths. See the seam's JSDoc - * (`src/db/types.ts`) for what this replaced: discovering the generations on - * disk walked the entire generation log on every open, reading out every - * file in every generation, to learn the set of integers the top-level - * directory names already spell. - * @param prefix - Storage-root-relative directory prefix. - * @returns The child directory names (not paths); empty when the prefix does - * not exist. - */ - public override async listRawPrefixes(prefix: string): Promise { - await this.ensureInitialized() - const fullPath = path.join(this.rootDir, prefix) - try { - const entries = await fs.promises.readdir(fullPath, { withFileTypes: true }) - return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory()) - .map((e: { name: string }) => e.name) - } catch (error: any) { - if (error?.code === 'ENOENT') return [] - throw error - } - } - /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing @@ -1943,41 +1849,18 @@ export class FileSystemStorage extends BaseStorage { } } - // THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see - // WriterCloseRecord). A lock file whose release was RECORDED is - // bookkeeping left by an orderly shutdown, not evidence of anything — - // and that is true whether the previous holder was another process or - // an earlier instance in THIS one. A production restart reported - // "Re-acquiring writer lock ... this is a bug" immediately after a clean - // close, sending an operator hunting for a leak that did not exist. - const closeRecord = existing ? await this.readWriterCloseRecord() : null - const releasedCleanly = - existing !== null && - closeRecord !== null && - this.closeRecordVouchesFor(closeRecord, existing) - if (existing) { // Same-process re-open: a second Brainy instance in this Node process // (e.g. test "simulate server restart" patterns, or a consumer that // explicitly re-instantiates without closing first). This isn't the // dangerous cross-process case the lock exists to prevent — the two // instances share a memory space and can't silently diverge from each - // other beyond what their callers already see. Warn and take over — - // unless the record proves the previous instance already let go, in - // which case there is nothing to warn about. + // other beyond what their callers already see. Warn and take over. if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { - if (releasedCleanly) { - console.warn( - `[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` + - `instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` + - `${closeRecord!.closedAt} but could not remove the file. Nothing to recover.` - ) - } else { - console.warn( - `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + - `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` - ) - } + console.warn( + `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` + ) const info: WriterLockInfo = { pid: myPid, hostname, @@ -1987,18 +1870,11 @@ export class FileSystemStorage extends BaseStorage { rootDir: this.rootDir } await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) - await this.clearWriterCloseRecord() this.installWriterLock(info) return info } - // A cleanly-released lock is stale by RECORD, not by inference. Only - // when no record vouches for this lock do we fall back to pid - // liveness, and then we say THAT honestly too: an unrecorded lock - // means the writer did not complete its close, so the store was not - // closed cleanly and this open pays recovery. - const stale = - releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing))) + const stale = !options?.force && (await this.isWriterLockStale(existing)) if (!options?.force && !stale) { // Consumer-facing error contract: callers detect this case via // err.code and read the holder's details from err.lockInfo. @@ -2009,16 +1885,8 @@ export class FileSystemStorage extends BaseStorage { options?.force ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + `(was held by PID ${existing.pid} on ${existing.hostname}).` - : releasedCleanly - ? `[brainy] Clearing the leftover writer lock for ${this.rootDir} — ` + - `PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` + - `${closeRecord!.closedAt} but could not remove the file. ` + - `Nothing to recover.` - : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + - `(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` + - `clean-close record — that writer did not finish closing, so this ` + - `store was not closed cleanly; open will run crash recovery and ` + - `report its wall).` + : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + + `(PID ${existing.pid} on ${existing.hostname} appears dead).` ) // Takeover: verify the file still holds the lock we judged (a live // successor may have claimed meanwhile), then remove it and fall @@ -2072,12 +1940,6 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.unlink(claimTmp).catch(() => {}) } - // CONSUME the previous writer's clean-close record. It described the - // lock generation that just ended; leaving it in place would let it - // vouch for OUR lock if this process later dies without closing — - // turning a real crash into a "closed cleanly" verdict. One unlink. - await this.clearWriterCloseRecord() - this.installWriterLock(info) return info } @@ -2201,27 +2063,13 @@ export class FileSystemStorage extends BaseStorage { return } const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) - const released = this.writerLockInfo try { // Only delete if we still own it — avoid clobbering a successor that // claimed the lock via force-override. const current = await this.readWriterLock() - const ours = - current === null || - (current.pid === released.pid && current.hostname === released.hostname) - if (current && ours) { + if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) { await fs.promises.unlink(lockFile) } - // THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this - // instance gives up a lock nobody else has taken — the unlink above - // having succeeded OR the file already being gone. The next open reads - // it instead of guessing from pid liveness: a recorded release is an - // orderly shutdown, an absent record is a writer that never finished - // closing. Not written when a successor holds the lock: our release is - // then a no-op and a record would slander their live lock. - if (ours) { - await this.writeWriterCloseRecord(released) - } } catch (err: any) { if (err.code !== 'ENOENT') { console.warn('[brainy] Failed to release writer lock file:', err) @@ -2231,97 +2079,6 @@ export class FileSystemStorage extends BaseStorage { } } - /** - * @description Read the clean-close record at `locks/_writer.close`, or - * `null` when it is absent or unparseable. A torn record is treated as - * absent — the conservative direction, since an unreadable record can - * vouch for nothing. - * @returns The record, or null. - */ - public async readWriterCloseRecord(): Promise { - await this.ensureInitialized() - const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) - try { - const raw = await fs.promises.readFile(recordFile, 'utf-8') - const parsed = JSON.parse(raw) as WriterCloseRecord - if ( - typeof parsed?.pid !== 'number' || - typeof parsed?.hostname !== 'string' || - typeof parsed?.startedAt !== 'string' || - typeof parsed?.closedAt !== 'string' - ) { - return null - } - return parsed - } catch (err: any) { - if (err.code === 'ENOENT') return null - return null - } - } - - /** - * @description Whether a clean-close record describes the very lock - * generation `lock` represents. The match is pid + hostname + `startedAt`: - * `startedAt` is the lock generation's identity, so a record can never - * vouch for a LATER lock taken by the same pid on the same host (the - * same-process re-open path mints a fresh `startedAt`). - * @param record - The clean-close record read from disk. - * @param lock - The lock file's contents. - */ - private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean { - return ( - record.pid === lock.pid && - record.hostname === lock.hostname && - record.startedAt === lock.startedAt - ) - } - - /** - * @description Write the clean-close record for a lock this instance just - * released. Atomic (temp + rename) so a concurrent opener never reads half - * a record. A failure here costs the next open nothing but the honest - * fallback (pid liveness), so it warns rather than failing the close. - * @param released - The lock info this instance held. - */ - private async writeWriterCloseRecord(released: WriterLockInfo): Promise { - const record: WriterCloseRecord = { - pid: released.pid, - hostname: released.hostname, - startedAt: released.startedAt, - closedAt: new Date().toISOString(), - version: released.version - } - const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) - try { - await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2)) - } catch (err) { - // ENOENT = the lock directory is gone, i.e. the whole store was removed - // under us. There is no next open to inform. - if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return - console.warn( - `[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` + - `the next open will fall back to pid liveness and may report this orderly ` + - `shutdown as a crash:`, - err - ) - } - } - - /** - * @description Remove the clean-close record. Called by every successful - * lock claim so a record never outlives the lock generation it describes. - */ - private async clearWriterCloseRecord(): Promise { - const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) - try { - await fs.promises.unlink(recordFile) - } catch (err: any) { - if (err.code !== 'ENOENT') { - console.warn('[brainy] Failed to clear the writer clean-close record:', err) - } - } - } - public override async readWriterLock(): Promise { await this.ensureInitialized() const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) @@ -2408,115 +2165,36 @@ export class FileSystemStorage extends BaseStorage { /** * Start watching for cross-process flush requests. Called by Brainy.init() - * in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers - * the supplied callback (`brain.flush()`), after which an `.ack` is written - * to `locks/_flush_responses/` with the same request ID. Stale `.req` files - * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep. - * - * THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request - * directory every 500 ms, per brain, for the entire life of every writer — - * armed on every non-reader brain whether or not any inspector process - * existed. MEASURED on a production process holding 21 brains: 42 directory - * reads per second on a completely idle service, plus a stale-request GC - * pass on every one of them. The engine does no periodic work without a - * cause, and a request that has not been made is not a cause. - * - * `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is - * seen SOONER than the old poll saw it. Two honest concessions ride with it: - * - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because - * `fs.watch` can miss events on network and fuse filesystems and because - * the stale-request GC needs some tick of its own. At 30s that is 0.7 - * reads/s across 21 brains where the poll cost 42. - * - a filesystem that cannot watch at all falls back to the ORIGINAL - * 500 ms poll, narrated once, because correctness outranks idle cost: - * an inspector whose request is never seen waits forever. + * in writer mode. Polls `locks/_flush_requests/` every + * FLUSH_WATCH_INTERVAL_MS — each new `.req` file triggers the supplied + * callback (`brain.flush()`), after which an `.ack` is written to + * `locks/_flush_responses/` with the same request ID. Stale `.req` files + * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick. */ public override startFlushRequestWatcher(onRequest: () => Promise): void { - // Already watching — or already ARMING. The arm is asynchronous (the - // request directory is created before it can be watched), so neither the - // watcher nor the interval exists yet during that window; the callback is - // the flag that covers it. Without this a second call in the window would - // leave two watchers and two sweeps running for the life of the store. - if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return + if (this.flushWatcherInterval) return // already watching this.flushWatcherOnRequest = onRequest const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) - const sweep = (): void => { - if (this.flushWatcherInFlight) return // skip overlapping sweep + // Ensure both dirs exist up front so the first .req drop doesn't race with mkdir. + this.ensureDirectoryExists(reqDir).catch(() => {}) + this.ensureDirectoryExists(ackDir).catch(() => {}) + + this.flushWatcherInterval = setInterval(() => { + if (this.flushWatcherInFlight) return // skip overlapping tick this.flushWatcherInFlight = true this.processFlushRequests(reqDir, ackDir).finally(() => { this.flushWatcherInFlight = false }) - } - - // Ensure both dirs exist up front so the first .req drop doesn't race with - // mkdir — and so there is a directory to watch. - void this.ensureDirectoryExists(reqDir) - .then(() => this.ensureDirectoryExists(ackDir)) - .then(() => { - if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile - try { - const watcher = fs.watch(reqDir, () => sweep()) - this.flushWatcher = watcher - watcher.on('error', (err: Error) => { - // A watch that dies mid-life must not leave the door deaf. - console.warn( - `[brainy] Flush-request watch failed (${err.message}) — falling back to polling.` - ) - this.flushWatcher?.close() - this.flushWatcher = undefined - // The SAFETY sweep must go first. It is already armed at 30s, and - // startFlushRequestPolling() declines to arm over an existing - // interval — so leaving it would quietly leave this store answering - // flush requests on a 30s cadence instead of the 500ms one the door - // promises. A degrade nobody asked for is still a degrade. - if (this.flushWatcherInterval) { - clearInterval(this.flushWatcherInterval) - this.flushWatcherInterval = undefined - } - this.startFlushRequestPolling(sweep) - }) - if (typeof watcher.unref === 'function') watcher.unref() - // The safety sweep: missed events on exotic filesystems, and the - // stale-request GC. - this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS) - if (typeof this.flushWatcherInterval.unref === 'function') { - this.flushWatcherInterval.unref() - } - // One sweep now: a request may have been dropped before the watch armed. - sweep() - } catch (err) { - console.warn( - `[brainy] Flush-request directory cannot be watched on this filesystem ` + - `(${(err as Error).message}) — polling every ` + - `${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.` - ) - this.startFlushRequestPolling(sweep) - } - }) - .catch(() => { - // The request directory could not be created; nothing to watch. A - // cross-process flush request cannot be made either, so there is - // nothing to miss. - }) - } - - /** The original 500 ms poll — the fallback when a directory cannot be watched. */ - private startFlushRequestPolling(sweep: () => void): void { - if (this.flushWatcherInterval) return - this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) + }, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) if (typeof this.flushWatcherInterval.unref === 'function') { this.flushWatcherInterval.unref() } } public override stopFlushRequestWatcher(): void { - if (this.flushWatcher) { - this.flushWatcher.close() - this.flushWatcher = undefined - } if (this.flushWatcherInterval) { clearInterval(this.flushWatcherInterval) this.flushWatcherInterval = undefined @@ -2889,73 +2567,24 @@ export class FileSystemStorage extends BaseStorage { // record reads), persist, and never scan again. Absent keys are a // legacy file, not a zero — a zero here would make every provider's // coverage ledger read "over-posted" on a populated store. - let needsPersist = false if ( typeof counts.totalNounCountAll === 'number' && typeof counts.totalVerbCountAll === 'number' ) { this.totalNounCountAll = counts.totalNounCountAll this.totalVerbCountAll = counts.totalVerbCountAll - if (counts.allCountsDerivedBy === 'identity-record') { - // Derived (or recounted) under the honest rule — one counted - // entity per metadata content leg. Trust the persisted suspect - // flag as-is; an unprovable delete since may still have set it. - this.allCountsDerivedBy = 'identity-record' - this.allCountsSuspect = counts.allCountsSuspect === true - } else { - // The ALL scalars exist but predate the identity-record stamp — - // they were derived under the legacy rule that counted one - // entity per id DIRECTORY, so orphaned ghost/scar containers (a - // pre-8.3.1 partial-delete defect — see pruneOrphanedEntities()) - // were counted as entities too. O(1) field read, NEVER a walk - // here: force suspect and name it loudly. A sanctioned recount - // (repairIndex) restores exact denominators and clears this. - this.allCountsDerivedBy = undefined - this.allCountsSuspect = true - needsPersist = true - prodLog.narrate( - '[FileSystemStorage] canonical count ledger was derived under the legacy ' + - 'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' + - 'container inflates it. Marked suspect, and an honest recount is scheduled to ' + - 'run in the background after this open; until it lands, do not subtract ' + - 'against these ALL scalars.' - ) - // A suspect ledger used to stay wrong for the life of the store, - // waiting for an operator to run repairIndex. A downstream index - // heal took its "remaining" figure from these inflated - // denominators and reported work that did not exist. The ledger - // now HEALS ITSELF — in the background, because a denominator is - // a derived scalar and no read is ever served from it. - this.scheduleCountLedgerDerivation('legacy container-rule ledger') - } + this.allCountsSuspect = counts.allCountsSuspect === true } else { - // No ALL scalars at all. There is nothing to serve in the meantime — - // a zero would read as an empty store — so the scalars stay unknown - // and SUSPECT until the background derivation lands. The open does - // not wait for it: an id-tree walk is O(ids) and this file has been - // the whole reason a 24k-id store opened in silence. - this.allCountsSuspect = true - this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger') - } - - // The vectored-noun scalar (shipped after the ALL scalars above — a - // counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but - // still predate THIS key). Unlike the ALL scalars, presence cannot be - // decided from the id-directory listing alone: a deferred-embed - // noun's `vectors.json` EXISTS with an empty `vector: []` until its - // embed lands, so this derivation reads every noun's `vectors.json` - // ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost. - if (typeof counts.totalVectoredNounCount === 'number') { - this.totalVectoredNounCount = counts.totalVectoredNounCount - } else { - // O(nouns) CONTENT reads — the most expensive derivation of the - // three, and the one most likely to have been the silent minutes at - // the front of a large store's open. Background, suspect until it - // lands, same as the ALL scalars. - this.allCountsSuspect = true - this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger') - } - if (needsPersist) { + const nouns = await this.scanCanonicalEntities('nouns') + const verbs = await this.scanCanonicalEntities('verbs') + this.totalNounCountAll = nouns.count + this.totalVerbCountAll = verbs.count + this.allCountsSuspect = false + console.warn( + `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + + `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + + `every tier) and persisted; no further scan.` + ) await this.persistCounts() } @@ -2982,22 +2611,6 @@ export class FileSystemStorage extends BaseStorage { * Initialize counts by scanning disk (only done once) */ private async initializeCountsFromDisk(): Promise { - const startedAt = Date.now() - // THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating: - // it derives `totalNounCount` / `totalVerbCount`, the scalars - // `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would - // make a populated store answer "0 entities" until the walk landed — a - // wrong answer, not a slow one, and the serving law grades a failure by - // whether an answer could be wrong. The ALL-visibility denominators, which - // no read is served from, DO run in the background (see - // scheduleCountLedgerDerivation). What this walk owes the operator instead - // is narration: it announces itself, and reports its wall. - prodLog.narrate( - `[FileSystemStorage] no usable counts.json — deriving the entity counters from ` + - `the canonical id tree now. This is O(ids) listings plus one vectors.json read ` + - `per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` + - `served from it. It runs once; the result is persisted.` - ) try { // Count the CANONICAL 8.0 layout (`entities////…`) — // the tree saveNoun/getNouns actually read and write. The previous scan @@ -3014,13 +2627,6 @@ export class FileSystemStorage extends BaseStorage { this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false - this.allCountsDerivedBy = 'identity-record' - // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT - // (a deferred-embed noun's file exists but holds an empty vector until - // its embed lands), so this is a full O(nouns) content scan — see - // scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here, - // alongside the rest of this from-disk recovery. - this.totalVectoredNounCount = await this.scanVectoredNounCount() // Sample some entities for the type distribution (don't read all). // Read the metadata files DIRECTLY with fs — this runs inside init(), @@ -3045,11 +2651,6 @@ export class FileSystemStorage extends BaseStorage { } await this.persistCounts() - prodLog.narrate( - `[FileSystemStorage] counter derivation from the canonical id tree finished in ` + - `${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` + - `${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.` - ) } catch (error) { console.error('Error initializing counts from disk:', error) } @@ -3057,132 +2658,11 @@ export class FileSystemStorage extends BaseStorage { /** * Walk the canonical `entities//<2-hex-shard>//` tree, counting - * one entity per id directory that holds the metadata CONTENT leg - * (`metadata.json` or its `.json.gz` variant — see - * {@link hasMetadataContentLeg}). A bare container — a ghost (a stale - * `vectors.json` left with no metadata leg) or a scar (an empty directory), - * both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the - * identity record IS the population (ADR-008 G1), never the directory. - * This is the ONE-TIME legacy derivation walk (see callers); a prior - * version of this scan counted every id directory regardless of content, - * over-counting any store carrying orphaned containers — see - * `allCountsDerivedBy` for how a counts.json derived under that old rule is - * marked suspect on load. Returns up to 100 sampled *counted* entity - * directories (absolute paths) — nouns feed the type-distribution estimate - * above. An absent tree (fresh store) counts zero. + * one entity per id directory (the layout `getNounVectorPath`/`getNouns` + * use). Returns up to 100 sampled entity directories (absolute paths) — + * nouns feed the type-distribution estimate above. An absent tree (fresh + * store) counts zero. */ - /** - * @description Derive the ALL-visibility count ledger honestly — one entity - * per IDENTITY RECORD, never per id directory — IN THE BACKGROUND, once, - * and persist the result stamped `identity-record`. - * - * Why background: these scalars are DENOMINATORS. No read is served from - * them, so deriving them cannot be allowed to hold an open hostage — a - * store with 24,898 ids spent minutes of a production restart inside walks - * exactly like these, in silence, before serving anything. Why at all: a - * ledger derived under the old container rule stayed wrong for the life of - * the store, and a downstream index heal subtracted against it and reported - * remaining work that did not exist (measured on a real store: 14,231 - * derived against 14,056 identity records — precisely the store's 25 noun - * scar directories; verbs 72,729 against 72,679, its 50 verb scars). - * - * Idempotent: a second call while one is in flight joins the first. - * @param reason - What made the ledger untrustworthy, quoted in narration. - * @returns Nothing; observe completion with {@link whenCountLedgerSettled}. - */ - private scheduleCountLedgerDerivation(reason: string): void { - if (this.countLedgerDerivation) return - this.countLedgerDerivation = (async () => { - const startedAt = Date.now() - prodLog.narrate( - `[FileSystemStorage] count-ledger derivation started in the background ` + - `(${reason}) — counting identity records, not id directories; the open does ` + - `not wait for it and no read is served from these scalars.` - ) - try { - const beforeNouns = this.totalNounCountAll - const beforeVerbs = this.totalVerbCountAll - const beforeVectored = this.totalVectoredNounCount - // A walk that RACED A WRITE cannot prove its number: a row that landed - // mid-walk may or may not have been in the shard the walk had already - // passed. Rather than persist a figure that might be off by one and - // stamp it "exact", the walk is repeated once on a quiet store, and if - // the store is never quiet the ledger stays SUSPECT and says so. One - // retry, never a spin. - let attempt = 0 - let derived: { nouns: number; verbs: number; vectored: number } | null = null - while (attempt < 2 && derived === null) { - attempt++ - const activityBefore = this.ledgerActivityStamp() - const nouns = await this.scanCanonicalEntities('nouns') - const verbs = await this.scanCanonicalEntities('verbs') - const vectored = await this.scanVectoredNounCount() - if (this.ledgerActivityStamp() === activityBefore) { - derived = { nouns: nouns.count, verbs: verbs.count, vectored } - } - } - if (derived === null) { - this.allCountsSuspect = true - prodLog.narrate( - `[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` + - `after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` + - `every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` + - `against; brain.repairIndex() derives them under a recount barrier.` - ) - return - } - this.totalNounCountAll = derived.nouns - this.totalVerbCountAll = derived.verbs - this.totalVectoredNounCount = derived.vectored - this.allCountsDerivedBy = 'identity-record' - this.allCountsSuspect = false - await this.persistCounts() - prodLog.narrate( - `[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` + - `${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` + - (beforeNouns !== derived.nouns || - beforeVerbs !== derived.verbs || - beforeVectored !== derived.vectored - ? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` + - `difference is ghost and scar containers the old rule counted as entities)` - : ' (unchanged)') + - ` — persisted, stamped identity-record, no longer suspect.` - ) - } catch (error) { - // The ledger stays suspect and the next open retries. Loud: a - // denominator nobody can derive is a fact an operator must have. - this.allCountsSuspect = true - prodLog.error( - `[FileSystemStorage] count-ledger derivation FAILED after ` + - `${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` + - `and must not be subtracted against; the next open retries:`, - error - ) - } - })() - } - - /** - * @description A cheap witness that the ledger changed while a walk was - * running. Every landed write moves one of these live counters, so an - * unchanged stamp across a walk means no write landed during it. - * @returns A value that differs whenever the live ALL scalars have moved. - */ - private ledgerActivityStamp(): string { - return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}` - } - - /** - * @description Resolve once any background count-ledger derivation has - * settled (succeeded or failed). Resolves immediately when none was needed. - * Exists so tests and operators can observe the ledger's honest value rather - * than race it; nothing in the read path waits on this. - * @returns A promise that settles with the derivation. - */ - public async whenCountLedgerSettled(): Promise { - await this.countLedgerDerivation - } - private async scanCanonicalEntities( kind: 'nouns' | 'verbs' ): Promise<{ count: number; sampleDirs: string[] }> { @@ -3198,21 +2678,9 @@ export class FileSystemStorage extends BaseStorage { const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) for (const entry of ids) { if (!entry.isDirectory()) continue - const idAbs = path.join(shardPath, entry.name) - let legs: string[] - try { - legs = await fs.promises.readdir(idAbs) - } catch (error: any) { - if (error?.code === 'ENOENT') continue - throw error - } - // No metadata content leg → a ghost or scar container → not an - // entity. Same test pruneOrphanedEntities() uses, so the two agree - // by construction. - if (!this.hasMetadataContentLeg(legs)) continue count++ if (sampleDirs.length < SAMPLE_MAX) { - sampleDirs.push(idAbs) + sampleDirs.push(path.join(shardPath, entry.name)) } } } @@ -3244,72 +2712,6 @@ export class FileSystemStorage extends BaseStorage { } } - /** - * Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs - * — the vector-side mirror of {@link readEntityMetadataRaw}, same - * reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`). - * @param entityDir - Absolute `entities/nouns//` directory. - * @returns The parsed vector record, or null when absent/unreadable. - */ - private async readEntityVectorRaw(entityDir: string): Promise { - const base = path.join(entityDir, 'vectors.json') - try { - return JSON.parse(await fs.promises.readFile(base, 'utf-8')) - } catch { - // fall through to the compressed variant - } - try { - const gz = await fs.promises.readFile(`${base}.gz`) - return JSON.parse(zlib.gunzipSync(gz).toString('utf-8')) - } catch { - return null - } - } - - /** - * Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector — - * the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, - * presence cannot be decided from the id-directory listing alone: a - * deferred-embed noun's `vectors.json` EXISTS (written at `add()` time - * with `vector: []`) until its embed LANDS, so this walk reads every - * noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing. - * ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts - * here either (Brainy's write paths normalize an explicit zero-norm - * vector to `[]` at write time, but a store created before that fix may - * still carry legacy all-zero rows on disk; this derivation must agree - * with the live ledger's definition of "vectored" regardless of when the - * row was written). Used ONLY for a one-time legacy-counts.json derivation - * or a lost/corrupted counts.json recovery; the result is persisted so - * this scan never repeats. - */ - private async scanVectoredNounCount(): Promise { - const base = path.join(this.rootDir, 'entities', 'nouns') - let vectored = 0 - try { - const shards = await fs.promises.readdir(base, { withFileTypes: true }) - for (const shard of shards) { - if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue - const shardPath = path.join(base, shard.name) - const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) - for (const entry of ids) { - if (!entry.isDirectory()) continue - const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) - if ( - record && - Array.isArray(record.vector) && - record.vector.length > 0 && - !isZeroNormVector(record.vector) - ) { - vectored++ - } - } - } - } catch (error: any) { - if (error?.code !== 'ENOENT') throw error - } - return vectored - } - /** * Persist counts to filesystem storage */ @@ -3326,30 +2728,14 @@ export class FileSystemStorage extends BaseStorage { // written before the ledger existed; initializeCounts() derives them once. totalNounCountAll: this.totalNounCountAll, totalVerbCountAll: this.totalVerbCountAll, - // Vectored-noun ledger scalar — absent in files written before it - // existed; initializeCounts() derives it once (a content scan, see - // scanVectoredNounCount()'s JSDoc). - totalVectoredNounCount: this.totalVectoredNounCount, allCountsSuspect: this.allCountsSuspect, - // Derivation-rule stamp for the ALL scalars above — 'identity-record' - // when they were counted one-per-metadata-content-leg (the honest - // rule); omitted (JSON.stringify drops `undefined`) when the current - // in-memory scalars came from a legacy container-rule counts.json - // that hasn't been through a sanctioned recount yet, so a future load - // keeps naming them suspect rather than trusting an unproven value. - allCountsDerivedBy: this.allCountsDerivedBy, lastUpdated: new Date().toISOString() } - // ATOMIC (temp + rename), never a plain writeFile. A direct write - // truncates the file first, so every persist opened a window — measured - // at roughly 750ms after a flush or close on a real store — in which a - // concurrent reader saw counts.json EMPTY. An empty file is unparseable, - // and an unparseable ledger sends the next open down the full-rescan - // path: the cheapest file in the store was costing the most expensive - // recovery. The rename is atomic, so a reader sees the old ledger or the - // new one, never neither. - await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2)) + await fs.promises.writeFile( + this.countsFilePath, + JSON.stringify(counts, null, 2) + ) } catch (error) { console.error('Error persisting counts:', error) } diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index ab2a52d3..f55a5626 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -520,12 +520,6 @@ export class MemoryStorage extends BaseStorage { let totalNouns = 0 let totalVerbs = 0 - // Vectored-noun scalar: unlike the bare presence check above, this needs - // the vectors.json RECORD'S content — a deferred-embed noun's record - // exists with an empty `vector: []` until its embed lands. In-memory this - // is a free field access (no I/O), unlike the filesystem adapter's - // per-noun disk read. - let totalVectoredNouns = 0 // Scan all paths in objectStore for (const path of this.objectStore.keys()) { @@ -534,10 +528,6 @@ export class MemoryStorage extends BaseStorage { if (nounMatch) { // Type is in metadata, not path - just count total totalNouns++ - const record = this.objectStore.get(path) as { vector?: unknown } | undefined - if (Array.isArray(record?.vector) && record.vector.length > 0) { - totalVectoredNouns++ - } } // Count verbs (entities/verbs/{shard}/{id}/vectors.json) @@ -553,7 +543,6 @@ export class MemoryStorage extends BaseStorage { // A scan of every canonical record IS the ALL-visibility count. this.totalNounCountAll = totalNouns this.totalVerbCountAll = totalVerbs - this.totalVectoredNounCount = totalVectoredNouns this.allCountsSuspect = false } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index d8bcb780..6f4c7f0e 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -125,36 +125,6 @@ export interface WriterLockInfo { rootDir?: string // Convenience for log lines / error messages } -/** - * THE CLEAN-CLOSE RECORD. Written by `releaseWriterLock()` at the instant it - * gives up the writer lock, naming the lock identity it released. The next - * `acquireWriterLock()` reads it and can then say — from a RECORD, not from a - * guess — whether the previous writer left on purpose. - * - * Why a record and not PID liveness: "the recorded PID is no longer alive" is - * true of every orderly restart AND of every crash, so the two were reported - * identically ("appears dead") and neither could be trusted. Worse, the same - * inference fails the other way when the operating system RECYCLES the pid — - * a live unrelated process makes a long-dead writer's lock look held, and the - * store refuses to open naming a pid that was never Brainy. A record settles - * both: matched → the previous writer closed cleanly, nothing to recover; - * absent → say so, and name what recovery the open will now run. - * - * Lifecycle: written at release, consumed (deleted) by the next successful - * lock claim — a record must never outlive the lock generation it describes, - * or it would vouch for a later crash. - */ -export interface WriterCloseRecord { - pid: number - hostname: string - /** `startedAt` of the lock this close released — the identity match key. */ - startedAt: string - /** ISO timestamp at which the lock was released. */ - closedAt: string - /** Brainy version that performed the close. */ - version: string -} - /** * FNV-1a hash returning a 2-char hex bucket (00-ff). * Distributes system keys across 256 sub-prefixes to avoid @@ -233,40 +203,6 @@ function idFromVectorPath(path: string): string { return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix } -/** - * @description Extract the entity id embedded in a metadata path - * (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD - * mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their - * population on this file (ADR-008 G1: the metadata record IS the population; - * the vector leg is optional), so walk ordering and cursor resume derive the - * id from THIS path, never the vector path — a row with metadata and no - * vector file must still be listed, ordered, and resumable. - * @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`). - * @returns The entity id (the path segment immediately before `/metadata.json`). - */ -function idFromMetadataPath(path: string): string { - const withoutSuffix = path.replace(/\/metadata\.json$/, '') - const lastSlash = withoutSuffix.lastIndexOf('/') - return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix -} - -/** - * @description The sanctioned UNVECTORED shape for a noun hydrated during - * enumeration when its identity record (metadata.json) exists but its vector - * leg (vectors.json) does not — a fold-born metadata-only after-image, or any - * row genuinely without a vector yet. Mirrors the shape - * `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned - * unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a - * walk-yielded unvectored row is byte-shape-identical to one produced by that - * migration. Callers already handle `vector: []` as first-class - * (validateAddParams exempts it; index gates key on `length > 0`). - * @param id - The noun id. - * @returns A structurally-valid, vector-empty `HNSWNoun`. - */ -function unvectoredNoun(id: string): HNSWNoun { - return { id, vector: [], connections: new Map>(), level: 0 } -} - /** * Get ID-first path for verb metadata * No type parameter needed - direct O(1) lookup by ID @@ -1437,29 +1373,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.listObjectsUnderPath(prefix) } - /** - * @description The IMMEDIATE child directory names under a prefix — one - * level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a - * separate door exists. This default derives them from the recursive - * listing, so it is never WRONG, only never faster; the filesystem adapter - * overrides it with a single directory read. - * @param prefix - Storage-root-relative directory prefix. - * @returns The child directory names (not paths), in listing order. - */ - public async listRawPrefixes(prefix: string): Promise { - await this.ensureInitialized() - const paths = await this.listObjectsUnderPath(prefix) - const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/` - const names = new Set() - for (const p of paths) { - const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null - if (rest === null) continue - const slash = rest.search(/[/\\]/) - if (slash > 0) names.add(rest.slice(0, slash)) - } - return [...names] - } - /** * Remove every object under a storage-root-relative prefix. The filesystem * adapter overrides this with a recursive directory removal; this default @@ -1540,18 +1453,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * rollups are derived state with their own rebuild paths * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`). * - * EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on - * purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on - * this to legitimately un-write a vector a failed transaction added. This - * is deliberately NOT "preserve if absent" — a caller replaying an - * AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must - * apply preserve-if-absent itself BEFORE calling this, by reading the - * current vector and carrying it forward when the after-image's own - * vector leg is null/undefined but its metadata is not (see `replayFact` - * for the implementation and full rationale). A caller that genuinely - * wants to unvector a row uses the sanctioned, ledger-correct path - * (`Brainy.unvectorNounForRootMigration`) — never this primitive. - * * @param id - The entity id. * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}. */ @@ -1588,9 +1489,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Restore a relationship's raw stored objects byte-for-byte (verb-side - * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats, - * same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the - * fold's preserve-if-absent logic lives at its call site, not here). + * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats). * * @param id - The relationship id. * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}. @@ -1863,10 +1762,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a noun from storage - * @param hadVector - OPTIONAL vectored-noun ledger hint, forwarded to - * {@link deleteNounMetadata} unchanged — see its JSDoc. */ - public async deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise { + public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise { await this.ensureInitialized() // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the @@ -1883,7 +1780,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // LONGER wrapped in a blind catch that masked faults as "file didn't exist". // `priorMetadata` (the caller's pre-delete read) keeps the decrement honest // even when the canonical read inside returns null (replace race / ghost). - await this.deleteNounMetadata(id, priorMetadata, hadVector) + await this.deleteNounMetadata(id, priorMetadata) // Remove the now-empty entity container (a no-op for key/prefix stores). await this.removeCanonicalContainer(getNounVectorPath(id)) @@ -2284,18 +2181,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by noun id) so offset windows and cursor resume // are deterministic; ids come from the path so skipped nouns are never read. - // - // IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record - // (not the vector) IS the population — a noun with metadata and no vector - // file (a fold-born after-image, see writeNounRaw's preserve-if-absent - // contract) must still enumerate. Keying on metadata.json here means the - // ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed) - // and this walk agree on population by construction. Ordering is - // unaffected for a healthy store: every vectored noun has both legs, so - // the id set and sort order are identical to the old vectors.json keying. const entries = nounFiles - .filter((p) => p.includes('/metadata.json')) - .map((p) => ({ path: p, id: idFromMetadataPath(p) })) + .filter((p) => p.includes('/vectors.json')) + .map((p) => ({ path: p, id: idFromVectorPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor @@ -2320,24 +2208,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { ) { const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) const hydrated = await Promise.all( - batch.map(async ({ path: metadataPath, id }) => { + batch.map(async ({ path: nounPath }) => { try { - const metadata = await this.readCanonicalObject(metadataPath) + const noun = await this.readCanonicalObject(nounPath) + if (!noun) return null + const deserialized = this.deserializeNoun(noun) + const metadata = await this.getNounMetadata(deserialized.id) if (!metadata) return null - // The vector leg is OPTIONAL (population law): a metadata-only - // row hydrates with the sanctioned unvectored shape rather than - // being dropped from the walk. A fault reading the vector leg - // is treated the same as absence — best-effort, matching the - // canonical recount's tolerance for an unreadable vectors.json - // (rebuildTypeCounts) — a vector-leg problem never hides an - // otherwise-good identity record. - let deserialized: HNSWNoun - try { - const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id)) - deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id) - } catch { - deserialized = unvectoredNoun(id) - } return { deserialized, metadata } } catch (error) { // A TORN record must surface typed — a paginated read that @@ -2347,9 +2224,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // walk's job is to HEAL PAST it — skip the victim, serve the rest. // Identity point-reads (get-by-id) still throw typed upstream. if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } - // Skip nouns whose IDENTITY record fails to load (the metadata - // read above) — that is the one leg this walk cannot proceed - // without. + // Skip nouns that fail to load return null } }) @@ -2470,14 +2345,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/nouns/${shardHex}` try { const nounFiles = await this.listCanonicalObjects(shardDir) - // IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching - // comment in getNounsWithPagination: metadata.json is the population; - // the vector leg is optional, so a metadata-only row must still be - // listed (and here, for the unfiltered case, needs ZERO reads either - // way — the id comes straight from the path). const entries = nounFiles - .filter((p) => p.includes('/metadata.json')) - .map((p) => idFromMetadataPath(p)) + .filter((p) => p.includes('/vectors.json')) + .map((p) => idFromVectorPath(p)) .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) const toWalk = cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries @@ -2688,79 +2558,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by verb id) so offset windows and cursor resume // are deterministic and consistent across calls. Ids come from the path, so // verbs skipped by the cursor are never read. - // - // IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of - // this comment in getNounsWithPagination applies here too: metadata.json - // is the population; keying on it here means this walk and the ledger - // recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed) - // agree on population by construction. Unchanged for a healthy store — - // `relate()` always writes both legs of a verb in the same commit, so - // the id set and order match the old vectors.json keying exactly; this - // only additionally surfaces a fold-born metadata-only row (see - // writeVerbRaw's preserve-if-absent contract). const entries = verbFiles - .filter((p) => p.includes('/metadata.json')) - .map((p) => ({ path: p, id: idFromMetadataPath(p) })) + .filter((p) => p.includes('/vectors.json')) + .map((p) => ({ path: p, id: idFromVectorPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) - for (const { path: metadataPath, id: verbId } of entries) { + for (const { path: verbPath, id: verbId } of entries) { if (collected.length >= peekCount) break // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id // (later shards are processed in full). No read for skipped verbs. if (cursor && shard === cursor.shard && verbId <= cursor.id) continue try { - // Identity leg first — required. A verb this walk cannot read - // metadata for cannot be hydrated at all (same as before). - const metadata = await this.readCanonicalObject(metadataPath) - if (!metadata) continue + const rawVerb = await this.readCanonicalObject(verbPath) + if (!rawVerb) continue - // The vector leg is the verb's STRUCTURAL core (verb/sourceId/ - // targetId live there — see coreTypes.ts HNSWVerb), unlike a - // noun's vector, which is pure embedding data. `relate()` always - // writes both legs atomically and verbs have no deferred-embed - // path, so a healthy store's verbs always have both. A vector-leg - // absence here can only be a fold-born after-image (see - // writeVerbRaw's preserve-if-absent contract) — and unlike a - // noun, this walk cannot safely FABRICATE sourceId/targetId to - // synthesize a structurally-valid verb (an empty-string endpoint - // would silently create a phantom edge — worse than omission). - // If the metadata record happens to carry its own sourceId/ - // targetId (never true for current production writes, but not - // disallowed — e.g. a future schema or a repair tool could - // populate them), reconstruct from those; otherwise this row is - // loudly skipped — counted by the ledger, but not returned as an - // item, until a repair can supply the missing endpoints. - const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId)) - let verb: HNSWVerb - if (rawVerb) { - verb = this.deserializeVerb(rawVerb) - } else { - const metaSourceId = (metadata as Record).sourceId - const metaTargetId = (metadata as Record).targetId - const metaVerbType = (metadata as Record).verb - if ( - typeof metaSourceId === 'string' && metaSourceId.length > 0 && - typeof metaTargetId === 'string' && metaTargetId.length > 0 && - typeof metaVerbType === 'string' && metaVerbType.length > 0 - ) { - verb = { - id: verbId, - vector: [], - connections: new Map>(), - verb: metaVerbType as VerbType, - sourceId: metaSourceId, - targetId: metaTargetId - } - } else { - prodLog.error( - `[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` + - `record but no vector leg and no recoverable sourceId/targetId — ` + - `skipping (counted by the ledger, not yielded; needs repair).` - ) - continue - } - } + // Deserialize connections Map from JSON storage format + const verb = this.deserializeVerb(rawVerb) // Apply type filter if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) { @@ -2777,6 +2591,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { continue } + // Load metadata + const metadata = await this.getVerbMetadata(verb.id) + // Apply subtype filter (requires metadata — checked AFTER load) if (filterSubtypes) { const subtype = metadata?.subtype as string | undefined @@ -3575,12 +3392,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Save noun metadata to storage (now typed) * Routes to correct sharded location based on UUID - * @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc. */ - public async saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise { + public async saveNounMetadata(id: string, metadata: NounMetadata): Promise { // Validate noun type in metadata - storage boundary protection validateNounType(metadata.noun) - return this.saveNounMetadata_internal(id, metadata, hasVector) + return this.saveNounMetadata_internal(id, metadata) } /** @@ -3591,10 +3407,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { * This ensures counts are updated AFTER metadata exists, fixing the race condition * where storage adapters tried to read metadata before it was saved. * - * @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc. * @protected */ - protected async saveNounMetadata_internal(id: string, metadata: NounMetadata, hasVector?: boolean): Promise { + protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise { await this.ensureInitialized() // ID-first path - no type needed! @@ -3650,14 +3465,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { // record persists here so the ALL scalar never lags the tree. if (isNew) { this.totalNounCountAll++ - // Vectored-noun ledger: rides the SAME isNew gate (once per id, at - // creation) — this seam is metadata-write-driven and never re-runs on - // the HNSW neighbor-link re-saves that hit saveNoun_internal, so it - // cannot double-count. A deferred-embed insert passes hasVector=false - // (or omits it); its vector lands later via noteVectorLanded(). - if (hasVector) { - this.totalVectoredNounCount++ - } if (!(metadata.noun && isCounted)) { this.scheduleCountPersist().catch(() => { // Ignore persist errors — the in-memory count is authoritative; a later op retries. @@ -4053,18 +3860,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { * the skip permanently inflated the persisted totals (adds counted, paired * removals not decremented), and `Math.max(totalNounCount, scanned)` made * the inflation unfixable by any disk cleanup. - * @param hadVector - OPTIONAL vectored-noun ledger hint — see - * {@link StorageAdapter.deleteNounMetadata}'s JSDoc. This method never - * reads `vectors.json` to answer the question itself (a canonical read - * the delete path must never add); a caller that cannot supply the fact - * for free leaves it `undefined`, and the ledger goes SUSPECT rather - * than guessing. */ - public async deleteNounMetadata( - id: string, - priorRecord?: NounMetadata | null, - hadVector?: boolean - ): Promise { + public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise { await this.ensureInitialized() // Direct O(1) delete with ID-first path. Read the canonical record BEFORE @@ -4088,15 +3885,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { } else { this.markAllCountsSuspect('noun', id) } - - // Vectored-noun ledger: a KNOWN vector fact decrements (or no-ops); - // an UNKNOWN one goes suspect rather than guessing — see @param hadVector. - if (hadVector === true) { - if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- - else this.markAllCountsSuspect('noun-vector', id) - } else if (hadVector === undefined) { - this.markAllCountsSuspect('noun-vector', id) - } this.scheduleCountPersist().catch(() => { // Ignore persist errors — the in-memory count is authoritative; a later op retries. }) @@ -4761,16 +4549,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { // construction. let allNouns = 0 let allVerbs = 0 - // Vectored-noun scalar: unlike `allNouns` (decided from the metadata.json - // LISTING alone), presence cannot be decided from the vectors.json - // listing alone — a deferred-embed noun's vectors.json EXISTS with an - // empty `vector: []` until its embed lands, so the file's CONTENT must be - // read. This walk already lists every path per shard (including - // vectors.json entries — `listCanonicalObjects` yields both legs), so - // reading them here costs one EXTRA read per noun beyond the metadata.json - // read above (doubling this walk's per-noun I/O) — honest cost, paid only - // by this diagnostic/repair recount, never on the hot path. - let allVectoredNouns = 0 // Scan noun shards for (let shard = 0; shard < 256; shard++) { @@ -4781,18 +4559,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const paths = await this.listCanonicalObjects(shardDir) for (const path of paths) { - if (path.includes('/vectors.json')) { - try { - const vectorRecord = await this.readCanonicalObject(path) - if (vectorRecord && Array.isArray(vectorRecord.vector) && vectorRecord.vector.length > 0) { - allVectoredNouns++ - } - } catch (error) { - // Skip vector records that fail to load — best-effort ground truth, - // same as the metadata read below. - } - continue - } if (!path.includes('/metadata.json')) continue allNouns++ @@ -4868,23 +4634,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { // IS the proof an unprovable delete could not give. const nounsAllBefore = this.totalNounCountAll const verbsAllBefore = this.totalVerbCountAll - const vectoredBefore = this.totalVectoredNounCount this.totalNounCountAll = allNouns this.totalVerbCountAll = allVerbs - this.totalVectoredNounCount = allVectoredNouns this.allCountsSuspect = false - // This walk counts one entity per metadata.json record (never per bare - // container) — the identity-record rule. Stamp it so a future load - // trusts these scalars instead of naming them suspect at open. - this.allCountsDerivedBy = 'identity-record' this.countCache.clear() await this.persistCounts() prodLog.info( `[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` + - `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs / ${allVectoredNouns} vectored nouns` + - (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs || vectoredBefore !== allVectoredNouns - ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore} / ${vectoredBefore})` + `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` + + (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs + ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})` : ' (unchanged)') + ` — scalar + per-type persisted` ) diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts index 46c67f44..64b18dc1 100644 --- a/src/storage/storageFactory.ts +++ b/src/storage/storageFactory.ts @@ -154,21 +154,6 @@ export function resolveFilesystemRoot( ) { throwRemovedStorageKey('fileSystemStorage.path') } - // A nested `config` object carrying a path-shaped key is the same hazard in - // a shape nobody ever supported: it used to fall through SILENTLY to the - // shared default root — every instance writing one directory while its - // caller believed each had its own. (Found live: an integration test's - // brains shared one store across a whole single-process run and a health - // probe refused on the foreign edges it sampled.) Loud, with the rename. - const nested = (config as Record).config - if (nested && typeof nested === 'object') { - const pathish = ['path', 'baseDir', 'rootDirectory', 'rootDir', 'dir', 'directory'] - const hit = pathish.find( - (k) => typeof (nested as Record)[k] === 'string' && - ((nested as Record)[k] as string).length > 0 - ) - if (hit) throwRemovedStorageKey(`config.${hit}`) - } // 3. Zero-config default. A `type: 'filesystem'` with no path lands here // intentionally ("persist, default location"). diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 1bbbca88..139c67fe 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -13,8 +13,6 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' -import { isZeroNormVector } from '../../utils/distance.js' -import { prodLog } from '../../utils/logger.js' /** * Backend identity stamped into an operation's emitted `name` string (e.g. @@ -90,30 +88,6 @@ export class AddToVectorIndexOperation implements Operation { } async execute(): Promise { - // THE ZERO-NORM LAW (the live provider-write seam's belt): a zero-norm - // vector is not a vector — it never crosses an engine boundary. This - // engine's own cosine distance treats an all-zero vector safely (a - // zero-norm operand always scores MAXIMUM distance, see - // {@link isZeroNormVector}'s JSDoc), but a downstream engine serving - // squared-euclidean distance cannot tell it apart from a legitimate - // origin point — a false attractor that silently darkens real results. - // The canonical write already landed (SaveNoun/SaveNounMetadata - // operations are staged ahead of this one in every caller) — only the - // INDEX INSERT is refused here, loudly, never a throw. A length-0 - // vector is the unrelated "unvectored" shape and is skipped silently - // (the same contract callers already rely on for deferred embeds). - if (this.vector.length === 0) { - return async () => {} - } - if (isZeroNormVector(this.vector)) { - prodLog.warn( - `[vector-index] refusing to index a zero-norm vector for entity ${this.id} — ` + - `a zero-norm vector is not a vector and never crosses an engine boundary ` + - `(the canonical write is unaffected; only the vector-index insert is skipped)` - ) - return async () => {} - } - // Check if item already exists (for rollback decision) const existed = await this.itemExists(this.id) @@ -289,52 +263,14 @@ export class ReplaceInVectorIndexOperation implements Operation { // One commit generation for the whole replace (both branches + rollback). const generation = this.generationFn?.() - // THE ZERO-NORM LAW (see AddToVectorIndexOperation's matching JSDoc): a - // real all-zero replacement vector must never land in the index — refuse - // loudly, canonical write unaffected. The row must not be left stale - // either: if it was genuinely indexed under `oldVector`, remove it - // rather than pretend the old vector still describes the row. A - // length-0 `newVector` (the unrelated "unvectored" shape) is handled the - // same way, silently — no caller today reaches this with an empty - // replacement (update() rejects a dimension-mismatched empty vector), - // but the seam stays consistent in case one ever legitimately does. - if (isZeroNormVector(this.newVector) || this.newVector.length === 0) { - const wasIndexed = this.oldVector.length > 0 && !isZeroNormVector(this.oldVector) - if (isZeroNormVector(this.newVector)) { - prodLog.warn( - `[vector-index] refusing to replace with a zero-norm vector for entity ${this.id} — ` + - `a zero-norm vector is not a vector and never crosses an engine boundary ` + - `(the canonical write is unaffected; the row is removed from the vector index instead)` - ) - } - if (wasIndexed) { - await this.index.removeItem(this.id, generation) - } - return async () => { - // Restore the declared before-state. - if (wasIndexed) { - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) - } - } - } - if (typeof index.updateItem === 'function') { // Atomic path: one in-place call, the row never leaves the index. await index.updateItem({ id: this.id, vector: this.newVector }, generation) return async () => { // Restore the declared before-state in place (see class JSDoc for - // the item-did-not-exist posture). A length-0 oldVector means the row - // was never actually indexed before this op ran (a length-0 vector is - // never a legal index member — see EmptyVectorIndexError) — there is - // no in-place "restore to empty" for the provider to perform, so - // rollback removes the row instead, leaving the same "not indexed" - // state the row was in before execute(). - if (this.oldVector.length > 0) { - await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) - } else { - await this.index.removeItem(this.id, generation) - } + // the item-did-not-exist posture). + await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) } } @@ -345,14 +281,9 @@ export class ReplaceInVectorIndexOperation implements Operation { return async () => { // updateItem-style restore via the same adjacent pair, back to the - // declared before-state. Same length-0 carve-out as the updateItem - // path above: an empty oldVector was never a legal index member, so - // rollback just leaves the row removed rather than attempting an - // illegal empty re-add. + // declared before-state. await this.index.removeItem(this.id, generation) - if (this.oldVector.length > 0) { - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) - } + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) } } } diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index 8b2ebffe..c1e9f1c1 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -52,15 +52,7 @@ export class SaveNounMetadataOperation implements Operation { private readonly storage: StorageAdapter, private readonly id: string, private readonly metadata: NounMetadata, - private readonly isNew: boolean = false, - /** - * OPTIONAL vectored-noun ledger hint: `true` when this write's paired - * vector (the SAME insert's `vector` local) is real/non-empty — see - * {@link StorageAdapter.saveNounMetadata}'s JSDoc for the isNew-gated, - * double-count-proof seam this rides. Default `false`: a deferred-embed - * insert (or any caller that doesn't know) never counts here. - */ - private readonly hasVector: boolean = false + private readonly isNew: boolean = false ) {} async execute(): Promise { @@ -70,7 +62,7 @@ export class SaveNounMetadataOperation implements Operation { : await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata') // Save new metadata - await this.storage.saveNounMetadata(this.id, this.metadata, this.hasVector) + await this.storage.saveNounMetadata(this.id, this.metadata) // Return rollback action return async () => { @@ -78,10 +70,8 @@ export class SaveNounMetadataOperation implements Operation { // Restore previous metadata await this.storage.saveNounMetadata(this.id, previousMetadata) } else { - // Delete newly created metadata. `this.hasVector` is the SAME fact - // this operation's own execute() used to (maybe) count the vectored - // ledger — reversing with it on rollback needs no new read. - await this.storage.deleteNounMetadata(this.id, undefined, this.hasVector) + // Delete newly created metadata + await this.storage.deleteNounMetadata(this.id) } } } @@ -150,9 +140,7 @@ export class SaveNounOperation implements Operation { // Note: Not all adapters implement deleteNoun // This is acceptable - metadata deletion makes entity invisible if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') { - // `this.noun.vector` is the SAME record just written — the - // vectored-noun ledger fact is free (no added read) and exact. - await this.storage.deleteNoun(this.noun.id, undefined, this.noun.vector.length > 0) + await this.storage.deleteNoun(this.noun.id) } } } @@ -210,21 +198,14 @@ export class DeleteNounMetadataOperation implements Operation { return async () => {} } - // Vectored-noun ledger fact: `previousNoun` is already read above for the - // before-image capture — no added read. `undefined` (noun genuinely - // absent, metadata-only ghost) is passed through honestly; the storage - // layer marks the ledger suspect rather than guessing. - const hadVector = previousNoun ? previousNoun.vector.length > 0 : undefined - // Full removal: both canonical legs + the entity container + count decrement // (the prior record keeps the decrement honest on a null canonical read). - await this.storage.deleteNoun(this.id, previousMetadata, hadVector) + await this.storage.deleteNoun(this.id, previousMetadata) // Return rollback action return async () => { // Restore the vector leg, then the metadata leg through the count-aware - // save so deleteNoun()'s decrement is reversed (hadVector's mirror: - // re-increments the vectored ledger iff the restored vector is real). + // save so deleteNoun()'s decrement is reversed. if (previousNoun) { await this.storage.saveNoun({ id: previousNoun.id, @@ -234,7 +215,7 @@ export class DeleteNounMetadataOperation implements Operation { }) } if (previousMetadata) { - await this.storage.saveNounMetadata(this.id, previousMetadata, hadVector === true) + await this.storage.saveNounMetadata(this.id, previousMetadata) } } } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index a0d55c1e..78e310f3 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -215,7 +215,7 @@ export interface ScoreExplanation { * * @example * ```ts - * declare module '@soulcraftlabs/brainy' { + * declare module '@soulcraft/brainy' { * interface SubtypeRegistry { * // For NounType.Person, subtype 'employee': * 'person:employee': { employeeId: string; department: string } @@ -1217,13 +1217,6 @@ export interface RepairFamilyReport { skipped?: string /** Why the outcome is what it is when neither `detail` nor `skipped` says it. */ reason?: string - /** - * The phase's own wall, in milliseconds. A repair on a production store ran - * for over thirty minutes without a single line of output; an operator had - * to read `top` to know it was alive. A receipt that cannot say WHERE the - * time went is not a receipt — every row carries its own. - */ - durationMs?: number } /** The full receipt returned by repairIndex(). */ @@ -1823,16 +1816,10 @@ export interface BrainyConfig { | StorageAdapter /** - * RE-MEANT (the health-gate contract): `init()` (open) always verifies the - * durable generation of every derived index, and a needed rebuild ALWAYS - * runs at open — it is never deferred to the first read, regardless of - * dataset size or this flag. There is no first-query lazy-build path - * anymore: a read that finds a provider not serving throws a typed - * `*NotReadyError` rather than building anything (see - * `assessProviderHealth` / the read gate in `brainy.ts`). Setting this - * `true` no longer defers index construction to the first query — it has - * no effect on WHEN a needed rebuild runs. Full manual control over - * rebuilds remains available via `repairIndex({ rebuild: [...] })`. + * Disable the automatic index rebuild check during `init()`. By default + * Brainy auto-decides from dataset size: small datasets rebuild missing + * indexes inline, large datasets rebuild lazily on first query. Set `true` + * only when an operator wants full manual control via `repairIndex()`. */ disableAutoRebuild?: boolean @@ -1990,32 +1977,25 @@ export interface BrainyConfig { reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB) /** - * Controls whether `init()` starts a BACKGROUND warm of the WASM embedding - * engine. + * Controls when the WASM embedding engine is initialized. * - * **Adaptive default (8.0, background since the open-path fix):** when - * omitted, `init()` STARTS a background warm of the engine whenever the - * WASM embedder is the *active* one — i.e. no native `'embeddings'` - * provider is registered — and this instance is a writer (not - * `mode: 'reader'`) running outside unit tests. The WASM module (≈93MB with - * the embedded model) takes 90-140s to compile on throttled CPUs — but - * `init()` never awaits that compile. It only starts it, so N concurrent - * opens no longer serialize on the one process-global engine singleton. - * The first `embed()` call then waits for whichever finishes first: the - * background warm (if still running) or its own fresh init (if the warm - * never started, e.g. `eagerEmbeddings: false`) — both paths converge on - * the SAME shared promise inside the engine singleton, so the vector is - * always correct; only the timing of who pays the wait differs. + * **Adaptive default (8.0):** when omitted, the engine eagerly initializes + * during `init()` whenever the WASM embedder is the *active* one — i.e. no + * native `'embeddings'` provider is registered — and this instance is a + * writer (not `mode: 'reader'`) running outside unit tests. The WASM module + * (≈93MB with the embedded model) takes 90-140s to compile on throttled + * CPUs, so paying that during boot rather than on the first `embed()`-driven + * call is the right default for a single-process server. * * The adaptive path skips itself automatically when a native embeddings * provider owns embeddings, in reader-mode (readers query existing vectors * and never embed), and in unit-test mode (kept fast via the mock embedder). * - * - `true` — force the background warm to start during `init()` (the - * adaptive default already does this for the active-embedder writer - * case; set it explicitly to be unambiguous). - * - `false` — no warm at all. Fully lazy: the first `embed()` call pays the - * full cold-compile cost inline, on whichever request triggers it. + * - `true` — force eager init during `init()` (the adaptive default already + * does this for the active-embedder writer case; set it explicitly to be + * unambiguous). + * - `false` — explicit override to force lazy init (first `embed()` call) + * even when this instance is the active embedder. */ eagerEmbeddings?: boolean diff --git a/src/types/reservedFields.ts b/src/types/reservedFields.ts index ce2108f8..15b585c5 100644 --- a/src/types/reservedFields.ts +++ b/src/types/reservedFields.ts @@ -65,7 +65,7 @@ * | `_rev` | system-managed revision counter — pass `ifRev` to `update()` for CAS | * * @example - * import { RESERVED_ENTITY_FIELDS } from '@soulcraftlabs/brainy' + * import { RESERVED_ENTITY_FIELDS } from '@soulcraft/brainy' * const isReserved = (key: string) => * (RESERVED_ENTITY_FIELDS as readonly string[]).includes(key) */ diff --git a/src/utils/brainyTypes.ts b/src/utils/brainyTypes.ts index a008a5e6..7db469bb 100644 --- a/src/utils/brainyTypes.ts +++ b/src/utils/brainyTypes.ts @@ -6,7 +6,7 @@ * * @example * ```typescript - * import { BrainyTypes } from '@soulcraftlabs/brainy' + * import { BrainyTypes } from '@soulcraft/brainy' * * // Get all available types * const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...] diff --git a/src/utils/distance.ts b/src/utils/distance.ts index d61bc12e..36e9e8e5 100644 --- a/src/utils/distance.ts +++ b/src/utils/distance.ts @@ -65,29 +65,6 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number = return 1 - similarity } -/** - * True when `vector` is a REAL (non-empty) all-zero vector — the "false - * attractor" shape this engine's own cosine distance treats safely (a - * zero-norm operand always scores the MAXIMUM distance, see - * {@link cosineDistance}) but a downstream engine serving squared-euclidean - * distance cannot distinguish from a legitimate origin point. THE LAW: a - * zero-norm vector is not a vector — it never crosses an engine boundary - * (never handed to a vector-index provider as a searchable item). - * - * A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape - * (the deferred-embed stub, a permanently-vectorless system row) and is - * deliberately NOT zero-norm here — callers checking for "nothing to index" - * should test `vector.length === 0` separately; this only flags the - * dangerous non-empty all-zero case. - */ -export function isZeroNormVector(vector: readonly number[]): boolean { - if (vector.length === 0) return false - for (let i = 0; i < vector.length; i++) { - if (vector[i] !== 0) return false - } - return true -} - /** * Calculates the Manhattan (L1) distance between two vectors. * Lower values indicate higher similarity. diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts index f1b52e3b..16266bec 100644 --- a/src/utils/indexReadiness.ts +++ b/src/utils/indexReadiness.ts @@ -13,28 +13,13 @@ * `size()` or `isInitialized`. When `isReady()` is absent, callers must fall back * to a KNOWN-ITEM PROBE (a real search/lookup that must return a known-present * datum) before trusting an empty result — never a `size()` proxy. - * - * {@link assessProviderHealth} is the NEWER, PREFERRED authority: it reads a - * provider's NAMED, synchronous, O(1) {@link import('../plugin.js').HealthReport} - * when one is exposed, and falls back to this file's `isReady()` classifier only - * when the provider does not (yet) expose a health report. Read paths in - * `brainy.ts` call `assessProviderHealth` exclusively — `assessIndexReadiness` - * stays exported for the other call sites (`storage/baseStorage.ts`) and for the - * fallback branch inside `assessProviderHealth` itself. */ -import type { HealthReport } from '../plugin.js' - /** A provider that MAY expose the honest cold-load readiness signal. */ export interface MaybeReadyProvider { isReady?: () => boolean } -/** A provider that MAY expose the named, synchronous, O(1) health report. */ -export interface MaybeHealthReportingProvider { - healthReport?: () => HealthReport -} - /** Three-valued honest-readiness verdict. */ export type IndexReadiness = 'ready' | 'not-ready' | 'unknown' @@ -51,185 +36,3 @@ export function assessIndexReadiness(provider: unknown): IndexReadiness { if (p == null || typeof p.isReady !== 'function') return 'unknown' return p.isReady() ? 'ready' : 'not-ready' } - -/** - * @description Which signal {@link assessProviderHealth} actually consulted to - * produce its verdict — surfaced so callers can narrate (and tests can pin) how - * a provider was judged, not just what the judgment was. - * - `'health-report'` — the provider's `healthReport()` was called (the authority). - * - `'is-ready'` — no `healthReport()`; fell back to the provider's `isReady()`. - * - `'size-heuristic'` — no `healthReport()` and no `isReady()`; caller must keep its own size-based heuristic. - * - `'none'` — there was no provider to assess (`null`/`undefined`). - */ -export type ProviderHealthVia = 'health-report' | 'is-ready' | 'size-heuristic' | 'none' - -/** The result of {@link assessProviderHealth}. */ -export interface ProviderHealthAssessment { - /** The honest readiness verdict — see {@link IndexReadiness}. */ - readiness: IndexReadiness - /** The provider's raw {@link HealthReport}, when one was obtained; `null` otherwise. */ - report: HealthReport | null - /** Which signal produced the verdict — see {@link ProviderHealthVia}. */ - via: ProviderHealthVia - /** Human-readable reasons: named failing invariants (with `heal`), unledgered families, or the fallback-path explanation. Empty when the provider is healthy and ready. */ - reasons: string[] -} - -/** - * @description THE read-gate authority. Prefers a provider's NAMED, - * synchronous, O(1) {@link HealthReport} over the older `isReady()` / size - * heuristics; falls back to {@link assessIndexReadiness}'s semantics only when - * a provider does not (yet) expose `healthReport()`. - * - * Derivation: - * - `healthReport()` present → call it (wrapped in try/catch). A THROW is a - * CONTRACT VIOLATION, not "unknown": returns `readiness: 'not-ready'`, - * `via: 'health-report'`, and a reason naming the throw — never swallowed - * into `'unknown'`. - * - Otherwise → `readiness = report.serving ? 'ready' : 'not-ready'`; `reasons` - * names every invariant with `holds: false` (with its `heal`), plus an - * `unledgered: [...]` line when {@link HealthReport.unledgered} is non-empty. - * UNLEDGERED IS UNKNOWN: an unledgered family never flips a serving provider - * to not-ready, and never flips a not-serving provider to ready — `serving` - * is always the provider's own verdict, verbatim. - * - No `healthReport()` → fall back to {@link assessIndexReadiness}'s semantics: - * `via: 'is-ready'` when `isReady()` exists, `via: 'size-heuristic'` when - * neither hook exists (caller must keep its own size-based heuristic), - * `via: 'none'` when there is no provider at all. - * @param provider - Any index provider (vector / graph / metadata) or `null`/`undefined`. - */ -export function assessProviderHealth(provider: unknown): ProviderHealthAssessment { - const p = provider as (MaybeHealthReportingProvider & MaybeReadyProvider) | null | undefined - - if (p == null) { - return { readiness: 'unknown', report: null, via: 'none', reasons: ['no provider to assess'] } - } - - if (typeof p.healthReport === 'function') { - let report: HealthReport - try { - report = p.healthReport() - } catch (err) { - const message = err instanceof Error ? err.message : String(err) - return { - readiness: 'not-ready', - report: null, - via: 'health-report', - reasons: [`healthReport() threw: ${message} — a health-report throw is a contract violation, never read as healthy`] - } - } - - const reasons: string[] = [] - for (const invariant of report.invariants) { - if (!invariant.holds) { - reasons.push(`${invariant.name} (heal:${invariant.heal}): ${invariant.detail}`) - } - } - if (report.unledgered.length > 0) { - reasons.push(`unledgered: ${report.unledgered.join(', ')}`) - } - - return { - readiness: report.serving ? 'ready' : 'not-ready', - report, - via: 'health-report', - reasons - } - } - - const readiness = assessIndexReadiness(p) - if (readiness === 'unknown') { - return { - readiness, - report: null, - via: 'size-heuristic', - reasons: ['provider exposes neither healthReport() nor isReady() — falling back to the size heuristic'] - } - } - return { - readiness, - report: null, - via: 'is-ready', - reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] - } -} - -/** - * @description A provider's self-report that it is REBUILDING ITS OWN index - * right now. Returned by the optional `rebuildInProgress()` hook. - * - * The distinction this exists to make: a provider reporting `serving: false` - * because it is BROKEN and a provider reporting `serving: false` because it is - * BUSY BUILDING ITSELF look identical through `healthReport()` alone, and - * brainy treated both the same way — it called `rebuild()` and waited for it, - * on the foreground of `init()`. A production store whose metadata provider - * had to rebuild paid 641 SECONDS of that wait before `init()` returned, with - * every other family idle behind it. - * - * A provider that reports progress here owns its own rebuild: brainy neither - * starts one nor waits for it, `init()` returns, the other families serve, and - * THAT family's doors refuse by name — carrying this progress — until the - * provider reports itself serving. - * - * Every field but `phase` is optional and every field is a MEASUREMENT: a - * provider reports only what it actually tracks, never an estimate dressed as - * a fact. - */ -export interface ProviderRebuildProgress { - /** The provider's own name for what it is doing. Quoted verbatim in refusals. */ - phase: string - /** Units completed so far, if the provider counts them. */ - done?: number - /** Units expected in total, if the provider knows it. */ - total?: number - /** Epoch millis when this rebuild started, if the provider tracks it. */ - startedAt?: number -} - -/** A provider that can report a rebuild it is running itself. */ -interface MaybeRebuildingProvider { - rebuildInProgress?: () => ProviderRebuildProgress | null -} - -/** - * @description Ask a provider whether it is rebuilding itself right now. - * Synchronous, O(1), feature-detected: a provider without the hook reports - * nothing and is treated exactly as before. - * @param provider - Any index provider, or `null`/`undefined`. - * @returns The provider's progress, or `null` when it is not rebuilding (or - * does not implement the hook). - */ -export function assessProviderRebuild(provider: unknown): ProviderRebuildProgress | null { - const p = provider as MaybeRebuildingProvider | null | undefined - if (p == null || typeof p.rebuildInProgress !== 'function') return null - try { - const progress = p.rebuildInProgress() - if (!progress || typeof progress.phase !== 'string' || progress.phase.length === 0) { - return null - } - return progress - } catch { - // A throwing hook says nothing trustworthy about a rebuild; fall through to - // the ordinary health verdict rather than inventing one. - return null - } -} - -/** - * @description Render a rebuild progress report as one operator-facing clause, - * for a refusal message. Includes only what the provider actually measured. - * @param progress - The provider's report. - * @returns A clause such as `rebuilding ("metadata shadow build", 4,096/14,056, 12s elapsed)`. - */ -export function describeRebuildProgress(progress: ProviderRebuildProgress): string { - const parts: string[] = [`"${progress.phase}"`] - if (typeof progress.done === 'number' && typeof progress.total === 'number') { - parts.push(`${progress.done.toLocaleString()}/${progress.total.toLocaleString()}`) - } else if (typeof progress.done === 'number') { - parts.push(`${progress.done.toLocaleString()} done`) - } - if (typeof progress.startedAt === 'number') { - parts.push(`${Math.round((Date.now() - progress.startedAt) / 1000)}s elapsed`) - } - return `rebuilding (${parts.join(', ')})` -} diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 0d6b6594..5154d4fd 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -266,26 +266,6 @@ export const prodLog = { console.error(message, ...args) }, - /** - * THE NARRATION CHANNEL — always visible, exactly like `error`. - * - * `warn`/`info`/`log` below are clamped to ERROR in any environment that - * looks like production (see isProductionEnvironment), which is the right - * default for chatter and the wrong one for the two things an operator is - * entitled to hear from a database no matter what: WHY IT IS SLOW and WHAT - * IT IS DOING ABOUT IT. A production service opening a 16 GB store spent - * three minutes emitting nothing at all — the phase timings that would have - * named the slow phase were written to `warn` and thrown away by the log - * level. Progress and cost narration goes here; it is never a per-record - * line, always a phase, a wall, or a bounded-cadence heartbeat. - * - * `silent: true` still silences it — that is the consumer's explicit - * request, not a cost default. - */ - narrate: (message?: any, ...args: any[]) => { - console.warn(message, ...args) - }, - // These are suppressed in production unless BRAINY_LOG_LEVEL is set warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args), info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args), diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 3e0e3d17..13cf3bb4 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -20,7 +20,6 @@ import { type WatermarkVerdict, type WatermarkVerdictResult } from './projectionWatermark.js' -import type { FactScanHandle } from '../db/factLog.js' import { NounType, VerbType, @@ -78,31 +77,6 @@ export interface MetadataIndexStats { indexSize: number // in bytes } -/** - * @description What {@link MetadataIndexManager.applyWatermarkCatchup} did, - * for the caller's narration. - * - `'noop'` — the verdict was `null`/`'adopt'`: the artifact already - * reflects committed truth. Zero index writes. - * - `'rescan'` — the verdict was `'rescan'`, OR a `'catchup'` verdict was - * demoted (no window, or no fact log to scan) — either way a full - * {@link MetadataIndexManager.rebuild} already ran; `reason` names why. - * - `'caught-up'` — the `(from, to]` window folded successfully; the - * artifact is stamped and flushed at `to`. - */ -export interface CatchupApplyResult { - action: 'noop' | 'rescan' | 'caught-up' - /** Present on `'rescan'` — why the fold could not proceed as a catchup. */ - reason?: string - /** Present on `'caught-up'` — the fact-log window that was folded. */ - window?: { from: number; to: number } - /** Present on `'caught-up'` — noun ops applied (add/update/delete). */ - nounsApplied?: number - /** Present on `'caught-up'` — verb ops applied (add/update/delete). */ - verbsApplied?: number - /** Present on `'caught-up'` — distinct committed generations folded. */ - factsApplied?: number -} - export interface MetadataIndexConfig { maxIndexSize?: number // Max number of entries per field value (default: 10000) rebuildThreshold?: number // Rebuild if index is this % stale (default: 0.1) @@ -173,52 +147,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { private stampedWatermark: number | null = null /** The three-way verdict computed at init; null until init runs. */ private loadVerdict: WatermarkVerdictResult | null = null - /** - * Set only when {@link loadVerdict}.verdict is `'rescan'`: whether a - * persisted artifact existed at load (even an unstamped/unverifiable - * one) — distinguishes genuine first boot (nothing here yet, routine) - * from an artifact whose watermark is unverifiable (the loud case). The - * verdict value alone doesn't carry this distinction; see {@link - * watermarkArtifactPresent}. - */ - private rescanArtifactPresent = false - - /** - * @description THE BUILD-BESIDE SEAM (B3 Deliverable 3): when set (via - * {@link beginShadow}), every live `addToIndex`/`removeFromIndex` call on - * THIS instance also applies to the shadow instance — so a caller building - * a fresh replacement manager beside this one (walking canonical into it) - * never misses a write that lands during the build. This is the ONE seam - * that makes build-beside possible without touching every call site: every - * existing `AddToMetadataIndexOperation`/`RemoveFromMetadataIndexOperation` - * (and the JS manager's own `rebuild()`/catchup fold) keep calling the SAME - * serving instance exactly as before; only THIS instance knows it is also - * mirroring to a shadow. Null = no build in flight (the overwhelmingly - * common case; the check costs one property read per write). - */ - private shadow: MetadataIndexManager | null = null - - /** - * @description Start mirroring every `addToIndex`/`removeFromIndex` call on - * this instance to `shadow` too — see {@link shadow}'s JSDoc. The caller - * owns sequencing: writes mirrored WHILE a canonical walk is populating - * `shadow` may be clobbered by the walk's own (possibly stale) reads for - * the same id; the caller closes that window with a bounded fact-log fold - * AFTER the walk (the same mechanism {@link applyWatermarkCatchup} uses) - * before treating `shadow` as authoritative. - * @param shadow - The manager to mirror writes to. - */ - beginShadow(shadow: MetadataIndexManager): void { - this.shadow = shadow - } - - /** - * @description Stop mirroring writes to a shadow (see {@link beginShadow}). - * Idempotent; a no-op when no shadow is attached. - */ - endShadow(): void { - this.shadow = null - } // Cardinality and field statistics tracking private fieldStats = new Map() @@ -1676,15 +1604,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { for (const { field } of fields) { this.metadataCache.invalidatePattern(`field_values_${field}`) } - - // THE BUILD-BESIDE SEAM — see `shadow`'s JSDoc. Mirrors this write to a - // shadow manager under construction, if one is attached. `skipFlush: - // true` always: the shadow's own persistence is the build orchestrator's - // job (it flushes once, after the swap — never mid-build, to avoid - // colliding with this instance's own persisted keys). - if (this.shadow) { - await this.shadow.addToIndex(id, entityOrMetadata, true, false, generation) - } } /** @@ -1757,11 +1676,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { // the real commit watermark (the JS mapper ignores it). this.idMapper.remove(id, generation) await this.idMapper.flush() - - // THE BUILD-BESIDE SEAM — see `shadow`'s JSDoc. - if (this.shadow) { - await this.shadow.removeFromIndex(id, metadata, generation) - } } /** @@ -2241,74 +2155,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { break } - // ===== ARRAY SET OPERATORS ===== - // An element-indexed array field makes all three exact on the - // index path. They were previously ABSENT from this switch, so - // `fieldResults` kept its initial `[]` and the whole find() - // returned an empty page — a documented, matcher-implemented - // operator answering silently wrong. Served here instead. - - // hasAll: [a, b] — the field's array contains EVERY operand: - // the intersection of each element's posting set. - case 'hasAll': { - if (!Array.isArray(operand)) { - fieldResults = [] - break - } - if (operand.length === 0) { - // Vacuously true of every row that HAS the field. - const anyBitmap = (this.columnStore && this.columnStore.hasField(field)) - ? await this.columnStore.rangeQuery(field) - : await this.getExistsBitmapLegacy(field) - fieldResults = this.idMapper.intsIterableToUuids(anyBitmap) - break - } - let intersection: Set | null = null - for (const item of operand) { - const ids = new Set(await this.getIds(field, item)) - if (intersection === null) { - intersection = ids - } else { - for (const id of [...intersection]) { - if (!ids.has(id)) intersection.delete(id) - } - } - if (intersection.size === 0) break - } - fieldResults = intersection ? [...intersection] : [] - break - } - - // noneOf: [a, b] — the field's value is NONE of the operands: - // the complement of their union. - case 'noneOf': { - if (!Array.isArray(operand)) { - fieldResults = [] - break - } - const excludeInts: number[] = [] - for (const value of operand) { - for (const uuid of await this.getIds(field, value)) { - const intId = this.idMapper.getInt(uuid) - if (intId !== undefined) excludeInts.push(intId) - } - } - fieldResults = this.complementIds(excludeInts) - break - } - - // excludes: value — the field's array does NOT contain the value: - // the complement of `contains`. - case 'excludes': { - const excludeInts: number[] = [] - for (const uuid of await this.getIds(field, operand)) { - const intId = this.idMapper.getInt(uuid) - if (intId !== undefined) excludeInts.push(intId) - } - fieldResults = this.complementIds(excludeInts) - break - } - // ===== MISSING OPERATOR ===== // missing: boolean - equivalent to exists: !boolean case 'missing': { @@ -2325,27 +2171,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { } break } - - // ===== EVERYTHING ELSE: REFUSED BY NAME, NEVER ANSWERED EMPTY ==== - // An equality/range posting index cannot evaluate a substring, a - // pattern or an array length without reading every row, and this - // path exists precisely to avoid that. It used to fall out of the - // switch with `fieldResults` still `[]`, so `find({ where: { name: - // { startsWith: 'a' } } })` returned an empty page and looked like - // an answer. An accepted operator either works or refuses — the - // matcher's own support for these operators governs in-memory - // filtering, never an index-backed find(). - default: - throw new BrainyError( - `Filter operator "${op}" on field "${rawField}" cannot be served by the ` + - `metadata index: an equality/range posting index cannot evaluate substrings, ` + - `patterns or array lengths without reading every row. It is REFUSED rather ` + - `than answered with an empty page. Filter on an indexable operator ` + - `(equals/eq, notEquals/ne, oneOf/in, noneOf, greaterThan/gt, ` + - `greaterThanOrEqual/gte, lessThan/lt, lessThanOrEqual/lte, between, contains, ` + - `excludes, hasAll, exists, missing) and narrow the rest in your own code.`, - 'INVALID_QUERY' - ) } // Intersect this operator's matches with the running set (AND semantics // for multiple operators on the same field). @@ -2934,8 +2759,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { * `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped < * committed; the gap from {@link watermarkGap} awaits an incremental * fold), `'rescan'` (unstamped or stamped above committed — never - * trusted). Null until init() has run. The coordinator (`Brainy.open()`) - * consumes this via {@link applyWatermarkCatchup} right after init. + * trusted). Null until init() has run. Computed and exposed only; no + * load behavior changes ride on it yet. */ watermarkVerdict(): WatermarkVerdict | null { return this.loadVerdict?.verdict ?? null @@ -2949,223 +2774,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { return this.loadVerdict?.gap ?? null } - /** - * @description Meaningful only when {@link watermarkVerdict} is - * `'rescan'`: `true` when a persisted artifact existed at load (even an - * unstamped/unverifiable one — real prior state, worth narrating loudly); - * `false` for a genuine first boot (nothing persisted yet — a caller - * should narrate this at a routine log level, not as an alarm, even - * though the verdict value is the same `'rescan'` either way). - */ - watermarkArtifactPresent(): boolean { - return this.rescanArtifactPresent - } - - /** - * @description Consume the three-way watermark verdict {@link - * watermarkVerdict} computed at init — the cure for a crash-recovered - * store whose canonical reads/counts recover every acked write but whose - * metadata projection (flushed only periodically, not per-commit) keeps - * serving the pre-crash state. Call once, right after `init()`, before - * anything reads from this projection. - * - * - `null`/`'adopt'` → the artifact already reflects the store's - * committed generation. Zero index writes. - * - `'catchup'` → the caller-supplied `scan` (expected already opened - * over `(watermarkGap().from, watermarkGap().to]`) is folded in, ONE - * op at a time, through the SAME two legs {@link rebuild} uses (ADR-007 - * A4 — one mechanism, never a second hand-rolled add/update shape): a - * tombstone (`op.record === null`) retracts id-keyed (this projection - * keeps no per-record delta log, so the pre-crash metadata for that id - * — if any — is what a value-precise removal would need, and it isn't - * available; the same tradeoff `remove()`'s null-metadata closure - * already accepts elsewhere); an after-image retracts-then-reposts, so - * an update never leaves stale postings under the old field values. A - * fact outside the window is skipped defensively (belt: the scan is - * already opened to the window; suspenders: this loop never trusts an - * over-run). On success the artifact is stamped at `to` and flushed — - * the same STAMP-AFTER-DATA door {@link flush} always writes through. - * - `'rescan'` (or a `'catchup'` verdict with no window, or no `scan` to - * fold — the store hosts no fact log) → the persisted artifact is - * unverifiable; this method runs the existing {@link rebuild} itself - * rather than leave the caller to notice and trigger it separately. - * - * @param scan - An open fact scan covering the catchup window (see - * {@link Brainy.scanFacts}), or `null` when none is available/needed. - * Ignored when the verdict is not `'catchup'`. - * @returns What happened — see {@link CatchupApplyResult}. - */ - async applyWatermarkCatchup(scan: FactScanHandle | null): Promise { - const verdict = this.watermarkVerdict() - if (verdict === null || verdict === 'adopt') return { action: 'noop' } - - if (verdict === 'rescan') { - await this.rebuild() - return { - action: 'rescan', - reason: 'persisted artifact is unverifiable (unstamped, or stamped ABOVE the ' + - "store's committed generation) — never adopting unverifiable state" - } - } - - // verdict === 'catchup' - const window = this.watermarkGap() - if (window === null) { - await this.rebuild() - return { action: 'rescan', reason: "'catchup' verdict exposed no window — cannot bound a fold" } - } - if (scan === null) { - await this.rebuild() - return { - action: 'rescan', - reason: `no fact log available to fold the (${window.from}, ${window.to}] catchup window` - } - } - - const { nounsApplied, verbsApplied, factsApplied } = await this.foldFactWindow(scan, window.from, window.to) - - this.stampWatermark(window.to) - await this.flush() - return { action: 'caught-up', window, nounsApplied, verbsApplied, factsApplied } - } - - /** - * @description Fold an open fact scan's `(fromGeneration, toGeneration]` - * window into this projection, ONE op at a time, through the SAME two legs - * {@link rebuild} uses (ADR-007 A4 — one mechanism, never a second - * hand-rolled add/update shape): a tombstone retracts id-keyed; an - * after-image retracts-then-reposts. THE CORE LOOP shared by {@link - * applyWatermarkCatchup} (which stamps + flushes after) and {@link - * buildBeside} (which does neither — persistence is the caller's job, - * exactly once, after a swap). Never stamps, never flushes, never touches - * storage beyond what `addToIndex`/`removeFromIndex` do internally - * (skipFlush is always forced true). - * @param scan - An open fact scan. - * @param fromGeneration - Window lower bound (exclusive). - * @param toGeneration - Window upper bound (inclusive). - * @returns Counts for the caller's narration. - */ - private async foldFactWindow( - scan: FactScanHandle, - fromGeneration: number, - toGeneration: number - ): Promise<{ nounsApplied: number; verbsApplied: number; factsApplied: number }> { - let nounsApplied = 0 - let verbsApplied = 0 - let factsApplied = 0 - for await (const batch of scan.batches()) { - for (const fact of batch.facts) { - // Defensive containment: the scan is already opened to the window, - // but a fact outside it is never applied regardless. - if (fact.generation <= fromGeneration || fact.generation > toGeneration) continue - const generation = BigInt(fact.generation) - for (const op of fact.ops) { - if (op.record === null) { - // TOMBSTONE — the id-keyed removal path (no per-record delta - // log to recover the old field values from). - await this.removeFromIndex(op.id, undefined, generation) - } else { - // AFTER-IMAGE — retract any stale posting for this id, then - // repost the new shape. Covers both a fresh add (nothing to - // retract; a no-op-ish remove) and an update, through the same - // two calls. - await this.removeFromIndex(op.id, undefined, generation) - await this.indexStoredRecord(op.id, op.record.metadata, { - skipFlush: true, - deferWrites: false, - generation - }) - } - if (op.kind === 'noun') nounsApplied++ - else verbsApplied++ - } - factsApplied++ - } - } - return { nounsApplied, verbsApplied, factsApplied } - } - - /** - * @description B3 Deliverable 3 — the shadow-build lifecycle's init: the - * MINIMUM setup {@link buildBeside} needs, deliberately NOT the general - * {@link init} sequence. Two reasons general `init()` is unsafe for a - * build-beside shadow: - * 1. `init()` unconditionally re-initializes the id mapper from storage - * (`idMapper.init()`) — safe for a FRESH mapper, but this instance is - * constructed with the CURRENTLY-SERVING manager's SHARED, already-live - * mapper (identity is shared, never a second mapper — this train's own - * law). Re-running its init() would DISCARD every not-yet-flushed - * UUID↔int assignment sitting in memory, breaking the live manager's - * own serving mid-build. - * 2. `init()` loads the field registry and, on a registry that's - * missing/empty while canonical has entities (exactly the shape a - * rebuild is often invoked to FIX), triggers `rebuild()` itself — - * WITHOUT `inMemoryOnly`, which would touch the shared storage keys - * the live manager depends on. - * What this DOES run: the WASM roaring-bitmap library init (idempotent; - * needed before any column-store write) and the column store's OWN - * segment-manifest discovery (read-only against shared storage; needed so - * THIS instance's eventual post-swap flush continues segment numbering - * correctly instead of colliding with the retiring manager's segments). - */ - private async initForShadowBuild(): Promise { - await roaringLibraryInitialize() - try { - await this.columnStore.init(this.storage, this.idMapper) - } catch (err) { - prodLog.warn('[MetadataIndex] shadow build: column store storage discovery failed:', err) - } - } - - /** - * @description B3 Deliverable 3 — THE ONLINE REBUILD's manager-side half: - * populate THIS instance (expected fresh/empty, constructed with the SAME - * storage + idMapper as the manager it will replace — see {@link - * initForShadowBuild}) from canonical storage without ever touching the - * shared storage keys the currently-serving manager depends on — no chunk - * deletion, no flush, anywhere in this call. The caller (the brain's - * rebuild-beside orchestrator) is responsible for: - * 1. Attaching this instance as a {@link beginShadow} target on the OLD - * manager BEFORE calling this, so live writes during the walk mirror - * here too (best-effort — the walk below may still clobber a mirrored - * write with a stale read for the same id; the fold after the walk is - * what makes the final state authoritative, not the mirror). - * 2. Swapping its own reference to this instance once this resolves. - * 3. Calling {@link stampWatermark} + {@link flush} EXACTLY ONCE, after - * the swap — this instance never persists itself. - * @param committedGenerationAtStart - The store's committed generation - * captured by the caller BEFORE this call — the fold's lower bound. - * @returns The generation this instance's canonical data reflects once the - * walk + fold settle — the fold's upper bound (writes committed after - * this point but before the swap only reach this instance via the live - * {@link beginShadow} mirror, so the caller re-reads the store's - * committed generation right before stamping, rather than trusting this - * return value as final). - * @throws If canonical advanced during the walk but no fact log is - * available to fold the gap — never a silently incomplete shadow. - */ - async buildBeside(committedGenerationAtStart: number): Promise { - await this.initForShadowBuild() - await this.rebuild({ inMemoryOnly: true }) - - const committedAfterWalk = this.storage.committedGeneration?.() ?? committedGenerationAtStart - if (committedAfterWalk > committedGenerationAtStart) { - const scan = this.storage.scanFacts?.({ - fromGeneration: committedGenerationAtStart + 1, - toGeneration: committedAfterWalk - }) ?? null - if (scan === null) { - throw new Error( - `MetadataIndexManager.buildBeside: canonical advanced from generation ` + - `${committedGenerationAtStart} to ${committedAfterWalk} during the walk, but this ` + - `store hosts no fact log to fold the gap — refusing a silently incomplete shadow` - ) - } - await this.foldFactWindow(scan, committedGenerationAtStart, committedAfterWalk) - } - return committedAfterWalk - } - /** * @description Write the pending watermark stamp as a sidecar record — * always called AFTER the data it certifies is durable. A stamp-write @@ -3218,7 +2826,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { if (result.verdict === 'rescan') { const artifactPresent = this.fieldIndexes.size > 0 || stamped !== null - this.rescanArtifactPresent = artifactPresent if (artifactPresent) { prodLog.warn( `[MetadataIndex] watermark verdict: RESCAN — persisted index is ` + @@ -3877,48 +3484,13 @@ export class MetadataIndexManager implements MetadataIndexProvider { } } - /** - * @description Index one raw stored noun/verb record — THE ONE add leg - * shared by {@link rebuild}'s canonical walk and {@link - * applyWatermarkCatchup}'s fact-log fold (ADR-007 A4: one mechanism, - * never a second hand-rolled shape). No conversion step is needed here: - * a raw stored record (`storage.getNounMetadata`/`getVerbMetadata`, or a - * fact's after-image `record.metadata`) is byte-identical — both read the - * exact same canonical path — and already the v2 nested-bag - * ("entity-record") shape {@link extractIndexableFields} expects. - * @param id - Entity/relationship id. - * @param storedMetadata - The raw stored metadata record. - * @param opts.skipFlush - Forwarded to {@link addToIndex}. - * @param opts.deferWrites - Forwarded to {@link addToIndex}. - * @param opts.generation - Forwarded to {@link addToIndex}. - */ - private async indexStoredRecord( - id: string, - storedMetadata: unknown, - opts: { skipFlush: boolean; deferWrites: boolean; generation?: bigint } - ): Promise { - await this.addToIndex(id, storedMetadata, opts.skipFlush, opts.deferWrites, opts.generation) - } - /** * Rebuild entire index from scratch using pagination * Non-blocking version that yields control back to event loop * Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map) - * - * @param options.inMemoryOnly - B3 Deliverable 3 (build-beside): when - * `true`, this call never touches the shared storage keys another, - * currently-serving `MetadataIndexManager` over the SAME storage may - * depend on — it skips deleting persisted legacy chunk files AND skips - * the final `flush()` (which would otherwise write field indexes AND - * flush the column store's tail buffers to shared segment keys, - * colliding with a live manager's own writes). The caller ({@link - * buildBeside}) owns persistence entirely — exactly once, after this - * instance becomes the sole owner via an atomic swap. Default `false` - * (every other caller keeps today's clear-then-persist behavior). */ - async rebuild(options?: { inMemoryOnly?: boolean }): Promise { + async rebuild(): Promise { if (this.isRebuilding) return - const inMemoryOnly = options?.inMemoryOnly ?? false this.isRebuilding = true try { @@ -3947,22 +3519,15 @@ export class MetadataIndexManager implements MetadataIndexProvider { // here — it's always saved at the end of rebuild via flush(). This ensures // that if rebuild fails partway, the next init() can still discover fields // and trigger another rebuild attempt. - // - // SKIPPED for inMemoryOnly: these are the SHARED storage keys a live - // manager over the same storage may still be reading (see this - // method's JSDoc) — deleting them before the swap is a live-read - // hazard, not a cleanup. - if (!inMemoryOnly) { - prodLog.info('Clearing existing metadata index chunks from storage...') - const existingFields = await this.getPersistedFieldList() + prodLog.info('Clearing existing metadata index chunks from storage...') + const existingFields = await this.getPersistedFieldList() - if (existingFields.length > 0) { - for (const field of existingFields) { - await this.deleteFieldChunks(field) - } - - prodLog.info(`Cleared ${existingFields.length} field indexes from storage`) + if (existingFields.length > 0) { + for (const field of existingFields) { + await this.deleteFieldChunks(field) } + + prodLog.info(`Cleared ${existingFields.length} field indexes from storage`) } // EntityIdMapper is intentionally NOT cleared here. Rebuild re-iterates @@ -4017,7 +3582,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { for (const noun of result.items) { const metadata = metadataBatch.get(noun.id) if (metadata) { - await this.indexStoredRecord(noun.id, metadata, { skipFlush: true, deferWrites: true }) + await this.addToIndex(noun.id, metadata, true, true) } } @@ -4062,7 +3627,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { for (const verb of result.items) { const metadata = verbMetadataBatch.get(verb.id) if (metadata) { - await this.indexStoredRecord(verb.id, metadata, { skipFlush: true, deferWrites: true }) + await this.addToIndex(verb.id, metadata, true, true) } } @@ -4072,16 +3637,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Flush to storage. The column store's flush() handles tail-buffer-to- // segment promotion + manifest persistence. - // - // SKIPPED for inMemoryOnly — see this method's JSDoc: flush() writes - // the shared field-index keys AND flushes the column store's tail - // buffers to shared segment keys, which would race a live manager's - // own flushes over the SAME storage. The caller flushes exactly once, - // after the swap. - if (!inMemoryOnly) { - prodLog.debug('💾 Flushing metadata index to storage...') - await this.flush() - } + prodLog.debug('💾 Flushing metadata index to storage...') + await this.flush() prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`) diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 00790a4a..b8036746 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -540,11 +540,6 @@ function rejectForgedSystemKeys(metadata: Record | undefined, s export function validateAddParams(params: AddParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'add()') - // 'data' is ABSENT only when null/undefined — an empty string ('') is real - // content (a legitimate empty file's first write) and must not be treated - // as missing. Falsy-but-present values (0, false, '') all count as present; - // only the true "nothing was given" case is absent. - const hasData = params.data !== undefined && params.data !== null // MT5 deferred embedding: an explicit vector has nothing to defer, and a // deferral without data has nothing to embed — both are caller bugs that // must refuse with the fix, never be silently reinterpreted. @@ -555,14 +550,14 @@ export function validateAddParams(params: AddParams): void { `the vector is already computed; drop one of the two.` ) } - if (!hasData) { + if (!params.data) { throw new Error( `add(): deferEmbedding requires 'data' (the content the background worker will embed).` ) } } // Universal truth: must have data or vector - if (!hasData && !params.vector) { + if (!params.data && !params.vector) { throw new Error( `Invalid add() parameters: Missing required field 'data'\n` + `\nReceived: ${JSON.stringify({ @@ -588,14 +583,8 @@ export function validateAddParams(params: AddParams): void { ) } - // Validate vector dimensions if provided. A length-0 vector is the - // "unvectored" shape — an explicit `vector: []` (e.g. the VFS root's - // permanently-vectorless creation, see - // VirtualFileSystem.doInitializeRoot()'s zero-norm fix) carries no - // dimension information, exactly like an absent vector or a deferred - // embed's internal stub, so it is exempt from the dimension check rather - // than refused as a "0-dimensional vector". - if (params.vector && params.vector.length > 0) { + // Validate vector dimensions if provided + if (params.vector) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) @@ -608,29 +597,14 @@ export function validateAddParams(params: AddParams): void { */ export function validateUpdateParams(params: UpdateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'update()') - // Same absent-vs-empty distinction as validateAddParams: '' is a real new - // value (e.g. truncating a file to empty content via overwrite), only - // null/undefined means "no new data was given". - const hasData = params.data !== undefined && params.data !== null if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { - if (params.vector && params.vector.length === 0) { - // The nonsensical combination Leg D of the zero-norm/unvector-door law - // refuses: `vector: []` is the SANCTIONED UNVECTOR DOOR — an explicit - // instruction to remove the vector NOW, never "please embed" — so it - // cannot be paired with a request to defer an embed. - throw new Error( - `update(): 'vector: []' (the unvector door) cannot be combined with ` + - `'deferEmbedding: true' — an unvector is an explicit instruction to remove ` + - `the vector now, not a request to defer an embed. Drop one of the two.` - ) - } if (params.vector) { throw new Error( `update(): deferEmbedding cannot be combined with an explicit 'vector' — ` + `the vector is already computed; drop one of the two.` ) } - if (!hasData) { + if (!params.data) { throw new Error( `update(): deferEmbedding requires new 'data' — without a data change there is nothing to re-embed.` ) @@ -640,10 +614,10 @@ export function validateUpdateParams(params: UpdateParams): void { if (!params.id) { throw new Error('id is required for update') } - + // Universal truth: must update something if ( - !hasData && + !params.data && !params.metadata && !params.type && !params.vector && @@ -660,16 +634,8 @@ export function validateUpdateParams(params: UpdateParams): void { throw new Error(`invalid NounType: ${params.type}`) } - // Validate vector dimensions if provided. A length-0 vector is the - // SANCTIONED UNVECTOR DOOR (see brainy.ts update()'s matching comment): an - // explicit `vector: []` — or a real all-zero vector, normalized to `[]` - // upstream by the zero-norm law — carries no dimension information, - // exactly like validateAddParams's identical exemption, so it is exempt - // from the dimension check rather than refused as a "0-dimensional - // vector". (The `deferEmbedding` combination above already refuses - // `vector: []` paired with `deferEmbedding: true` — an empty array is - // truthy, so that guard fires unconditionally on any explicit `vector`.) - if (params.vector && params.vector.length > 0) { + // Validate vector dimensions if provided + if (params.vector) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/src/utils/version.ts b/src/utils/version.ts index 327f923c..d616cee3 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -1,6 +1,6 @@ /** * @module utils/version - * @description Resolves the running `@soulcraftlabs/brainy` package version. Brainy 8.0 + * @description Resolves the running `@soulcraft/brainy` package version. Brainy 8.0 * targets Node-like runtimes only (Node.js, Bun, Deno — all expose `node:fs`), so the * version is read **synchronously** from `package.json` on first call and cached. * @@ -83,27 +83,3 @@ export function getAugmentationVersion(service: string): { augmentation: string; version: getBrainyVersion() } } - -/** - * The API-contract version this build implements — a single integer that two - * engines can compare without probing prototypes. - * - * A MINOR release is ADDITIVE: doors and error codes may be added, never - * removed or narrowed, and the contract integer does not move. A MAJOR release - * is what a REQUIRED door's removal or a behavioural narrowing costs, and it - * bumps this integer. A consumer pinning `brainyContract` in a peer range is - * therefore pinning "what I may call", not "which build I run". - * - * Declared in package.json as `"brainyContract"` so a manifest, a tool, or a - * sibling package can read it without importing the engine, and returned here - * so a running process can state its own. - */ -export const BRAINY_CONTRACT_VERSION = 1 as const - -/** - * @description The API-contract version this build implements. - * @returns The contract integer — see {@link BRAINY_CONTRACT_VERSION}. - */ -export function contractVersion(): number { - return BRAINY_CONTRACT_VERSION -} diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 46c6a12d..ed272109 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -6,13 +6,11 @@ */ import { Readable, Writable } from 'stream' -import { prodLog } from '../utils/logger.js' import crypto from 'crypto' import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' import { Entity, AddParams, RelateParams, FindParams, Relation } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { isZeroNormVector } from '../utils/distance.js' import { PathResolver } from './PathResolver.js' import { mimeDetector } from './MimeTypeDetector.js' import { @@ -67,20 +65,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { private config: Required> & { rootEntityId?: string } private rootEntityId?: string private initialized = false - /** - * The one-time old-root sweep, in flight. See {@link sweepOldRootsIfNeeded}. - */ - private rootSweep?: Promise - /** - * Where the completed old-root sweep is recorded. Engine plumbing under - * `_system/`, like every other marker there — never enumerated as data. - */ - private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json' - /** - * Below this wall, a sweep that removed nothing says nothing — see - * {@link sweepOldRootsIfNeeded}. - */ - private static readonly ROOT_SWEEP_NARRATE_MS = 1_000 private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer features available via augmentation (brain.use('knowledge')) @@ -158,17 +142,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Create or find root entity this.rootEntityId = await this.initializeRoot() - // Clean up old UUID-based roots — ONCE PER STORE, BEHIND THE DOORS. - // This is a migration sweep for roots created before the fixed root id - // existed. It ran on EVERY open, forever: a filtered find over the whole - // store hunting for duplicates that a store has either always had or - // never will. MEASURED on a 14,056-noun / 72,679-verb store: the phase it - // dominates cost 43-53 SECONDS of every open, warm reopens included. - // Now: a durable marker records that the sweep has run, and a store - // carrying it never sweeps again; a store without one sweeps in the - // BACKGROUND (the sweep only removes duplicate roots — nothing serves - // from them — and it was always declared non-critical). - this.rootSweep = this.sweepOldRootsIfNeeded() + // Clean up old UUID-based roots (one-time migration) + await this.cleanupOldRoots() // Initialize projection registry with auto-discovery of built-in projections this.projectionRegistry = new ProjectionRegistry() @@ -256,11 +231,9 @@ export class VirtualFileSystem implements IVirtualFileSystem { private async doInitializeRoot(): Promise { const rootId = VirtualFileSystem.VFS_ROOT_ID - // Try to get existing root by fixed ID (O(1) lookup, not query). - // includeVectors: true — the zero-norm migration below (leg 2) needs to - // inspect the persisted vector to detect the legacy placeholder shape. + // Try to get existing root by fixed ID (O(1) lookup, not query) try { - const existingRoot = await this.brain.get(rootId, { includeVectors: true }) + const existingRoot = await this.brain.get(rootId) if (existingRoot) { // Root exists - verify metadata is correct @@ -277,34 +250,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { }) } - // ZERO-NORM ROOT MIGRATION (one-time): a pre-fix store persisted the - // root with a REAL all-zero placeholder vector — lawful inside - // brainy (cosineDistance treats a zero-norm operand as MAXIMUM - // distance, see src/utils/distance.ts) but a "false attractor" for a - // downstream engine serving squared-euclidean distance, which cannot - // tell an all-zero vector apart from a legitimate origin point (a - // production incident silently darkened 150+ rows in a partner - // engine's index this way). THE LAW: a zero-norm vector is not a - // vector — it never crosses an engine boundary. Detect the legacy - // shape via NORM, not length or dimension (any real all-zero vector - // qualifies, not just the historical 384-dim one), and rewrite it to - // the "unvectored" `[]` shape through the sanctioned migration path - // (Brainy.unvectorNounForRootMigration — see its JSDoc), which keeps - // `getCanonicalCounts().vectors.all` honest and removes the row from - // the vector index. Idempotent: a store already on the new shape - // (vector.length === 0) takes the false branch below on every - // subsequent init() — a permanent no-op, not a one-time flag. - const existingVector = existingRoot.vector ?? [] - if (existingVector.length > 0 && isZeroNormVector(existingVector)) { - const migrated = await this.brain.unvectorNounForRootMigration(rootId) - if (migrated) { - console.log( - 'VFS: migrated root vector from the legacy all-zero placeholder to the ' + - 'unvectored shape (zero-norm vectors never cross an engine boundary)' - ) - } - } - return rootId } } catch (error) { @@ -315,51 +260,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { try { console.log('VFS: Creating root directory (fixed ID: 00000000-0000-0000-0000-000000000000)') - // OPEN-PATH FIX: the VFS root is Brainy's own system-tier plumbing — it - // is hidden from find()/getNounCount()/stats() by default and nothing - // ever runs a semantic search against it — so it needs no REAL - // embedding. Historically this add() always called embed('/'), which - // meant every writer's FIRST-EVER open forced the process-global WASM - // engine to cold-compile its model (measured 90-140s on throttled - // CPUs) before the brain could even finish init(). This branch only - // runs once per store (a previously-opened store already has a root — - // see the migration above for the pre-fix shape — reopening never - // re-adds it), so the fix applies only to brand-new stores. - // - // ZERO-NORM LAW (current shape, superseding the historical all-zero - // placeholder): the root's vector is `[]` — the SAME "unvectored" - // empty-array shape used for a deferred embed's stub and any other - // not-yet-embedded row — never a real all-zero vector. A zero-norm - // vector is lawful inside brainy (`cosineDistance`, see - // src/utils/distance.ts, returns the MAXIMUM distance whenever either - // operand's norm is zero) but is a "false attractor" for a downstream - // engine serving squared-euclidean distance, which cannot tell a real - // all-zero vector apart from a legitimate origin point — it silently - // darkened 150+ rows in a partner engine's index in production. THE - // LAW: a zero-norm vector is not a vector — it never crosses an engine - // boundary. `vector: []` achieves the SAME cold-compile avoidance the - // original placeholder did (`add()`'s dimension-pin and HNSW-insert - // gates both key off `vector.length > 0`, so an empty vector never - // calls embed(), never pins `brain.dimensions`, and never reaches the - // vector index — see brainy.ts add()'s matching comments) while never - // persisting a searchable zero vector for a downstream engine to trip - // over. Deliberately NOT `deferEmbedding: true`: that flag's landing - // path (`kickEmbedWorker()`, called synchronously right after commit — - // see brainy.ts add()/update()) would still force the WASM engine to - // cold-compile within milliseconds of open (just off the awaited path - // instead of never paying it at all) AND would eventually embed the - // root's data for real, which this fix forbids — the root must NEVER - // be embedded, not merely "not yet". - // - // Only the default WASM engine gets this treatment — a plugin- - // registered native 'embeddings' provider has no cold-compile cost and - // may use a different dimension, so it keeps embedding the root for - // real (same as before this fix) rather than leave Brainy's own - // plumbing permanently unvectored on a store where embedding is cheap. - const rootVector = this.brain.usesDefaultWasmEmbedder() - ? ([] as number[]) - : undefined - await this.brain.add({ id: rootId, // Fixed ID - storage ensures uniqueness data: '/', @@ -371,8 +271,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { // public AddParams.visibility union ('public' | 'internal') — this is the single // sanctioned internal setter, hence the cast. visibility: 'system' as 'public' | 'internal', - metadata: this.getRootMetadata(), - ...(rootVector ? { vector: rootVector } : {}) + metadata: this.getRootMetadata() }) return rootId @@ -418,100 +317,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { * * This is a one-time migration helper that can be removed in future versions. */ - /** - * @description Run the old-root sweep at most once per store, in the - * background, and record that it ran. See the call site in {@link init} for - * the measurement that made this necessary. - * @returns A promise that settles when the sweep has finished (or was - * skipped); nothing in the read path awaits it. - */ - private async sweepOldRootsIfNeeded(): Promise { - const store = this.rawObjectStore() - if (store === null) { - // A storage adapter with no raw-object door cannot carry the marker. - // Sweep every open, as before — correctness over cost. - await this.cleanupOldRoots() - return - } - try { - const marker = await store.readRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH) - if (marker !== null && marker !== undefined) return - } catch { - // Unreadable marker: sweep, and rewrite it below. - } - // NARRATION HAS A THRESHOLD, like every other line this engine emits on the - // always-visible channel. On a fresh or small store this sweep finds - // nothing and costs a millisecond, and announcing it — twice — on a - // channel a production log level deliberately CANNOT silence would train - // operators to ignore the one channel that exists to be impossible to - // ignore. It speaks when it has something to say: duplicates removed, or a - // wall long enough that somebody watching a slow first open deserves to - // know what is running. Otherwise it does its work and stays quiet. - const startedAt = Date.now() - const duplicatesRemoved = await this.cleanupOldRoots() - const elapsedMs = Date.now() - startedAt - try { - await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, { - sweptAt: new Date().toISOString(), - durationMs: elapsedMs - }) - if (duplicatesRemoved > 0 || elapsedMs >= VirtualFileSystem.ROOT_SWEEP_NARRATE_MS) { - prodLog.narrate( - `[VFS] one-time old-root sweep complete in ${elapsedMs}ms` + - (duplicatesRemoved > 0 - ? `, ${duplicatesRemoved} pre-fixed-id root(s) removed` - : '') + - ' and recorded — no future open pays for it.' - ) - } - } catch (error) { - // Unrecorded sweep = the next open sweeps again. Conservative, and said - // out loud rather than quietly repeated forever. - prodLog.narrate( - `[VFS] old-root sweep finished in ${Date.now() - startedAt}ms but could NOT be ` + - `recorded (${(error as Error).message}) — the next open will sweep again.` - ) - } - } - - /** - * @description Settle once the background old-root sweep has finished. - * Resolves immediately when the store already carried the marker. Exists so - * tests and operators can observe the sweep instead of racing it; no read - * path waits on it. - * @returns A promise that settles with the sweep. - */ - public async whenRootSweepSettled(): Promise { - await this.rootSweep - } - - /** - * @description The brain's storage adapter, narrowed to the raw-object door - * this migration marker needs. Boundary: `Brainy.storage` is private, and - * this is the same reach-in the engine uses elsewhere for exactly this kind - * of engine-internal artifact. Returns null when the adapter has no - * raw-object door. - */ - private rawObjectStore(): { - readRawObject: (key: string) => Promise - writeRawObject: (key: string, value: unknown) => Promise - } | null { - const storage = (this.brain as unknown as { storage?: Record }).storage - if ( - storage && - typeof storage.readRawObject === 'function' && - typeof storage.writeRawObject === 'function' - ) { - return storage as unknown as { - readRawObject: (key: string) => Promise - writeRawObject: (key: string, value: unknown) => Promise - } - } - return null - } - - private async cleanupOldRoots(): Promise { - let removed = 0 + private async cleanupOldRoots(): Promise { try { // Find any old VFS roots with UUID-based IDs (not our fixed ID) const oldRoots = await this.brain.find({ @@ -533,7 +339,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { for (const duplicate of duplicates) { try { await this.brain.remove(duplicate.id) - removed++ console.log(`VFS: Deleted old root ${duplicate.id.substring(0, 8)}`) } catch (error) { console.warn(`VFS: Failed to delete old root ${duplicate.id}:`, error) @@ -546,7 +351,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Non-critical error - log and continue console.warn('VFS: Cleanup of old roots failed (non-critical):', error) } - return removed } /** @@ -1425,20 +1229,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * @description List a directory's contents. Non-recursive (default) - * returns direct children only, named by basename. `recursive: true` - * lists every descendant at any depth (files and directories), each - * reported as a path RELATIVE TO THE QUERIED DIRECTORY — matching Node's - * `fs.readdir(dir, { recursive: true })` convention — e.g. `'sub'` and - * `'sub/file.txt'` for a nested file. With `withFileTypes: true`, each - * {@link VFSDirent}'s `name` carries that same value (relative when - * recursive, basename otherwise); `path` is always the absolute VFS path - * either way. - * @param path - The directory to list. - * @param options - `recursive`, `withFileTypes`, `filter`, `sort`, - * `offset`/`limit` (pagination applies AFTER filter/sort, over the full - * recursive set when `recursive: true`). - * @throws {VFSError} ENOTDIR when `path` is not a directory. + * Read directory contents */ async readdir(path: string, options?: ReaddirOptions): Promise { await this.ensureInitialized() @@ -1451,12 +1242,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'readdir') } - // Direct children, or every descendant at any depth. gatherDescendants() - // is the same graph-traversal + ONE-batch-fetch path getTreeStructure()/ - // getDescendants() already use — no per-directory storage round trips. - let children = options?.recursive - ? await this.gatherDescendants(entityId, Infinity) - : await this.pathResolver.getChildren(entityId) + // Get children + let children = await this.pathResolver.getChildren(entityId) // Apply filters if (options?.filter) { @@ -1480,29 +1267,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Directory access time updates caused 50-100ms GCS write on EVERY readdir // await this.updateAccessTime(entityId) // ← REMOVED - // The queried directory's own canonical (already-normalized) path — the - // base every recursive entry's relative name is computed against. Using - // the resolved entity's OWN path (rather than the raw `path` argument) - // means no separate normalization step is needed here. - const baseDir = entity.metadata.path - const relativeToBase = (childPath: string): string => { - const prefix = baseDir === '/' ? '/' : `${baseDir}/` - return childPath.startsWith(prefix) ? childPath.slice(prefix.length) : childPath - } - // Return appropriate format if (options?.withFileTypes) { return children.map(child => ({ - name: options?.recursive ? relativeToBase(child.metadata.path) : child.metadata.name, + name: child.metadata.name, path: child.metadata.path, type: child.metadata.vfsType, entityId: child.id } as VFSDirent)) } - return children.map(child => - options?.recursive ? relativeToBase(child.metadata.path) : child.metadata.name - ) + return children.map(child => child.metadata.name) } // ============= Metadata Operations ============= diff --git a/src/vfs/types.ts b/src/vfs/types.ts index 17687dd3..9188476b 100644 --- a/src/vfs/types.ts +++ b/src/vfs/types.ts @@ -133,17 +133,8 @@ export interface VFSStats { * Directory entry (for readdir) */ export interface VFSDirent { - /** - * The entry's basename (e.g. `'file.txt'`) when `readdir()` was called - * WITHOUT `recursive: true`. When `recursive: true` was set, this is - * instead the entry's path RELATIVE TO THE QUERIED DIRECTORY (e.g. - * `'sub/file.txt'` for a nested file) — the same value that would appear - * in the plain string-array form of a recursive `readdir()` call. `path` - * below always carries the absolute VFS path regardless of `recursive`, - * so nothing is lost either way. - */ name: string - path: string // Full (absolute) VFS path — always absolute, recursive or not + path: string // Full path type: 'file' | 'directory' | 'symlink' entityId: string // Underlying entity ID } @@ -249,15 +240,7 @@ export interface ReaddirOptions { withFileTypes?: boolean // Return Dirent objects // VFS-specific options - /** - * List every descendant (files and directories, all depths), not just - * direct children. Entries are reported as paths RELATIVE TO THE QUERIED - * DIRECTORY (Node's `fs.readdir(dir, { recursive: true })` convention) — - * a string-array result contains e.g. `'sub/file.txt'`, and with - * `withFileTypes: true` each `VFSDirent.name` carries that same relative - * path (see {@link VFSDirent}). Default: `false` (direct children only). - */ - recursive?: boolean + recursive?: boolean // Include subdirectories limit?: number // Max results offset?: number // Skip N results cursor?: string // Pagination cursor diff --git a/tests/configs/vitest.integration.config.ts b/tests/configs/vitest.integration.config.ts index af86097d..3d3a3721 100644 --- a/tests/configs/vitest.integration.config.ts +++ b/tests/configs/vitest.integration.config.ts @@ -20,9 +20,6 @@ export default defineConfig({ // Include only integration tests include: [ 'tests/integration/**/*.test.ts', - // The lifecycle biography lane (day-in-the-life scenarios; see - // tests/lifecycle/README.md) runs in the integration gate. - 'tests/lifecycle/**/*.test.ts', 'tests/**/*.integration.test.ts' ], diff --git a/tests/integration/batchImportWithRelations.test.ts b/tests/integration/batchImportWithRelations.test.ts index 4095b51c..7fe8e511 100644 --- a/tests/integration/batchImportWithRelations.test.ts +++ b/tests/integration/batchImportWithRelations.test.ts @@ -15,7 +15,13 @@ describe('Batch Import with Immediate Relations (v5.7.3 Fix)', () => { // Initialize brain brain = new Brainy({ requireSubtype: false, - storage: { type: 'filesystem', path: testDir }, + storage: { + type: 'filesystem', + config: { + baseDir: testDir, + enableCompression: false // Faster tests + } + }, dimensions: 384 }) diff --git a/tests/integration/brainy-core.integration.test.ts b/tests/integration/brainy-core.integration.test.ts index e5b01caf..dd04703e 100644 --- a/tests/integration/brainy-core.integration.test.ts +++ b/tests/integration/brainy-core.integration.test.ts @@ -337,18 +337,12 @@ describe('Brainy 3.0 Core (Integration Tests - Real AI)', () => { describe('Error Handling and Edge Cases', () => { it('should handle invalid inputs gracefully', async () => { - // Empty string is REAL content (e.g. an empty file's first write), not - // a missing field — only null/undefined data (with no vector either) - // is rejected. See src/utils/paramValidation.ts validateAddParams(). + // Empty data is rejected with a clear validation error (8.0 requires a + // non-empty `data` or a `vector` — empty string carries no signal to embed). await expect(brain.add({ data: '', type: 'document' - })).resolves.toBeDefined() - - // Missing BOTH data and vector is still the real "nothing to embed" error. - await expect(brain.add({ - type: 'document' - } as any)).rejects.toThrow(/data/) + })).rejects.toThrow(/data/) // Test with very long text — valid input, resolves to an id. const longText = 'Lorem ipsum '.repeat(10000) diff --git a/tests/integration/canonical-count-ledger.test.ts b/tests/integration/canonical-count-ledger.test.ts index 3d292e97..8d0f46b8 100644 --- a/tests/integration/canonical-count-ledger.test.ts +++ b/tests/integration/canonical-count-ledger.test.ts @@ -17,11 +17,10 @@ * (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is * upgraded from the canonical id tree at open, then persisted. */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' -import * as zlib from 'node:zlib' import { Brainy } from '../../src/index.js' /** Count canonical `/entities///` directories — every tier. */ @@ -181,180 +180,3 @@ describe('canonical count ledger — ALL-visibility scalars, unclamped totals, r expect(ledger.nouns.all).toBe(truth) }) }) - -/** Count `/entities/nouns///vectors.json[.gz]` files holding a non-empty `vector`. */ -function countVectoredNouns(root: string): number { - const base = path.join(root, 'entities', 'nouns') - if (!fs.existsSync(base)) return 0 - let n = 0 - for (const shard of fs.readdirSync(base)) { - const shardDir = path.join(base, shard) - if (!fs.statSync(shardDir).isDirectory()) continue - for (const id of fs.readdirSync(shardDir)) { - const idDir = path.join(shardDir, id) - if (!fs.statSync(idDir).isDirectory()) continue - const plainPath = path.join(idDir, 'vectors.json') - const gzPath = `${plainPath}.gz` - let record: any = null - if (fs.existsSync(plainPath)) { - record = JSON.parse(fs.readFileSync(plainPath, 'utf-8')) - } else if (fs.existsSync(gzPath)) { - record = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf-8')) - } else { - continue - } - if (Array.isArray(record.vector) && record.vector.length > 0) n++ - } - } - return n -} - -describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s coverage denominator)', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - /** Baseline vectored count right after a fresh open() — init() creates a - * hidden system VFS-root noun, but (the zero-norm root cure) it is - * deliberately UNVECTORED (`vector: []`, never a real all-zero - * placeholder — a zero-norm vector never crosses an engine boundary), so - * a brand-new store's `vectors.all` is 0. Tests still assert DELTAS off - * this baseline rather than hardcoding it away, in case that ever - * changes again. */ - let baseline: number - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-')) - brain = await open() - baseline = (await brain.storage.getCanonicalCounts()).vectors.all - expect(baseline).toBe(0) // the unvectored VFS root contributes nothing - }) - afterEach(async () => { - vi.restoreAllMocks() - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('an explicit-vector add counts immediately; the ledger matches the on-disk vectors.json content', async () => { - await brain.add({ data: 'a', type: 'document', vector: Array(384).fill(0).map((_, i) => Math.sin(i)) }) - await brain.add({ data: 'b', type: 'document' }) // embedded (non-deferred) — also a real vector - await brain.flush() - - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 2) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - expect(ledger.suspect).toBe(false) - }) - - it('a deferred-embed add does NOT count until its embed LANDS', async () => { - // Hold the background worker's embed call open under manual control — a - // deterministic embedder is fast enough that the landing could otherwise - // race ahead of the "still unlanded" assertion below. - let resolveEmbed: ((v: number[]) => void) | undefined - vi.spyOn(brain, 'embed').mockImplementation( - () => new Promise((resolve) => { resolveEmbed = resolve }) - ) - - const id = await brain.add({ data: 'deferred content', type: 'document', deferEmbedding: true }) - await brain.flush() - - // Landed nothing yet — the ledger must not count the stub. - let ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - - // Release the held embed, then cross the barrier: the vector lands - // (system:embed-landing). - resolveEmbed!(Array(384).fill(0).map((_, i) => Math.cos(i))) - await brain.awaitPendingEmbeds() - const landed = await brain.get(id, { includeVectors: true }) - expect((landed!.vector as number[]).length).toBeGreaterThan(0) - - ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - expect(ledger.suspect).toBe(false) - }) - - it('a proven delete of a vectored noun decrements; a non-vectored (unlanded) delete does not', async () => { - const vectoredId = await brain.add({ data: 'v', type: 'document' }) // real embed, unmocked - // Block the embed worker AFTER the real add above — a deterministic - // embedder is fast enough that the deferred noun below could otherwise - // land before this test observes its "still unlanded" state. - vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) - const deferredId = await brain.add({ data: 'd', type: 'document', deferEmbedding: true }) - await brain.flush() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(baseline + 1) - - await brain.remove(vectoredId) - await brain.flush() - let ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline) - expect(ledger.suspect).toBe(false) - - await brain.remove(deferredId) // never had a real vector — no decrement, still unsuspect - await brain.flush() - ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline) - expect(ledger.suspect).toBe(false) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - }) - - it('the recount corrects a tampered vectors.all scalar, surviving reopen', async () => { - await brain.add({ data: 'real 1', type: 'document' }) - await brain.add({ data: 'real 2', type: 'document' }) - await brain.flush() - const truth = countVectoredNouns(dir) - expect(truth).toBe(baseline + 2) - - ;(brain.storage as any).totalVectoredNounCount = truth + 40 - await (brain.storage as any).persistCounts() - await brain.close() - brain = await open() - - // The lie survives reopen (never clamped). - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth + 40) - - await brain.repairIndex() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth) - - await brain.close() - brain = await open() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth) - }) - - it('a legacy counts.json without totalVectoredNounCount is derived once from vectors.json content and persisted', async () => { - await brain.add({ data: 'one', type: 'document' }) // real embed, unmocked - // Block the embed worker AFTER the real add above — a deterministic - // embedder is fast enough that the deferred noun below could otherwise - // land before close(), which would inflate this test's expected count. - vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) - await brain.add({ data: 'two deferred', type: 'document', deferEmbedding: true }) - await brain.flush() - await brain.close() - - const countsPath = path.join(dir, '_system', 'counts.json') - const raw = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) - expect(typeof raw.totalVectoredNounCount).toBe('number') - delete raw.totalVectoredNounCount - fs.writeFileSync(countsPath, JSON.stringify(raw, null, 2)) - - brain = await open() - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) // just the one non-deferred noun — the root is unvectored - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) - expect(persisted.totalVectoredNounCount).toBe(baseline + 1) - }) -}) diff --git a/tests/integration/cold-graph-connected-8.0.test.ts b/tests/integration/cold-graph-connected-8.0.test.ts index 7ee725f5..71ae345d 100644 --- a/tests/integration/cold-graph-connected-8.0.test.ts +++ b/tests/integration/cold-graph-connected-8.0.test.ts @@ -1,29 +1,29 @@ /** * @module tests/integration/cold-graph-connected-8.0 * @description BRAINY-COLD-GRAPH-CONNECTED (8.0) — regression coverage for the silent-empty - * graph-traversal bug, gated on the honest readiness signal: a sync `graphIndex.isReady()` - * that is true ONLY when the source→target EDGES are loaded (NOT the membership/manifest count). + * graph-traversal bug, gated on the converged 8.0 contract: a sync `graphIndex.isReady()` that + * is true ONLY when the source→target EDGES are loaded (NOT the membership/manifest count). * - * On the FIRST `find({ connected })` after a cold process start, a native graph adjacency can - * reload its relationship COUNT (so `size() > 0`) but NOT its edges — so `getNeighbors()` returns - * `[]` for EVERY source and brainy would serve that `[]` as if the anchor were genuinely edgeless. + * On the FIRST `find({ connected })` after a cold process start of a LARGE brain (≥10k nouns, + * which skips the eager index rebuild), a native graph adjacency can reload its relationship + * COUNT (so `size() > 0`) but NOT its edges — so `getNeighbors()` returns `[]` for EVERY source + * and brainy would serve that `[]` as if the anchor were genuinely edgeless. * - * RE-POINTED to the health-gate law: `verifyGraphAdjacencyLive` NEVER rebuilds and NEVER walks the - * store from a read — a read-path rebuild is exactly the dark-rebuild failure mode the law retires - * (open() alone owns building). The guard now: - * - `isReady() === false` → THROWS {@link GraphIndexNotReadyError} immediately — no rebuild attempt; + * The 8.0 guard (`verifyGraphAdjacencyLive`) prefers the honest `isReady()` signal: + * - `isReady() === false` → hydrate the id-mapper, rebuild from storage, re-check; a still-false + * `isReady()` throws {@link GraphIndexNotReadyError} instead of returning `[]` ('rebuilt' when + * the rebuild heals it); * - a genuinely edgeless anchor with `isReady() === true` verifies 'live' and the empty result - * stands — no spurious throw; - * - a provider WITHOUT `isReady()` falls back to the shipped known-edge-sample probe, which is - * now READ-ONLY: it refuses loudly (throws) rather than self-healing via rebuild. + * stands — no spurious rebuild, no throw; + * - a provider WITHOUT `isReady()` falls back to the shipped 7.x known-edge-sample probe. * * These exercise REAL `find({ connected })` against an in-memory brain whose graph index is - * instrumented with a test-double `isReady()` (and, for the fallback case, an always-empty + * instrumented with a test-double `isReady()` (and, for the fallback case, an empty-then-healed * `getNeighbors`). Only the readiness/edge surface is wrapped; the underlying real adjacency - * (built by `relate()`) is what a healthy provider actually serves. + * (built by `relate()`) is unmasked once a rebuild "heals" it. */ -import { describe, it, expect, afterEach, vi } from 'vitest' +import { describe, it, expect, afterEach } from 'vitest' import { Brainy } from '../../src/index.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' import { GraphIndexNotReadyError } from '../../src/errors/brainyError.js' @@ -63,17 +63,17 @@ async function buildBrain( } /** - * Instrument the brain's real graph index with a test-double `isReady()` (the honest-readiness - * contract) plus an edge surface that goes empty while NOT ready. `getNeighbors` returns `[]` - * while `!ready` (modelling the cold-unloaded adjacency) and delegates to the REAL index once - * `ready` flips true (used only by the "healthy" control cases — the guard itself never flips - * this anymore, since it never rebuilds). `rebuild` is counted so tests can assert it is NEVER - * called by a read. + * Instrument the brain's real graph index with a test-double `isReady()` (the 8.0 contract) plus + * an edge surface that goes empty while NOT ready. `getNeighbors` returns `[]` while `!ready` + * (modelling the cold-unloaded adjacency) and delegates to the REAL index once a rebuild flips + * `ready` on. `rebuild` is counted; it heals (`ready = true`) only when `healsOnRebuild` is set. + * Pass `failFirstRebuild` to make the FIRST rebuild throw a transient error (without healing) so + * the empty-result re-collect path in executeGraphSearch is exercised. */ function instrumentIsReady( brain: any, - opts: { ready: boolean } -): { rebuildCalls: number; ready: boolean } { + opts: { ready: boolean; healsOnRebuild: boolean; failFirstRebuild?: boolean } +): { rebuildCalls: number } { const gi = brain.graphIndex const origGetNeighbors = gi.getNeighbors.bind(gi) const state = { ready: opts.ready, rebuildCalls: 0 } @@ -85,6 +85,10 @@ function instrumentIsReady( gi.rebuild = async (): Promise => { state.rebuildCalls++ + if (opts.failFirstRebuild && state.rebuildCalls === 1) { + throw new Error('transient rebuild hiccup') + } + if (opts.healsOnRebuild) state.ready = true // unmask the real (already-populated) adjacency } return state @@ -92,12 +96,12 @@ function instrumentIsReady( /** * Fallback instrumentation — a provider WITHOUT `isReady()` (older cortex / JS baseline). Wraps - * `getNeighbors` to always return `[]` while `broken`. This is the shipped known-edge-sample - * probe path — now READ-ONLY: it refuses loudly rather than self-healing. + * `getNeighbors` to return `[]` while `broken` and delegates to the REAL index once a rebuild + * heals it. This is the shipped 7.x known-edge-sample probe path on 8.0. */ function instrumentNoIsReady( brain: any, - opts: { broken: boolean } + opts: { broken: boolean; healsOnRebuild: boolean } ): { rebuildCalls: number } { const gi = brain.graphIndex // Ensure the provider does NOT expose isReady() — the default JS provider doesn't. @@ -110,12 +114,13 @@ function instrumentNoIsReady( gi.rebuild = async (): Promise => { state.rebuildCalls++ + if (opts.healsOnRebuild) state.broken = false } return state } -describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a silent [], never rebuilds from a read', () => { +describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a silent []', () => { let brains: any[] = [] afterEach(async () => { for (const b of brains) { @@ -126,37 +131,35 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si } } brains = [] - vi.restoreAllMocks() }) - it('(a) isReady() false → THROWS GraphIndexNotReadyError immediately, no rebuild attempt', async () => { + it('(a) isReady() false → rebuild heals it true → find({ connected }) returns correct N (rebuilt)', async () => { + const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true }) + + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + + expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected not-ready + healed it + const ids = results.map((r: any) => r.id).sort() + expect(ids).toEqual(targetIds.sort()) // B, C, D — the real edges, served after the heal + }) + + it('(b) isReady() stays false after rebuild → throws GraphIndexNotReadyError (NOT a silent [])', async () => { const { brain, anchorId } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: false }) + instrumentIsReady(brain, { ready: false, healsOnRebuild: false }) // rebuild never makes it ready await expect( brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) ).rejects.toBeInstanceOf(GraphIndexNotReadyError) - - expect(state.rebuildCalls).toBe(0) // a read never rebuilds — it refuses loudly instead - }) - - it('(b) isReady() stays false → throws GraphIndexNotReadyError (NOT a silent [])', async () => { - const { brain, anchorId } = await buildBrain({ anchorEdges: true }) - brains.push(brain) - const state = instrumentIsReady(brain, { ready: false }) - - await expect( - brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - ).rejects.toBeInstanceOf(GraphIndexNotReadyError) - expect(state.rebuildCalls).toBe(0) }) it('(c) edgeless anchor + isReady() true → returns [] with NO rebuild and NO throw', async () => { // The anchor has no edges, but E -> F does — the adjacency is genuinely loaded (ready). const { brain, anchorId } = await buildBrain({ anchorEdges: false }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: true }) + const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) @@ -167,7 +170,7 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si it('(d) healthy isReady() true → correct results, NO rebuild', async () => { const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: true }) + const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) @@ -176,30 +179,30 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si expect(ids).toEqual(targetIds.sort()) }) - it('(e) provider WITHOUT isReady() → the known-edge-sample probe REFUSES LOUDLY (never self-heals)', async () => { - const { brain, anchorId } = await buildBrain({ anchorEdges: true }) + it('(e) provider WITHOUT isReady() → falls back to the known-edge-sample probe (self-heals)', async () => { + const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentNoIsReady(brain, { broken: true }) + const state = instrumentNoIsReady(brain, { broken: true, healsOnRebuild: true }) - await expect( - brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - expect(state.rebuildCalls).toBe(0) // the fallback probe is READ-ONLY — it never calls rebuild() + expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected the empty adjacency + healed it + const ids = results.map((r: any) => r.id).sort() + expect(ids).toEqual(targetIds.sort()) // B, C, D — served after the heal }) - it('(f) an empty connectedIds set re-verifies against a not-serving adjacency and throws, rather than serving [] as truth', async () => { - // executeGraphSearch's cold-load guard (connectedIds.size === 0 → re-verify) used to - // interpret a healed rebuild as "re-collect and serve." That rebuild-and-heal path is - // retired: the re-verify now either confirms a genuinely edgeless anchor ('live', case (c)) - // or — as here — discovers the adjacency itself is not serving, and throws. - const { brain, anchorId } = await buildBrain({ anchorEdges: true }) + it('(f) executeGraphSearch re-collect: a transient first rebuild leaves connectedIds empty; the empty-result guard then heals + re-collects', async () => { + // First verify (inside neighbors()) hits a transient rebuild failure → returns 'live' without + // healing, so getNeighbors stays empty and connectedIds is empty. The empty connectedIds set + // then drives executeGraphSearch's own verify, whose rebuild now heals → 'rebuilt' → re-collect. + const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: false }) + const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true, failFirstRebuild: true }) - await expect( - brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - ).rejects.toBeInstanceOf(GraphIndexNotReadyError) - expect(state.rebuildCalls).toBe(0) + const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + + expect(state.rebuildCalls).toBeGreaterThanOrEqual(2) // first transient, second heals + const ids = results.map((r: any) => r.id).sort() + expect(ids).toEqual(targetIds.sort()) // re-collected after the heal }) }) diff --git a/tests/integration/count-ledger-identity-record.test.ts b/tests/integration/count-ledger-identity-record.test.ts deleted file mode 100644 index 1066213a..00000000 --- a/tests/integration/count-ledger-identity-record.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -/** - * @module tests/integration/count-ledger-identity-record - * @description THE COUNT LEDGER COUNTS RECORDS, NOT DIRECTORIES — and heals - * itself when it was derived the other way. - * - * Measured on a real store: the ALL-visibility ledger read 14,231 nouns - * against 14,056 identity records, and 72,729 verbs against 72,679 — exactly - * that store's 25 noun and 50 verb SCAR directories (empty `/` containers - * left by a pre-8.3.1 partial delete). Two copies of the SAME archive derived - * different numbers, because each had been persisted at a different moment - * under the old container rule. A downstream index heal subtracted against - * those denominators and reported remaining work that did not exist. - * - * The membership predicate is the IDENTITY RECORD (the metadata content leg). - * The scan already applies it; what is pinned here is that a ledger persisted - * under the OLD rule does not go on lying — it is corrected in the background, - * without blocking the open, and two copies of one archive agree. - */ - -import { describe, it, expect, afterEach } from 'vitest' -import { - mkdtempSync, - mkdirSync, - rmSync, - writeFileSync, - readFileSync, - cpSync, - existsSync -} from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { FileSystemStorage as FileSystemStorageClass } from '../../src/storage/adapters/fileSystemStorage.js' -import type { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' - -const NOUN_COUNT = 6 -const NOUN_SCARS = 3 -const VERB_SCARS = 2 -/** A REAL two-hex shard — the scan skips any directory that is not one. */ -const SCAR_SHARD = 'ab' - -function makeTempDir(): string { - return mkdtempSync(join(tmpdir(), 'brainy-count-ledger-')) -} - -/** The FileSystemStorage behind a brain. */ -function storageOf(brain: Brainy): FileSystemStorage { - return (brain as unknown as { storage: FileSystemStorage }).storage -} - -/** - * Add `count` empty `/` container directories under - * `entities///` — scars, exactly as a partial delete leaves them. - */ -function addScarContainers(dir: string, kind: 'nouns' | 'verbs', count: number): void { - for (let i = 0; i < count; i++) { - const id = `${SCAR_SHARD}5ca4000-0000-0000-0000-00000000000${i}` - mkdirSync(join(dir, 'entities', kind, SCAR_SHARD, id), { recursive: true }) - } -} - -/** Add one GHOST container: a `vectors.json` leg with no identity record. */ -function addGhostContainer(dir: string): void { - const id = `${SCAR_SHARD}9405700-0000-0000-0000-000000000000` - const idDir = join(dir, 'entities', 'nouns', SCAR_SHARD, id) - mkdirSync(idDir, { recursive: true }) - writeFileSync(join(idDir, 'vectors.json'), JSON.stringify({ id, vector: [0.1, 0.2] })) -} - -/** - * Rewrite counts.json into the LEGACY shape: ALL scalars inflated by the - * containers, and no `allCountsDerivedBy` stamp — exactly what a store carried - * when it was last written by a build that counted directories. - */ -function writeLegacyCountsLedger(dir: string, inflateNouns: number, inflateVerbs: number): void { - const file = join(dir, '_system', 'counts.json') - const counts = JSON.parse(readFileSync(file, 'utf-8')) - counts.totalNounCountAll = (counts.totalNounCountAll ?? 0) + inflateNouns - counts.totalVerbCountAll = (counts.totalVerbCountAll ?? 0) + inflateVerbs - delete counts.allCountsDerivedBy - delete counts.allCountsSuspect - writeFileSync(file, JSON.stringify(counts, null, 2)) -} - -/** - * Seed a store and return the HONEST ledger it holds when freshly written — - * the baseline the correction must return to. Read from the engine rather than - * hardcoded: an open creates its own rows (the VFS root), and a pin that - * asserts a literal would be pinning that incidental fact instead of the rule. - */ -async function seedStore(dir: string): Promise<{ nouns: number; verbs: number }> { - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const ids: string[] = [] - for (let i = 0; i < NOUN_COUNT; i++) { - ids.push(await brain.add({ data: `entity number ${i}`, type: NounType.Concept })) - } - await brain.relate({ from: ids[0], to: ids[1], type: 'relatedTo' } as never) - await brain.relate({ from: ids[1], to: ids[2], type: 'relatedTo' } as never) - await brain.flush() - const ledger = await storageOf(brain).getCanonicalCounts() - const baseline = { nouns: ledger.nouns.all, verbs: ledger.verbs.all } - await brain.close() - return baseline -} - -/** - * Make the ledger walk take `ms` so a test can observe the open completing - * WITHOUT it. Patches the prototype before any brain is constructed; returns - * the restore function. - */ -function slowTheLedgerWalk(ms: number): () => void { - const proto = ( - FileSystemStorageClass as unknown as { - prototype: Record Promise> - } - ).prototype - const real = proto.scanCanonicalEntities - proto.scanCanonicalEntities = async function slow(this: unknown, ...args: unknown[]) { - await new Promise((r) => setTimeout(r, ms)) - return real.apply(this, args) - } - return () => { proto.scanCanonicalEntities = real } -} - -describe('the canonical count ledger', () => { - const dirs: string[] = [] - - afterEach(() => { - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - }) - - function trackDir(): string { - const dir = makeTempDir() - dirs.push(dir) - return dir - } - - it('corrects a legacy container-rule ledger in the background, counting identity records', async () => { - const dir = trackDir() - const baseline = await seedStore(dir) - - // Scars and a ghost: containers with no identity record. - addScarContainers(dir, 'nouns', NOUN_SCARS) - addScarContainers(dir, 'verbs', VERB_SCARS) - addGhostContainer(dir) - // The ledger as the old rule left it: every container counted. - writeLegacyCountsLedger(dir, NOUN_SCARS + 1, VERB_SCARS) - - const restore = slowTheLedgerWalk(1_500) - let brain: Brainy - try { - const openStarted = Date.now() - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const openMs = Date.now() - openStarted - const storage = storageOf(brain) - - // THE OPEN DID NOT WAIT. Two walks of 1.5s each would have added 3s. - expect(openMs).toBeLessThan(2_500) - // And while it runs, the scalars say so instead of being subtracted against. - const atOpen = await storage.getCanonicalCounts() - expect(atOpen.suspect).toBe(true) - expect(atOpen.nouns.all).toBe(baseline.nouns + NOUN_SCARS + 1) - - await storage.whenCountLedgerSettled() - } finally { - restore() - } - const storage = storageOf(brain!) - - const healed = await storage.getCanonicalCounts() - expect(healed.nouns.all).toBe(baseline.nouns) - expect(healed.verbs.all).toBe(baseline.verbs) - expect(healed.suspect).toBe(false) - - // And it is PERSISTED with the honest stamp — the correction survives a - // reopen instead of being re-derived (or re-lost) every time. - await brain!.close() - const persisted = JSON.parse(readFileSync(join(dir, '_system', 'counts.json'), 'utf-8')) - expect(persisted.totalNounCountAll).toBe(baseline.nouns) - expect(persisted.totalVerbCountAll).toBe(baseline.verbs) - expect(persisted.allCountsDerivedBy).toBe('identity-record') - - const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await reopened.init() - const afterReopen = await storageOf(reopened).getCanonicalCounts() - expect(afterReopen.nouns.all).toBe(baseline.nouns) - expect(afterReopen.suspect).toBe(false) - await reopened.close() - }, 180_000) - - it('derives the same number from two copies of one archive', async () => { - const source = trackDir() - const baseline = await seedStore(source) - addScarContainers(source, 'nouns', NOUN_SCARS) - addGhostContainer(source) - - // Two copies of the SAME bytes, each carrying a DIFFERENT legacy ledger — - // the situation that made one archive report 14,231 and its twin 14,081. - const copyA = trackDir() - const copyB = trackDir() - cpSync(source, copyA, { recursive: true }) - cpSync(source, copyB, { recursive: true }) - writeLegacyCountsLedger(copyA, NOUN_SCARS + 1, 0) - writeLegacyCountsLedger(copyB, 1, 0) - - const derived: number[] = [] - for (const dir of [copyA, copyB]) { - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const storage = storageOf(brain) - await storage.whenCountLedgerSettled() - derived.push((await storage.getCanonicalCounts()).nouns.all) - await brain.close() - } - expect(derived[0]).toBe(derived[1]) - expect(derived[0]).toBe(baseline.nouns) - }, 180_000) - - it('writes counts.json atomically — no reader ever sees it empty', async () => { - const dir = trackDir() - await seedStore(dir) - const file = join(dir, '_system', 'counts.json') - expect(existsSync(file)).toBe(true) - - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const storage = storageOf(brain) - - // Watch the ledger across many persists. A truncating write leaves a - // window in which the file parses as nothing; a temp+rename never does. - let sawUnparseable = 0 - const watcher = setInterval(() => { - try { - JSON.parse(readFileSync(file, 'utf-8')) - } catch { - sawUnparseable++ - } - }, 1) - for (let i = 0; i < 40; i++) { - await (storage as unknown as { persistCounts: () => Promise }).persistCounts() - } - clearInterval(watcher) - await brain.close() - expect(sawUnparseable).toBe(0) - }, 180_000) -}) diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts index 23cc0a15..deefc5e6 100644 --- a/tests/integration/entity-tree-stamp.test.ts +++ b/tests/integration/entity-tree-stamp.test.ts @@ -57,11 +57,7 @@ describe('entity-tree family stamp', () => { const invariants = (stamp.members as any).invariants expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the - // counter labelled the stamp with a generation a write in flight had merely - // claimed, so every crash inside a write window produced a spurious verdict - // at the next open (see the torn-tail pins below). - expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration()) + expect(stamp.sourceGeneration).toBe(brain.generation()) expect(stamp.generation).toBeGreaterThanOrEqual(1) }) @@ -116,96 +112,6 @@ describe('entity-tree family stamp', () => { expect(stillIncoherent).toEqual([]) }) - /** - * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's - * committed watermark — the durable shape a torn generation-log tail leaves - * behind (the stamp's fsync outlived the tail's). Fabricated rather than - * crash-produced so the pin is deterministic; the seeded-SIGKILL lane - * (`scripts/crash-consistency.mjs` in the engine repo) produces the same - * shape from a real abrupt termination. - */ - const fabricateTear = (ahead: number): FamilyStamp => { - const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`) - const zlib = require('node:zlib') - const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp - const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead } - fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn))) - return torn - } - - it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => { - for (let i = 0; i < 3; i++) - await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } }) - await brain.close() - const torn = fabricateTear(5) - - const warn = vi.spyOn(prodLog, 'warn') - const startedAt = Date.now() - brain = await open() - const openMs = Date.now() - startedAt - - const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) - expect(tearLines.length).toBe(1) - const said = String(tearLines[0][0]) - // Narrated PRECISELY: both generations, the file, and the named cure. - expect(said).toContain(`source generation ${torn.sourceGeneration}`) - expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`) - expect(said).toContain(ENTITY_TREE_STAMP_PATH) - expect(said).toContain('DEMOTED') - expect(said).toMatch(/repairIndex\(\)/) - // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear - // cannot turn an open into the 8-minute spin this class was reported as. - expect(openMs).toBeLessThan(30_000) - - // The store SERVES — a tear in a stamp never locks an owner out of the - // canonical tree the stamp merely describes. - expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3) - - // The demotion CONVERGED: the stamp now names committed truth, and the - // next open is quiet. A verdict that re-narrates every open is a wait - // wearing a different hat. - const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp - expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration()) - await brain.close() - const warn2 = vi.spyOn(prodLog, 'warn') - brain = await open() - expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([]) - }) - - it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => { - await brain.add({ data: 'ro', type: 'document', metadata: {} }) - await brain.close() - const torn = fabricateTear(3) - - const warn = vi.spyOn(prodLog, 'warn') - const reader: any = await Brainy.openReadOnly({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) - expect(tearLines.length).toBe(1) - const said = String(tearLines[0][0]) - expect(said).toContain('READ-ONLY') - expect(said).toContain('UNVERIFIED') - expect(said).toMatch(/repairIndex\(\)/) - await reader.close() - - // A reader never rewrites the store: read the bytes back off disk (not - // through a writer open, which would demote them) — the torn stamp is - // exactly as it was found. - const onDisk = JSON.parse( - require('node:zlib') - .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) - .toString('utf-8') - ) as FamilyStamp - expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration) - expect(onDisk.generation).toBe(torn.generation) - - brain = await open() - }) - it('the one verifier handles both member modes', () => { const rollup: FamilyStamp = { family: 'x', @@ -221,13 +127,7 @@ describe('entity-tree family stamp', () => { stampSource: 5, head: 9 }) - // AHEAD is its own class — a torn generation-log tail, never folded in - // with `incoherent`: the two have opposite cures (recount vs demote). - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({ - state: 'torn', - stampSource: 5, - head: 3 - }) + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) const enumerated: FamilyStamp = { diff --git a/tests/integration/enumeration-population-law.test.ts b/tests/integration/enumeration-population-law.test.ts deleted file mode 100644 index eaab7432..00000000 --- a/tests/integration/enumeration-population-law.test.ts +++ /dev/null @@ -1,333 +0,0 @@ -/** - * @module tests/integration/enumeration-population-law - * @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk - * and the canonical ALL scalar must agree on the population — a row's - * IDENTITY RECORD (metadata.json) is what defines membership; the vector leg - * is optional data, never a gate on visibility. Before this fix, the walk - * (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination) - * enumerated by keying on the VECTOR leg (`vectors.json`), so a row with - * metadata and no vector file was counted by the ledger (already - * metadata.json-keyed — see `rebuildTypeCounts`) but never yielded by the - * walk: a permanent "counted but invisible" phantom for any downstream - * consumer (a health-coverage row, an index-fill walk) that iterates the walk - * to account for the ledger's total. - * - * Two legs are pinned here: - * (a)/(b) LEG 1 — the walk re-keys on metadata.json. A fold-born - * metadata-only row (the exact shape `GenerationStore.replayFact` can - * leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept) - * must be YIELDED, hydrated with the sanctioned unvectored shape - * (`vector: []`) — not merely counted. - * - * For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are - * HNSWVerb's structural core and live ONLY in the vector leg (never in - * metadata — see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which - * does not include them). A metadata-only verb row therefore cannot be - * safely reconstructed without FABRICATING an edge's endpoints — which - * would silently create a phantom relationship, strictly worse than the - * original defect. The walk recovers the row when its metadata happens - * to carry `sourceId`/`targetId` (a defensive, forward-compatible - * fallback — never true for a CURRENT production write, but not - * disallowed either); otherwise it counts the row (ledger, unchanged) - * but loudly skips yielding it, logging the gap instead of hiding it. - * Closing this fully requires persisting `sourceId`/`targetId` in verb - * metadata — a schema change out of this task's scope; see the session - * report for the explicit call-out. - * - * (c)/(d) LEG 2 — the recovery fold's preserve-if-absent contract, exercised - * directly against `GenerationStore`/`FactLog` (below the `Brainy` API): - * a metadata-only after-image replayed over an already-vectored row must - * PRESERVE the existing vector leg (never delete it); a genuine tombstone - * (both legs absent) still removes both legs. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { randomUUID } from 'node:crypto' -import { Brainy } from '../../src/index.js' -import { GenerationStore } from '../../src/db/generationStore.js' -import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' -import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js' -import type { CommitFact } from '../../src/db/factLog.js' - -describe('enumeration population law — LEG 1 (identity-keyed walk)', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-')) - brain = await open() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => { - // Ordinary, fully-vectored background population. - await brain.add({ data: 'one', type: 'document' }) - await brain.add({ data: 'two', type: 'document' }) - await brain.flush() - - // THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no - // vector ever existed for it. Written through the raw primitive directly, - // exactly as `GenerationStore.replayFact` (the recovery fold) applies a - // replayed after-image whose vector leg came back null. - const freshId = randomUUID() - await brain.storage.writeNounRaw(freshId, { - metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, - vector: null - }) - - // writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the - // sanctioned recount brings the ledger scalar to ground truth. This walk - // was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so - // the recount's answer does not depend on today's change. - await brain.repairIndex() - - const ledger = await brain.storage.getCanonicalCounts() - const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } }) - - expect(walk.items.length).toBe(ledger.nouns.all) - expect(walk.totalCount).toBe(ledger.nouns.all) - - const yielded = walk.items.find((n: any) => n.id === freshId) - expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined() - expect(yielded.vector).toEqual([]) - }) - - it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => { - await brain.add({ data: 'one', type: 'document' }) - await brain.flush() - - const freshId = randomUUID() - await brain.storage.writeNounRaw(freshId, { - metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, - vector: null - }) - await brain.repairIndex() - - const ledger = await brain.storage.getCanonicalCounts() - const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 }) - expect(page.ids.length).toBe(ledger.nouns.all) - expect(page.ids).toContain(freshId) - }) - - it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => { - const a = await brain.add({ data: 'a', type: 'document' }) - const b = await brain.add({ data: 'b', type: 'document' }) - await brain.relate({ from: a, to: b, type: 'relatedTo' }) - await brain.flush() - - // Case 1 — the REALISTIC production shape: metadata carries the verb - // type (a reserved field, kept for backward compat) but never - // sourceId/targetId — those are HNSWVerb's structural core and live - // ONLY in the vector leg. The walk cannot safely fabricate them (an - // empty-string endpoint would silently create a phantom edge), so this - // row is counted by the ledger but not yielded — a documented, - // loudly-logged gap, not a silent one. - const gapId = randomUUID() - await brain.storage.writeVerbRaw(gapId, { - metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 }, - vector: null - }) - - // Case 2 — endpoints ARE recoverable from metadata (never true for a - // current production write; modeled here as what a repair tool or a - // future schema could supply): the walk reconstructs and yields it. - const recoveredId = randomUUID() - await brain.storage.writeVerbRaw(recoveredId, { - metadata: { - verb: 'relatedTo', - sourceId: a, - targetId: b, - createdAt: Date.now(), - updatedAt: Date.now(), - weight: 1 - }, - vector: null - }) - - await brain.repairIndex() - const ledger = await brain.storage.getCanonicalCounts() - const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } }) - - // The ledger counts every identity record — the real edge plus both - // synthetic metadata-only rows — unaffected by whether the walk can - // safely hydrate them. - expect(ledger.verbs.all).toBe(3) - - const recovered = walk.items.find((v: any) => v.id === recoveredId) - expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined() - expect(recovered.sourceId).toBe(a) - expect(recovered.targetId).toBe(b) - expect(recovered.vector).toEqual([]) - - // The documented gap: counted, not yielded — this is the one corner of - // the population law this task does NOT close (see the session report). - const gapped = walk.items.find((v: any) => v.id === gapId) - expect(gapped).toBeUndefined() - expect(walk.items.length).toBeLessThan(ledger.verbs.all) - }) -}) - -describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => { - /** A GenerationStore whose brain has already flipped to log authority — the - * precondition for `replayFact` (the recovery fold) to run at open(). */ - async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> { - const storage = new MemoryStorage() - await storage.init() - await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' }) - const store = new GenerationStore(storage) - await store.open() - return { storage, store } - } - - it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => { - const { storage, store } = await openLogAuthorityStore() - const id = randomUUID() - const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 } - - // Generation 1 — a real, honest commit: both legs land together. - await store.commitTransaction({ - touched: { nouns: [id], verbs: [] }, - execute: async () => { - await storage.writeNounRaw(id, { - metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, - vector: vectorRecord - }) - } - }) - const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all - - // THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction, - // whose honest read-after-write could never produce this on its own): - // metadata changed, vector leg null, while the row is STILL vectored on - // disk. This is exactly the shape the recovery fold must tolerate — - // modeling the confirmed production defect at the replay boundary. - const factLog = store.getFactLog()! - const anomalousFact: CommitFact = { - generation: 2, - timestamp: Date.now(), - ops: [ - { - kind: 'noun', - id, - record: { - metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 }, - vector: null - } - } - ] - } - await factLog.append(anomalousFact) - await factLog.sync() - - // Reopen — a fresh GenerationStore over the SAME storage. Generation 2's - // fact sits above the (still generation-1) manifest, so it replays - // through the recovery fold — `replayFact`'s own call site. - const store2 = new GenerationStore(storage) - await store2.open() - - const after = await storage.readNounRaw(id) - expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() - expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3]) - expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply - - // writeNounRaw bypasses ledger bookkeeping either way (by design — see - // its JSDoc), so this scalar is unaffected by the replay regardless of - // outcome; asserted for completeness against the task's exact wording. - const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all - expect(afterVectoredCount).toBe(beforeVectoredCount) - }) - - it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => { - const { storage, store } = await openLogAuthorityStore() - const id = randomUUID() - const sourceId = randomUUID() - const targetId = randomUUID() - const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId } - - await store.commitTransaction({ - touched: { nouns: [], verbs: [id] }, - execute: async () => { - await storage.writeVerbRaw(id, { - metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 }, - vector: vectorRecord - }) - } - }) - - const factLog = store.getFactLog()! - const anomalousFact: CommitFact = { - generation: 2, - timestamp: Date.now(), - ops: [ - { - kind: 'verb', - id, - record: { - metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 }, - vector: null - } - } - ] - } - await factLog.append(anomalousFact) - await factLog.sync() - - const store2 = new GenerationStore(storage) - await store2.open() - - const after = await storage.readVerbRaw(id) - expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() - expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId) - expect((after.vector as { targetId: string }).targetId).toBe(targetId) - expect((after.metadata as { weight: number }).weight).toBe(2) - }) - - it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => { - const { storage, store } = await openLogAuthorityStore() - const id = randomUUID() - const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 } - - await store.commitTransaction({ - touched: { nouns: [id], verbs: [] }, - execute: async () => { - await storage.writeNounRaw(id, { - metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, - vector: vectorRecord - }) - } - }) - expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed - - const factLog = store.getFactLog()! - await factLog.append({ - generation: 2, - timestamp: Date.now(), - ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent - }) - await factLog.sync() - - const store2 = new GenerationStore(storage) - await store2.open() - - const after = await storage.readNounRaw(id) - expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull() - expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull() - }) -}) diff --git a/tests/integration/filter-operator-conformance.test.ts b/tests/integration/filter-operator-conformance.test.ts deleted file mode 100644 index 628017e7..00000000 --- a/tests/integration/filter-operator-conformance.test.ts +++ /dev/null @@ -1,151 +0,0 @@ -/** - * @module tests/integration/filter-operator-conformance - * @description THE OPERATOR SET, AND WHAT EACH TOKEN DOES ON THE INDEX PATH. - * - * The contract-1 manifest splits this engine's `where` operators three ways — - * served, served-beyond-baseline, refused-by-name — and two engines must agree - * token for token. This lane is the machine-checkable side of that agreement: - * it asserts the EXACT accepted set (so a manifest can be diffed against a run - * rather than against prose), and it pins each of the three classes. - * - * The defect it closes: the metadata index's operator switch had no default - * case, so an operator it does not implement — `hasAll`, `noneOf`, `excludes`, - * `startsWith`, `endsWith`, `matches`, `length` — left the field's match set at - * its initial `[]` and `find()` returned an empty page. A documented operator, - * implemented in the in-memory matcher, answering silently wrong. Three of the - * seven are now SERVED on the index path; the other four are REFUSED BY NAME, - * because an equality/range posting index cannot evaluate a substring, a - * pattern or an array length without reading every row. - */ - -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync, readFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { contractVersion, BRAINY_CONTRACT_VERSION } from '../../src/utils/version.js' - -/** The accepted `where` value-operator tokens, as a sorted list. */ -const ACCEPTED_OPERATORS = [ - 'between', 'contains', 'endsWith', 'eq', 'equals', 'excludes', 'exists', - 'greaterThan', 'greaterThanOrEqual', 'gt', 'gte', 'hasAll', 'in', 'length', - 'lessThan', 'lessThanOrEqual', 'lt', 'lte', 'matches', 'missing', 'ne', - 'noneOf', 'notEquals', 'oneOf', 'startsWith' -] as const - -/** Served on the index path with exact posting-set semantics. */ -const SERVED_ON_INDEX = [ - 'between', 'contains', 'eq', 'equals', 'exists', 'greaterThan', - 'greaterThanOrEqual', 'gt', 'gte', 'in', 'lessThan', 'lessThanOrEqual', - 'lt', 'lte', 'missing', 'ne', 'notEquals', 'oneOf', - 'excludes', 'hasAll', 'noneOf' -] as const - -/** Accepted by name, refused by the index path — never answered empty. */ -const REFUSED_BY_INDEX = ['endsWith', 'length', 'matches', 'startsWith'] as const - -describe('filter operator conformance', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - }) - - async function seeded(): Promise { - const dir = mkdtempSync(join(tmpdir(), 'brainy-operators-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - await brain.add({ - data: 'a document about ferrets', - type: NounType.Document, - metadata: { tags: ['ferret', 'small', 'furry'], team: 'alpha' } - }) - await brain.add({ - data: 'a document about whales', - type: NounType.Document, - metadata: { tags: ['whale', 'large'], team: 'beta' } - }) - await brain.flush() - return brain - } - - it('the accepted operator set is exactly these 25 tokens', async () => { - const brain = await seeded() - // The engine names its own valid set in the refusal it raises for an - // unknown token — the honest place to read it from. - let message = '' - try { - await brain.find({ where: { team: { notIn: ['alpha'] } } } as never) - } catch (err) { - message = (err as Error).message - } - expect(message).toMatch(/Unknown filter operator "notIn"/) - const listed = (message.match(/Valid operators: ([^.]+)\./)?.[1] ?? '') - .split(',') - .map((t) => t.trim()) - .filter(Boolean) - .sort() - expect(listed).toEqual([...ACCEPTED_OPERATORS].sort()) - expect(listed.length).toBe(25) - // Four tokens a sibling manifest listed as served aliases are NOT in this - // engine's set and never have been — they raise INVALID_QUERY. - for (const absent of ['is', 'isNot', 'greaterEqual', 'lessEqual']) { - expect(listed).not.toContain(absent) - await expect( - brain.find({ where: { team: { [absent]: 'alpha' } } } as never) - ).rejects.toThrow(/Unknown filter operator/) - } - }, 120_000) - - it('serves hasAll, noneOf and excludes on the index path — never an empty page', async () => { - const brain = await seeded() - - const hasAll = await brain.find({ where: { tags: { hasAll: ['ferret', 'furry'] } } } as never) - expect(hasAll.length).toBe(1) - expect((hasAll[0] as { metadata?: Record }).metadata?.team).toBe('alpha') - - const noneOf = await brain.find({ where: { team: { noneOf: ['alpha'] } } } as never) - expect(noneOf.length).toBe(1) - expect((noneOf[0] as { metadata?: Record }).metadata?.team).toBe('beta') - - const excludes = await brain.find({ where: { tags: { excludes: 'whale' } } } as never) - expect(excludes.length).toBe(1) - expect((excludes[0] as { metadata?: Record }).metadata?.team).toBe('alpha') - - // hasAll with an operand nothing carries is EMPTY because it is empty — - // the honest zero, reached by evaluating the operator. - const none = await brain.find({ where: { tags: { hasAll: ['ferret', 'whale'] } } } as never) - expect(none.length).toBe(0) - }, 120_000) - - it('refuses the four index-unserveable operators BY NAME', async () => { - const brain = await seeded() - for (const op of REFUSED_BY_INDEX) { - const operand = op === 'length' ? 3 : 'a' - await expect( - brain.find({ where: { team: { [op]: operand } } } as never), - `${op} must refuse, never answer an empty page` - ).rejects.toThrow(new RegExp(`Filter operator "${op}".*cannot be served by the metadata index`, 's')) - } - }, 120_000) - - it('declares its contract version in code and in package.json', async () => { - expect(contractVersion()).toBe(1) - expect(BRAINY_CONTRACT_VERSION).toBe(1) - const pkg = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8')) - expect(pkg.brainyContract).toBe(contractVersion()) - }) - - it('the three classes partition the accepted set', () => { - expect([...SERVED_ON_INDEX, ...REFUSED_BY_INDEX].sort()).toEqual([...ACCEPTED_OPERATORS].sort()) - }) -}) diff --git a/tests/integration/flush-watcher-event-driven.test.ts b/tests/integration/flush-watcher-event-driven.test.ts deleted file mode 100644 index 4b2e80c4..00000000 --- a/tests/integration/flush-watcher-event-driven.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * @module tests/integration/flush-watcher-event-driven - * @description THE FLUSH-REQUEST WATCH IS EVENT-DRIVEN. - * - * It used to `readdir` the request directory every 500 ms, per brain, for the - * life of every writer — armed on every non-reader brain whether or not any - * inspector process existed. MEASURED on a production process holding 21 - * brains: 42 directory reads per second on a completely idle service, plus a - * stale-request GC pass on every one of them. - * - * The law: a request that has not been made is not a cause. The arrival itself - * wakes the watcher, so the request is seen SOONER than the poll saw it, and a - * slow safety sweep covers filesystems that drop watch events and the GC. - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' -import * as nodeFs from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -describe('the flush-request watcher', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - vi.restoreAllMocks() - }) - - async function openWriter(): Promise<{ brain: Brainy; dir: string }> { - const dir = mkdtempSync(join(tmpdir(), 'brainy-flush-watch-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - await brain.add({ data: 'a row', type: NounType.Concept }) - await brain.flush() - return { brain, dir } - } - - it('does not poll the request directory on an idle writer', async () => { - const { dir } = await openWriter() - const reqDir = join(dir, 'locks', '_flush_requests') - - // Count real reads of the request directory over a window far longer than - // the old 500ms poll (which would have made ~16 of them). - const realReaddir = nodeFs.promises.readdir - let requestDirReads = 0 - const spy = vi - .spyOn(nodeFs.promises, 'readdir') - .mockImplementation((async (p: unknown, ...rest: unknown[]) => { - if (String(p) === reqDir) requestDirReads++ - return (realReaddir as unknown as (...a: unknown[]) => Promise)(p, ...rest) - }) as typeof nodeFs.promises.readdir) - - await new Promise((r) => setTimeout(r, 8_000)) - spy.mockRestore() - - // The old poll: 500ms → ~16 reads. The safety sweep is 30s → 0 in this window. - expect(requestDirReads).toBeLessThanOrEqual(1) - }, 120_000) - - it('answers a request that arrives, without waiting for the sweep', async () => { - const { brain, dir } = await openWriter() - const reqDir = join(dir, 'locks', '_flush_requests') - const ackDir = join(dir, 'locks', '_flush_responses') - mkdirSync(reqDir, { recursive: true }) - - // Drop a request exactly as an out-of-process inspector does. - const id = 'test-request-0001' - writeFileSync(join(reqDir, `${id}.req`), JSON.stringify({ at: Date.now() })) - - // The ack must land far sooner than the 30s safety sweep. - const deadline = Date.now() + 10_000 - let acked = false - while (Date.now() < deadline) { - try { - const entries = await nodeFs.promises.readdir(ackDir) - if (entries.some((e) => e.startsWith(id))) { acked = true; break } - } catch { /* dir not created yet */ } - await new Promise((r) => setTimeout(r, 100)) - } - expect(acked, 'the watcher must answer an arriving request').toBe(true) - void brain - }, 120_000) -}) diff --git a/tests/integration/health-gate.test.ts b/tests/integration/health-gate.test.ts deleted file mode 100644 index f2952116..00000000 --- a/tests/integration/health-gate.test.ts +++ /dev/null @@ -1,367 +0,0 @@ -/** - * @module tests/integration/health-gate - * @description Pins for the health-by-accounting read gate: the read gate stops - * consulting an unnamed `isReady()` boolean and reads a NAMED, sync, O(1) - * {@link HealthReport}; no read path may ever start a store walk; the open path - * brings every provider to serving before it returns; an explicit operator door - * (`repairIndex({ rebuild: [...] })`) rebuilds a named leg unconditionally. - * - * Providers here are white-box test doubles: a `healthReport()` (or, for the - * interim-path pins, an `isReady()`) function assigned directly onto the LIVE - * JS provider object, the same pattern `tests/unit/validate-invariants-delegation.test.ts` - * uses for `validateInvariants`. This exercises brainy's real gate/verify code - * against a controlled provider self-report — no engine mocks. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { - Brainy, - NounType, - VerbType, - GraphIndexNotReadyError, - MetadataIndexNotReadyError, - VectorIndexNotReadyError -} from '../../src/index.js' -import type { HealthReport, LedgerInvariantResult } from '../../src/plugin.js' -import { prodLog } from '../../src/utils/logger.js' -import { createTestConfig } from '../helpers/test-factory.js' - -/** The white-box surface these pins drive on a live brain instance. */ -interface BrainInternals { - storage: { - getNoun(id: string): Promise - getNounMetadata(id: string): Promise - getNouns(options?: unknown): Promise - getVerbs(options?: unknown): Promise - } - index: { healthReport?: () => HealthReport; isReady?: () => boolean; rebuild(): Promise } - metadataIndex: { - healthReport?: () => HealthReport - isReady?: () => boolean - rebuild(): Promise - validateInvariants?: () => Promise - } - graphIndex: { - healthReport?: () => HealthReport - isReady?: () => boolean - rebuild(): Promise - validateInvariants?: () => Promise - } - rebuildIndexesIfNeeded(force?: boolean): Promise -} - -function internalsOf(brain: Brainy): BrainInternals { - return brain as unknown as BrainInternals -} - -function invariant(overrides: Partial = {}): LedgerInvariantResult { - return { - name: 'manifest-residency', - holds: true, - detail: 'ok', - heal: 'none', - source: 'ledger', - ...overrides - } -} - -function healthReport(overrides: Partial = {}): HealthReport { - return { - provider: 'vector', - healthy: true, - serving: true, - invariants: [], - checkedAt: Date.now(), - durationMs: 1, - generation: 1, - unledgered: [], - ...overrides - } -} - -const brains: Brainy[] = [] -const dirs: string[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) - vi.restoreAllMocks() -}) - -describe('health gate (a) — not-serving refuses loudly, ZERO canonical reads during the refusal', () => { - it('metadata not-serving: find() throws MetadataIndexNotReadyError naming the failing invariant', async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) - await brain.flush() - - const internals = internalsOf(brain) - internals.metadataIndex.healthReport = () => - healthReport({ - provider: 'metadata', - serving: false, - healthy: false, - invariants: [invariant({ name: 'posted-count-floor', holds: false, heal: 'rebuild', detail: 'posted 2 < canonical 5' })] - }) - - const getNounSpy = vi.spyOn(internals.storage, 'getNoun') - const getNounMetadataSpy = vi.spyOn(internals.storage, 'getNounMetadata') - const getNounsSpy = vi.spyOn(internals.storage, 'getNouns') - - await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) - await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/posted-count-floor/) - - expect(getNounSpy).not.toHaveBeenCalled() - expect(getNounMetadataSpy).not.toHaveBeenCalled() - expect(getNounsSpy).not.toHaveBeenCalled() - - delete internals.metadataIndex.healthReport - }) - - it('graph not-serving: related() throws GraphIndexNotReadyError naming the failing invariant, no canonical reads', async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - const a = await brain.add({ data: 'a', type: NounType.Person }) - const b = await brain.add({ data: 'b', type: NounType.Person }) - await brain.relate({ from: a, to: b, type: VerbType.Knows }) - await brain.flush() - - const internals = internalsOf(brain) - internals.graphIndex.healthReport = () => - healthReport({ - provider: 'graph', - serving: false, - healthy: false, - invariants: [invariant({ name: 'adjacency-residency', holds: false, heal: 'rebuild', detail: 'edges not loaded' })] - }) - - const getNounSpy = vi.spyOn(internals.storage, 'getNoun') - const getVerbsSpy = vi.spyOn(internals.storage, 'getVerbs') - - await expect(brain.related({ from: a })).rejects.toBeInstanceOf(GraphIndexNotReadyError) - await expect(brain.related({ from: a })).rejects.toThrow(/adjacency-residency/) - - expect(getNounSpy).not.toHaveBeenCalled() - expect(getVerbsSpy).not.toHaveBeenCalled() - - delete internals.graphIndex.healthReport - }) -}) - -describe('health gate (b) — unledgered is unknown: never blocks a serving provider', () => { - it('serving:true with an unledgered family and no failing invariant serves normally; at most one narration', async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) - await brain.flush() - - const internals = internalsOf(brain) - internals.metadataIndex.healthReport = () => - healthReport({ - provider: 'metadata', - serving: true, - healthy: true, - invariants: [], - unledgered: ['canonical-verb-coverage'] - }) - - const warnSpy = vi.spyOn(prodLog, 'warn') - - const r1 = await brain.find({ where: { team: 'atlas' } }) - const r2 = await brain.find({ where: { team: 'atlas' } }) - expect(r1.length).toBe(1) - expect(r2.length).toBe(1) - - const narrations = warnSpy.mock.calls.filter( - ([msg]) => typeof msg === 'string' && msg.includes('canonical-verb-coverage') - ) - expect(narrations.length).toBe(1) // one narration at most across both reads (same generation) - - delete internals.metadataIndex.healthReport - }) -}) - -describe('health gate (c) — degraded-but-serving narrates once per generation', () => { - // PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the - // degraded report lives on the family the read actually consults. - it('a heal:"repair" failure serves; narrates once per DISTINCT VERDICT, not once per generation bump', async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) - await brain.flush() - - const internals = internalsOf(brain) - let generation = 1 - let detail = 'counter drift' - internals.metadataIndex.healthReport = () => - healthReport({ - provider: 'vector', - serving: true, - healthy: false, - invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail })], - generation - }) - - const warnSpy = vi.spyOn(prodLog, 'warn') - const countNarrations = () => - warnSpy.mock.calls.filter(([msg]) => typeof msg === 'string' && msg.includes('stale-vector-counter')).length - - await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(1) // same verdict both times — one narration - - // THE DEDUPE KEY IS THE VERDICT, NOT THE COUNTER. A provider's `generation` - // bumps on every ledger mutation and every rebuild boundary, so keying the - // narration on it re-printed an UNCHANGED health line on every read that - // consulted a busy provider — and, in the other direction, let a provider - // that never bumped suppress a line whose reasons had genuinely changed. - // An unchanged verdict is silent however the counter moves: - generation = 2 - await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(1) // generation bumped, verdict identical — still silent - - // ...and a CHANGED verdict is always heard, bump or no bump: - detail = 'counter drift widened to 12 rows' - await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(2) // the reasons changed — a new narration - - delete internals.metadataIndex.healthReport - }) -}) - -describe('health gate (d) — interim isReady()-only path (no healthReport) is unchanged', () => { - it('isReady() === true serves; isReady() === false refuses via the typed NotReady error', async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) - await brain.flush() - - const internals = internalsOf(brain) - internals.metadataIndex.isReady = () => true - await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - - internals.metadataIndex.isReady = () => false - await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) - - delete internals.metadataIndex.isReady - }) -}) - -describe('health gate (e) — open builds; the first read never does', () => { - it('disableAutoRebuild:true on a populated store: open narrates + builds; the first find() triggers zero rebuilds', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-healthgate-open-')) - dirs.push(dir) - - const writer = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - silent: true, - disableAutoRebuild: true - }) - await writer.init() - brains.push(writer) - await writer.add({ data: 'row one', type: NounType.Document, metadata: { team: 'atlas' } }) - await writer.flush() - await brains.pop()!.close() - - const warnSpy = vi.spyOn(prodLog, 'warn') - const reader = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - silent: true, - disableAutoRebuild: true - }) - const internals = internalsOf(reader) - const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded') - - await reader.init() - brains.push(reader) - - expect(rebuildSpy).toHaveBeenCalledTimes(1) // open() built it, exactly once - expect( - warnSpy.mock.calls.some( - ([msg]) => typeof msg === 'string' && msg.includes('open() is building') - ) - ).toBe(true) - - rebuildSpy.mockClear() - const rows = await reader.find({ where: { team: 'atlas' } }) - expect(rebuildSpy).toHaveBeenCalledTimes(0) // the read never builds - expect(rows.length).toBe(1) - }, 30000) -}) - -describe('health gate (f) — the ceremony door: explicit rebuild bypasses invariant consultation', () => { - it("repairIndex({ rebuild: ['graph'] }) rebuilds unconditionally without consulting validateInvariants", async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - await brain.add({ data: 'x', type: NounType.Concept }) - await brain.flush() - - const internals = internalsOf(brain) - let validateCalls = 0 - internals.graphIndex.validateInvariants = async () => { - validateCalls++ - return healthReport({ provider: 'graph' }) - } - const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild') - - const report = await brain.repairIndex({ rebuild: ['graph'] }) - - expect(rebuildSpy).toHaveBeenCalledTimes(1) - expect(validateCalls).toBe(0) // the door never consults validateInvariants to decide - - const graphFamily = report.families.find((f) => f.family === 'provider:graph') - expect(graphFamily?.rebuilt).toBe(true) - expect(graphFamily?.checked).toBe(true) - expect(graphFamily?.reason).toBe('explicit rebuild requested') - - delete internals.graphIndex.validateInvariants - }) - - it('bare repairIndex() on a healthy provider calls no rebuild()', async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - await brain.add({ data: 'x', type: NounType.Concept }) - await brain.flush() - - const internals = internalsOf(brain) - internals.graphIndex.validateInvariants = async () => healthReport({ provider: 'graph', healthy: true, serving: true }) - const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild') - - await brain.repairIndex() - - expect(rebuildSpy).not.toHaveBeenCalled() - - delete internals.graphIndex.validateInvariants - }) -}) - -describe('health gate (g) — a throwing healthReport() is a contract violation, never read as healthy', () => { - // PER-FAMILY LAW (10.4.1): the throwing report sits on the family the read consults. - it('healthReport() that throws refuses loudly with the typed NotReady error naming the throw', async () => { - const brain = new Brainy(createTestConfig({ silent: true })) - await brain.init() - brains.push(brain) - await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) - await brain.flush() - - const internals = internalsOf(brain) - internals.metadataIndex.healthReport = () => { - throw new Error('accelerator: mmap window busy') - } - - await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) - await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/mmap window busy/) - - delete internals.metadataIndex.healthReport - }) -}) diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index bb07268d..2bcee038 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,7 +16,6 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' -import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -58,107 +57,6 @@ describe('history repacking — the two-tier lifecycle', () => { } }) - /** - * THE HOLE, END TO END — the shape a real store carries. - * - * A forensic fixture was measured with generation directories 1..2503 - * present except for exactly one: 1416. Its fact-log segment already showed - * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration - * 1940 (531 generations) while recording only 530 facts. - * - * Before the fix, repacking such a store folded ACROSS that hole: the batch - * skipped 1416 (no readable delta) and the sealed segment declared a range - * spanning it anyway. The next open merged that declared range back into - * committedRanges, re-admitting 1416 as committed history, and every - * subsequent auto-compaction pass then asked the packed tier for a frame - * that was never written — producing, on EVERY run, the non-fatal narration - * - * Auto-compaction of generational history failed (non-fatal): generation - * N is inside sealed segment seg-....bgs's declared range but has no frame - * — packed history is damaged - * - * This pin removes a generation directory to make the same hole, then - * requires repack + reopen + compaction to complete cleanly. - */ - it('a missing generation directory does not poison the packed tier', async () => { - const dir = tempDir() - // `retention: 'all'` throughout: close() otherwise auto-compacts the - // history away, and this pin needs the cold generations still on disk so - // there is something to punch a hole in. The live window stays at its - // production default for the build phase, so nothing folds yet. - const archival = async (): Promise => { - const b = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - embeddingFunction: stub, - retention: 'all' - }) - await b.init() - return b - } - const brain = await archival() - - const id = await brain.add({ - data: 'holed-entity', - type: NounType.Document, - metadata: { v: 0 } - }) - // One flush per update: single-op writes coalesce inside a flush window, - // so a history deep enough to have a middle needs the windows separated. - for (let v = 1; v <= 12; v++) { - await brain.update({ id, metadata: { v } }) - await brain.flush() - } - await brain.close() - - // Punch the hole: delete ONE generation directory in the middle of the - // cold range, exactly as the real store presents it. - const genRoot = path.join(dir, '_generations') - const numeric = fs - .readdirSync(genRoot, { withFileTypes: true }) - .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) - .map((e) => Number(e.name)) - .sort((a, b) => a - b) - expect(numeric.length).toBeGreaterThan(6) - const victim = numeric[Math.floor(numeric.length / 2)] - fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) - - // Now shrink the live window and reopen. close() repacks automatically - // (brainy.ts phase 0b), so this is the production sequence exactly: a - // store with a hole in its history gets folded by ordinary housekeeping, - // with nobody asking for it. - ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 - const reopened = await archival() - const result = await reopened.repackHistory() - expect(result.foldedGenerations).toBeGreaterThan(0) - - const segDir = path.join(dir, SEGMENTS_PREFIX) - const manifestPath = ['manifest.json', 'manifest.json.gz'] - .map((f) => path.join(segDir, f)) - .find((p) => fs.existsSync(p))! - const raw = manifestPath.endsWith('.gz') - ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') - : fs.readFileSync(manifestPath, 'utf8') - const manifest = JSON.parse(raw) as { - segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> - } - - // THE LAW: every sealed segment declares exactly as many generations as it - // holds frames, and none of them spans the victim. - for (const s of manifest.segments) { - expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) - expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) - } - - await reopened.close() - - // And the pass that used to fail on every run now completes: reopen (which - // re-seeds committedRanges from the packed tier) then compact history. - const third = await openBrain(dir) - await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() - await third.close() - }) - it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/integration/hybrid-search-vfs.test.ts b/tests/integration/hybrid-search-vfs.test.ts index 219c4c83..c881fa97 100644 --- a/tests/integration/hybrid-search-vfs.test.ts +++ b/tests/integration/hybrid-search-vfs.test.ts @@ -21,16 +21,10 @@ describe('Hybrid Search with VFS', () => { testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`) fs.mkdirSync(testDir, { recursive: true }) - // `storage.path`, NOT the pre-8.0 `options.basePath` alias. That alias was - // removed at the 8.0 major and configures nothing, so this suite silently - // opened the DEFAULT store instead of its own temp directory — sharing one - // on-disk brain with every other run on the machine, accumulating tens of - // thousands of rows, and eventually failing on that shared store's graph - // adjacency rather than on anything it was written to test. brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', - path: testDir + options: { basePath: testDir } } }) await brain.init() diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts deleted file mode 100644 index b5c386cf..00000000 --- a/tests/integration/idle-costs-nothing.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -/** - * @module tests/integration/idle-costs-nothing - * @description AN IDLE BRAIN DOES NO WORK. - * - * A flush used to re-persist state identical to what was already on disk — - * the provider flushes, the watermark stamps, the generation counter, the - * entity-tree stamp, roughly 28 writes — because `flush()` never asked whether - * anything had changed. - * - * The field observation that started this: a production process holding 21 - * brains printed "All indexes flushed to disk in 216–601ms" per brain every - * ~35 seconds and idled at 1.26 cores, with no writes for ten minutes. This - * engine's cadence is WRITE-DRIVEN, so that observation is NOT explained by - * the cadence and is not claimed to be fixed here — what is fixed is that such - * a call now costs nothing. Who was calling flush() remains open. - * - * The laws pinned here: - * (a) the persistence cadence arms only on a write — a brain nobody writes - * to flushes zero times, however long it is left open; - * (b) a flush on a clean brain is O(1): no provider is called, nothing is - * written, and nothing is printed; - * (c) one write earns exactly one flush's worth of work, and no more. - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -/** Wait for any in-flight background flush, then let the idle timer settle. */ -async function drainCadence(brain: Brainy): Promise { - const inner = brain as unknown as { _persistBackgroundFlight: Promise | null } - await new Promise((r) => setTimeout(r, 3_000)) - await (inner._persistBackgroundFlight ?? Promise.resolve()) - await new Promise((r) => setTimeout(r, 500)) -} - -/** How long an idle brain is watched. Longer than the 30s flush interval. */ -const IDLE_WATCH_MS = 90_000 - -describe('an idle brain costs nothing', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - vi.restoreAllMocks() - }) - - async function openBrain(): Promise { - const dir = mkdtempSync(join(tmpdir(), 'brainy-idle-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - return brain - } - - it('flushes zero times over 90 idle seconds, and prints nothing', async () => { - const brain = await openBrain() - // One write and one flush to reach a clean, settled state — then nothing. - await brain.add({ data: 'the only write this test performs', type: NounType.Concept }) - await brain.flush() - - const logged: string[] = [] - const origLog = console.log - console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log - - // Watch the providers directly: a flush that runs calls all of them. - const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage - const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex - const graphIndex = (brain as unknown as { graphIndex: { flush: () => Promise } }).graphIndex - const countsSpy = vi.spyOn(storage, 'flushCounts') - const metadataSpy = vi.spyOn(metadataIndex, 'flush') - const graphSpy = vi.spyOn(graphIndex, 'flush') - - try { - await new Promise((r) => setTimeout(r, IDLE_WATCH_MS)) - } finally { - console.log = origLog - } - - // (a) + (b): nothing ran, nothing was said. - expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) - expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([]) - expect(countsSpy).not.toHaveBeenCalled() - expect(metadataSpy).not.toHaveBeenCalled() - expect(graphSpy).not.toHaveBeenCalled() - }, 180_000) - - it('an explicit flush over a clean brain calls no provider and prints nothing', async () => { - const brain = await openBrain() - await brain.add({ data: 'one write', type: NounType.Concept }) - await brain.flush() // this one does the work - - const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage - const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex - const countsSpy = vi.spyOn(storage, 'flushCounts') - const metadataSpy = vi.spyOn(metadataIndex, 'flush') - const logged: string[] = [] - const origLog = console.log - console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log - try { - await brain.flush() // ...and this one has nothing to do - await brain.flush() - await brain.flush() - } finally { - console.log = origLog - } - - expect(countsSpy).not.toHaveBeenCalled() - expect(metadataSpy).not.toHaveBeenCalled() - expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) - }, 120_000) - - it('one write earns exactly one flush', async () => { - const brain = await openBrain() - await brain.add({ data: 'first', type: NounType.Concept }) - await brain.flush() - // Settle: the first write also kicked a BACKGROUND flush, which is not - // awaited by design. Drain it before counting, or its provider calls land - // inside this test's window and are attributed to the write below. - await drainCadence(brain) - - // Count the flushes that actually RAN. (Provider spies cannot answer this: - // the storage adapter's own count ledger is write-through, so a write calls - // flushCounts() on its own account, with no flush involved.) - const logged: string[] = [] - const origLog = console.log - console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log - const ran = () => logged.filter((l) => /All indexes flushed to disk/.test(l)).length - try { - await brain.add({ data: 'second — this is the cause', type: NounType.Concept }) - await brain.flush() - expect(ran()).toBe(1) - - // No further cause, no further work. - await brain.flush() - await brain.flush() - expect(ran()).toBe(1) - } finally { - console.log = origLog - } - }, 120_000) -}) diff --git a/tests/integration/index-skips-unvectored.test.ts b/tests/integration/index-skips-unvectored.test.ts deleted file mode 100644 index c65aaa01..00000000 --- a/tests/integration/index-skips-unvectored.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -/** - * @module tests/integration/index-skips-unvectored - * @description THE UNVECTORED-ROW CURE — two integration tests - * (`tests/lifecycle/biography.test.ts`'s Ch4/5/6 chapter and - * `tests/integration/clear-persistence.test.ts`'s multi-cycle test) started - * failing after a canonical-storage change made a vector-less row (the - * class-J shape: `vector: []`, e.g. the VFS root, a deferred embed not yet - * landed, or any other legitimately-unvectored canonical record) VISIBLE to - * the enumeration walk `getNounsWithPagination()` for the first time — before - * that change such rows were simply invisible to the walk. `hnswIndex.ts`'s - * `rebuild()` never guarded against that shape: it inserted every row the - * walk yielded into the live in-memory index, including ones with a length-0 - * vector, because `storage.getVectorIndexData()` derives its {level, - * connections} answer straight from the noun's OWN record — it returns - * non-null for ANY existing noun, whether or not that noun was ever actually - * indexed via `addItem()`. A vector-less node admitted into the graph could - * become the entry point (or occupy any graph position), and the very next - * real-vectored `addItem()` then ran a distance calculation against it — - * `cosineDistance` throws "Vectors must have the same dimensions" the moment - * one operand is a length-0 array. - * - * THE FIX, at two layers (`src/hnsw/hnswIndex.ts`): - * (1) FILL/REBUILD/LOAD consumers treat `vector.length === 0` as "unvectored — - * nothing to index" and skip the row (normal, not an error; one summary - * count line, never per-row spam) — `rebuild()`'s loop now checks this - * BEFORE ever creating a graph node, so an unvectored row can never - * become an index member, entry point, or dimension-setter. - * (2) THE INDEX ITSELF refuses a length-0 vector in `addItem()` / - * `updateItem()` with a typed `EmptyVectorIndexError`, loudly, instead of - * ever pinning `dimension = 0` or storing a vector-less node — so no - * future fill/rebuild/load path can silently poison the index even if it - * forgets law (1). - * - * Four legs pinned here: - * (a) `rebuild()` over a store mixing real-vectored rows and `vector: []` - * rows indexes ONLY the vectored ones — size === vectored count, - * dimension pinned to the real (non-zero) length. - * (b) `clear()` then real adds afterward never trip a dimension mismatch — - * the exact `clear-persistence.test.ts` regression shape, reproduced - * directly against the index/storage seam this module owns. - * (c) `index.addItem({ id, vector: [] })` throws `EmptyVectorIndexError` - * (and `updateItem` does too, for an existing node). - * (d) crash -> repair: the crashed generation's entities survive, the ledger - * recounts honestly, and a fresh real-vectored add afterward never trips - * a dimension mismatch against a leftover vector-less phantom. - */ -import { describe, it, expect, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' -import { EmptyVectorIndexError } from '../../src/hnsw/hnswIndex.js' -import { abandonAsCrashed, openBrain as openKillMatrixBrain, uid, vec } from '../helpers/durabilityKillMatrix.js' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - for (const d of tmpDirs.splice(0)) { - try { - fs.rmSync(d, { recursive: true, force: true }) - } catch { - /* best-effort cleanup */ - } - } -}) - -/** A filesystem-backed brain with explicit vectors (no embedder needed) and - * manual persistence — mirrors `durabilityKillMatrix.ts`'s `openBrain` so - * every write in this module is explicit and provably durable. */ -function openBrain(dir: string): any { - return new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - persistence: { policy: 'manual' } - }) -} - -describe('HNSW index skips unvectored rows', () => { - it('(a) rebuild() indexes only vectored rows: size === vectored count, dimension pinned to the real length', async () => { - const dir = mkTmp() - let brain = openBrain(dir) - await brain.init() - - // 5 real-vectored rows. - const vectoredIds: string[] = [] - for (let i = 0; i < 5; i++) { - const id = uid(`vectored-${i}`) - await brain.add({ id, data: `real entity ${i}`, type: NounType.Document, vector: vec(i) }) - vectoredIds.push(id) - } - // 3 explicit unvectored rows — the class-J "vector: []" shape, a normal, - // enumerable, countable canonical row that must never reach the index. - const unvectoredIds: string[] = [] - for (let i = 0; i < 3; i++) { - const id = uid(`unvectored-${i}`) - await brain.add({ id, data: `unvectored entity ${i}`, type: NounType.Document, vector: [] }) - unvectoredIds.push(id) - } - await brain.flush() - - // The canonical ledger already agrees before any rebuild: nouns.all - // counts every row (8 + the VFS root); vectors.all counts only the real - // ones (5) — the VFS root and the 3 explicit unvectored rows are excluded. - const ledgerBeforeReopen = await brain.storage.getCanonicalCounts() - expect(ledgerBeforeReopen.vectors.all).toBe(5) - expect(ledgerBeforeReopen.nouns.all).toBe(9) // 5 vectored + 3 unvectored + 1 VFS root - - await brain.close() - - // Reopen: open()'s index build IS hnswIndex.rebuild() run fresh from - // storage — this is the exact path that used to admit unvectored rows. - brain = openBrain(dir) - await brain.init() - - const status = await brain.getIndexStatus() - expect(status.hnswIndex.size, 'the rebuilt index must contain ONLY the 5 real-vectored rows').toBe(5) - - // Dimension is pinned to the REAL embedded length (384 via `vec()`), not - // 0 — adding a wrong-length vector must be refused naming that real - // dimension, proving no vector-less row ever set it. - const realDimension = vec(0).length - let mismatchMessage: string | undefined - try { - await brain.index.addItem({ id: uid('dimension-probe'), vector: vec(0).slice(0, realDimension - 1) }) - expect.fail('expected a dimension mismatch error') - } catch (err) { - mismatchMessage = (err as Error).message - } - expect(mismatchMessage).toContain(`expected ${realDimension}`) - - // Every unvectored row is still a normal, enumerable, readable canonical - // record — class-J semantics survive the rebuild fix untouched. - for (const id of unvectoredIds) { - const entity = await brain.get(id, { includeVectors: true }) - expect(entity, `unvectored entity ${id} must remain readable`).not.toBeNull() - expect(entity.vector).toEqual([]) - } - // A correct-dimension add succeeds cleanly against the pinned dimension. - const freshId = uid('post-reopen-fresh') - await expect(brain.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(50) })).resolves.toBe( - freshId - ) - - await brain.close() - }) - - it('(b) clear() then real adds afterward never trip a dimension mismatch (the clear-persistence regression shape)', async () => { - const dir = mkTmp() - let brain = openBrain(dir) - await brain.init() // the VFS root (vector: []) is the store's only row - - await brain.clear() - await brain.close() - - // Reopen over a store whose only surviving row is the recreated, - // unvectored VFS root — this is exactly the shape that used to poison - // the entry point / dimension in `clear-persistence.test.ts`. - brain = openBrain(dir) - await brain.init() - expect((await brain.getIndexStatus()).hnswIndex.size).toBe(0) - - const id1 = uid('after-clear-1') - await expect(brain.add({ id: id1, data: 'after clear 1', type: NounType.Document, vector: vec(1) })).resolves.toBe( - id1 - ) - const id2 = uid('after-clear-2') - await expect(brain.add({ id: id2, data: 'after clear 2', type: NounType.Document, vector: vec(2) })).resolves.toBe( - id2 - ) - expect((await brain.getIndexStatus()).hnswIndex.size).toBe(2) - - await brain.close() - }) - - it('(c) index.addItem/updateItem refuse a length-0 vector with EmptyVectorIndexError', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - await expect(brain.index.addItem({ id: uid('empty-add'), vector: [] })).rejects.toThrow(EmptyVectorIndexError) - - // updateItem on an EXISTING (real-vectored) node must refuse the same way. - const existingId = uid('existing-for-update') - await brain.add({ id: existingId, data: 'existing', type: NounType.Document, vector: vec(9) }) - await expect(brain.index.updateItem({ id: existingId, vector: [] })).rejects.toThrow(EmptyVectorIndexError) - - // The index was never disturbed by either refused call. - expect((await brain.getIndexStatus()).hnswIndex.size).toBe(1) - - await brain.close() - }) - - it('(d) crash -> repair: the crashed generation survives, the ledger recounts honestly, and a fresh add afterward never trips a dimension mismatch', async () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-crash-')) - try { - let brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) - - // Baseline: real-vectored entities, durably flushed. - const baselineIds: string[] = [] - for (let i = 0; i < 5; i++) { - const id = uid(`baseline-${i}`) - await brain.add({ id, data: `baseline entity ${i}`, type: NounType.Document, vector: vec(i) }) - baselineIds.push(id) - } - await brain.flush() - - // Crash window: at-ack writes that are never flushed before the crash. - const crashedIds: string[] = [] - for (let i = 0; i < 4; i++) { - const id = uid(`crashed-${i}`) - await brain.add({ id, data: `crash-window entity ${i}`, type: NounType.Document, vector: vec(100 + i) }) - crashedIds.push(id) - } - await abandonAsCrashed(brain) - - // Reopen — logAuthority: 'adopt' replays the at-ack log for the crash window. - brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) - for (const id of [...baselineIds, ...crashedIds]) { - expect(await brain.get(id), `entity ${id} must survive the crash`).not.toBeNull() - } - - // Repair — must not disturb any entity, and must recount the ledger honestly. - const report = await brain.repairIndex() - expect(report.families.length).toBeGreaterThan(0) - for (const id of [...baselineIds, ...crashedIds]) { - expect(await brain.get(id), `entity ${id} must survive repair`).not.toBeNull() - } - - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - expect(ledger.vectors.all).toBe(baselineIds.length + crashedIds.length) - - // Second life: a fresh real-vectored add must never trip a dimension - // mismatch against a vector-less phantom left in the index — the exact - // mechanism `clear-persistence.test.ts` and the biography lane hit. - const secondLifeId = uid('second-life') - await expect( - brain.add({ id: secondLifeId, data: 'second life entity', type: NounType.Document, vector: vec(200) }) - ).resolves.toBe(secondLifeId) - expect(await brain.get(secondLifeId)).not.toBeNull() - - await brain.close() - } finally { - fs.rmSync(dir, { recursive: true, force: true }) - } - }) -}) diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts deleted file mode 100644 index 7d09e893..00000000 --- a/tests/integration/ledger-derivation-identity.test.ts +++ /dev/null @@ -1,224 +0,0 @@ -/** - * @module tests/integration/ledger-derivation-identity - * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD - * count, never a container count. A pre-8.3.1 partial-delete defect can - * leave a "ghost" container (a stale `vectors.json` with no metadata content - * leg) or a "scar" container (an empty `entities////` - * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need - * the metadata content leg — yet the legacy derivation counted one entity - * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever - * (they were never clamped and never re-derived). Laws under test: - * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per - * metadata content leg (`metadata.json` or `.json.gz`), the same test - * `pruneOrphanedEntities()` uses, so the two agree by construction. - * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AND THE OPEN NEVER WALKS — a - * counts.json that carries the ALL scalars but no - * `allCountsDerivedBy: 'identity-record'` stamp predates this fix; - * loading it marks `suspect = true` from a single field read alone and - * warns exactly once naming the cause. The open itself never pays a - * directory walk. - * (2b) AND IT HEALS ITSELF. The ledger used to stay wrong for the life of the - * store, waiting for an operator to run `repairIndex()` — and a - * downstream index heal subtracted against the inflated denominator and - * reported work that did not exist. An honest derivation now runs in the - * BACKGROUND after the open (never blocking it, observable via - * `whenCountLedgerSettled()`), and refuses to stamp a number it derived - * while writes were landing. - * (3) THE SANCTIONED RECOUNT ALSO CLEARS IT — `repairIndex()` prunes the - * orphaned containers, recounts from the canonical metadata.json walk, - * and re-stamps — the ALL scalar is exact and the containers are gone. - * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store - * with no counts.json stamps as it writes, so a brand-new store never - * carries the legacy signature. - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy, FileSystemStorage } from '../../src/index.js' -import { prodLog } from '../../src/utils/logger.js' - -const countsPath = (root: string) => path.join(root, '_system', 'counts.json') - -/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */ -function plantGhost(root: string, shard: string, id: string): void { - const idDir = path.join(root, 'entities', 'nouns', shard, id) - fs.mkdirSync(idDir, { recursive: true }) - fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] })) -} - -/** Plant a scar container: an empty id directory, no legs at all. */ -function plantScar(root: string, shard: string, id: string): void { - fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true }) -} - -describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => { - let dir: string - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(() => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-')) - }) - afterEach(() => { - vi.restoreAllMocks() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => { - let brain = await open() - const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone - for (let i = 0; i < 3; i++) { - await brain.add({ data: `real ${i}`, type: 'document' }) - } - await brain.flush() - const realTotal = baseline + 3 - await brain.close() - - // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) — - // neither is a live entity. - for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) - for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) - - // Remove counts.json so open() re-derives from scratch (the one-time - // legacy/lost-file derivation path). - fs.rmSync(countsPath(dir), { force: true }) - - brain = await open() - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing - expect(ledger.suspect).toBe(false) - - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(raw.totalNounCountAll).toBe(realTotal) - expect(raw.allCountsDerivedBy).toBe('identity-record') - - await brain.close() - }) - - it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => { - let brain = await open() - await brain.add({ data: 'one', type: 'document' }) - await brain.add({ data: 'two', type: 'document' }) - await brain.flush() - await brain.close() - - // Confirm a normal close under the fix DOES stamp — then strip the stamp - // to simulate a counts.json produced before this fix existed. - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(raw.allCountsDerivedBy).toBe('identity-record') - expect(typeof raw.totalNounCountAll).toBe('number') - expect(typeof raw.totalVerbCountAll).toBe('number') - expect(typeof raw.totalVectoredNounCount).toBe('number') - delete raw.allCountsDerivedBy - fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) - - const narrateSpy = vi.spyOn(prodLog, 'narrate') - // The derivation walks live on FileSystemStorage's prototype. Slow them - // deliberately: the OPEN must not wait for them, and on a two-row store a - // real walk finishes too fast to tell "not awaited" from "instant". - const proto = FileSystemStorage.prototype as any - const realScanEntities = proto.scanCanonicalEntities - let scanEntitiesCalls = 0 - proto.scanCanonicalEntities = async function slow(this: any, ...args: any[]) { - scanEntitiesCalls++ - await new Promise((r) => setTimeout(r, 1_200)) - return realScanEntities.apply(this, args) - } - try { - const openStarted = Date.now() - brain = await open() - const openMs = Date.now() - openStarted - - // THE OPEN DID NOT WALK: two slowed walks would have added 2.4s to it. - expect(openMs).toBeLessThan(2_000) - - // The stamp check itself is an O(1) field read, and it names the cause. - const atOpen = await brain.storage.getCanonicalCounts() - expect(atOpen.suspect).toBe(true) - const stampWarnings = narrateSpy.mock.calls.filter( - ([msg]: any[]) => String(msg).includes('legacy') && String(msg).includes('container rule') - ) - expect(stampWarnings.length).toBe(1) // exactly one, loud - - // ...and the honest derivation is already running behind the open. - await brain.storage.whenCountLedgerSettled() - expect(scanEntitiesCalls).toBeGreaterThan(0) - const healed = await brain.storage.getCanonicalCounts() - expect(healed.suspect).toBe(false) - expect(healed.nouns.all).toBe(raw.totalNounCountAll) - } finally { - proto.scanCanonicalEntities = realScanEntities - } - - await brain.close() - }) - - it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => { - let brain = await open() - const baseline = (await brain.storage.getCanonicalCounts()).nouns.all - for (let i = 0; i < 3; i++) { - await brain.add({ data: `real ${i}`, type: 'document' }) - } - await brain.flush() - const realTotal = baseline + 3 - await brain.close() - - for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) - for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) - - // Force the legacy (unstamped, container-rule-inflated) shape directly — - // the shape a pre-existing production store actually carries. - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars - delete raw.allCountsDerivedBy - fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) - - brain = await open() - // Named suspect at load, then healed in the background WITHOUT the - // operator asking — the inflated container count is corrected to the - // identity-record population, though the orphaned containers themselves - // are still on disk (only repairIndex() removes those). - await brain.storage.whenCountLedgerSettled() - let healed = await brain.storage.getCanonicalCounts() - expect(healed.suspect).toBe(false) - expect(healed.nouns.all).toBe(realTotal) - - await brain.repairIndex() - - let ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again - - const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(persisted.allCountsDerivedBy).toBe('identity-record') - expect(persisted.allCountsSuspect).toBe(false) - expect(persisted.totalNounCountAll).toBe(realTotal) - - await brain.close() - brain = await open() - ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - expect(ledger.nouns.all).toBe(realTotal) - await brain.close() - }) - - it('(d) a fresh store derives with the stamp and is never suspect', async () => { - const brain = await open() - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(raw.allCountsDerivedBy).toBe('identity-record') - await brain.close() - }) -}) diff --git a/tests/integration/metadata-online-rebuild.test.ts b/tests/integration/metadata-online-rebuild.test.ts deleted file mode 100644 index bf7cebdb..00000000 --- a/tests/integration/metadata-online-rebuild.test.ts +++ /dev/null @@ -1,167 +0,0 @@ -/** - * @module tests/integration/metadata-online-rebuild - * @description THE ONLINE JS METADATA REBUILD (B3 Deliverable 3) pins. - * `MetadataIndexManager.rebuild()` used to be clear-then-walk — reads went - * dark for the duration. `repairIndex({ rebuild: ['metadata'] })` now builds - * a fresh replacement index BESIDE the live one (walk canonical + mirror - * every live write via `beginShadow`/`endShadow` + a bounded fact-log fold), - * then atomically swaps the brain's reference — `find()` never observes a - * half-built index, and a write landing DURING the build is never lost. - */ -process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' -import type { MetadataIndexManager } from '../../src/utils/metadataIndex.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -function metadataIndexOf(brain: Brainy): MetadataIndexManager { - return (brain as unknown as { metadataIndex: MetadataIndexManager }).metadataIndex -} - -async function openBrain(): Promise<{ brain: Brainy; dir: string }> { - const dir = mkdtempSync(join(tmpdir(), 'brainy-online-rebuild-')) - dirs.push(dir) - const brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - persistence: { policy: 'manual' }, - logAuthority: 'adopt' - }) - await brain.init() - brains.push(brain) - return { brain, dir } -} - -describe('repairIndex({ rebuild: ["metadata"] }) — the online build-beside rebuild', () => { - it( - 'a find() polled throughout the rebuild of a 2k-noun store never returns fewer rows than ' + - 'before the build started, and a write landing DURING the build is never lost', - async () => { - const { brain, dir } = await openBrain() - void dir - - const N = 2000 - const ids: string[] = [] - for (let i = 0; i < N; i++) { - ids.push( - await brain.add({ - data: `entity ${i}`, - type: NounType.Person, - metadata: { status: i % 2 === 0 ? 'active' : 'inactive' } - }) - ) - } - for (let i = 0; i < 20; i++) { - await brain.relate({ - from: ids[i], to: ids[i + 1], type: VerbType.WorksWith, metadata: { tag: 'orig' } - }) - } - await brain.flush() - - const baseline = await brain.find({ where: { status: 'active' }, limit: 10000 }) - expect(baseline.length).toBe(N / 2) - - // Kick off the online rebuild WITHOUT awaiting — poll reads and - // perform a live write concurrently with it. - const repairPromise = brain.repairIndex({ rebuild: ['metadata'] }) - - let minObserved = Infinity - let polls = 0 - const pollPromise = (async () => { - // Poll until the rebuild settles — bounded so a slow CI box can't - // spin forever, generous enough to actually overlap the walk. - while (polls < 200) { - const rows = await brain.find({ where: { status: 'active' }, limit: 10000 }) - minObserved = Math.min(minObserved, rows.length) - polls++ - await new Promise((resolve) => setTimeout(resolve, 1)) - } - })() - - const newId = await brain.add({ - data: 'added during the rebuild', - type: NounType.Person, - metadata: { status: 'active' } - }) - const newRelId = await brain.relate({ - from: newId, to: ids[0], type: VerbType.WorksWith, metadata: { tag: 'during-build' } - }) - - const [report] = await Promise.all([repairPromise, pollPromise]) - - // THE PIN: never fewer rows than the pre-build baseline, at any polled - // instant — reads served the OLD (fully-populated) manager throughout. - expect(polls).toBeGreaterThan(0) - expect(minObserved).toBeGreaterThanOrEqual(baseline.length) - - // The repair report still accounts for the family (same receipt shape - // regardless of which rebuild mechanism actually ran underneath). - const metadataFamily = report.families.find((f) => f.family === 'provider:metadata') - expect(metadataFamily?.checked).toBe(true) - expect(metadataFamily?.rebuilt).toBe(true) - - // Post-swap correctness: the live write during the build was never - // lost (the beginShadow mirror + post-walk fold caught it). - const afterActive = await brain.find({ where: { status: 'active' }, limit: 10000 }) - expect(afterActive.length).toBe(baseline.length + 1) - expect(afterActive.some((r) => r.id === newId)).toBe(true) - - const index = metadataIndexOf(brain) - expect(await index.getIds('tag', 'during-build')).toEqual([newRelId]) - expect((await index.getIds('tag', 'orig')).length).toBe(20) - - // The swap stamped the watermark — a reopen adopts, zero rebuild. - await brain.close() - brains.length = 0 // already closed above; afterEach must not double-close - const reopened = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - persistence: { policy: 'manual' }, - logAuthority: 'adopt' - }) - await reopened.init() - brains.push(reopened) - const reopenedIndex = metadataIndexOf(reopened) - expect(reopenedIndex.watermarkVerdict()).toBe('adopt') - const reopenedActive = await reopened.find({ where: { status: 'active' }, limit: 10000 }) - expect(reopenedActive.length).toBe(afterActive.length) - }, - 60000 - ) - - it('repairIndex({ rebuild: ["metadata"] }) on an empty store is a trivial no-op walk', async () => { - const { brain } = await openBrain() - const report = await brain.repairIndex({ rebuild: ['metadata'] }) - const metadataFamily = report.families.find((f) => f.family === 'provider:metadata') - expect(metadataFamily?.checked).toBe(true) - expect(await brain.getNounCount()).toBe(0) - }) - - it('two consecutive online rebuilds both leave the index correct (idempotent)', async () => { - const { brain } = await openBrain() - const a = await brain.add({ data: 'a', type: NounType.Person, metadata: { status: 'active' } }) - await brain.add({ data: 'b', type: NounType.Person, metadata: { status: 'inactive' } }) - await brain.flush() - - await brain.repairIndex({ rebuild: ['metadata'] }) - const first = await brain.find({ where: { status: 'active' } }) - expect(first.map((r) => r.id)).toEqual([a]) - - await brain.repairIndex({ rebuild: ['metadata'] }) - const second = await brain.find({ where: { status: 'active' } }) - expect(second.map((r) => r.id)).toEqual([a]) - }) -}) diff --git a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts deleted file mode 100644 index a46ad6a5..00000000 --- a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -/** - * @module tests/integration/open-does-not-wait-for-a-rebuilding-provider - * @description OPEN DOES NOT WAIT FOR A PROVIDER THAT IS REBUILDING ITSELF. - * - * Measured on a production store: a metadata provider that had to rebuild made - * `init()` pay the ENTIRE rebuild on the foreground — 641 seconds — with every - * other family idle behind it, because a provider reporting `serving: false` - * because it is BUSY BUILDING and one reporting `serving: false` because it is - * BROKEN were indistinguishable, and both were answered the same way: call - * `rebuild()`, and wait. - * - * The law: a provider that reports `rebuildInProgress()` owns its own rebuild. - * `init()` returns; every other family serves; THAT family's doors refuse by - * name, carrying the provider's own progress; and the doors open by themselves - * when the provider reports serving. Nothing is ever served empty. - */ - -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import type { ProviderRebuildProgress } from '../../src/utils/indexReadiness.js' - -/** How long the stub provider claims to be rebuilding. */ -const REBUILD_MS = 6_000 - -describe('a provider rebuilding itself never blocks open', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - }) - - it('init() returns in milliseconds, the family refuses by name, then answers', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-provider-')) - dirs.push(dir) - - // Seed a store so the open has something to (not) rebuild. - const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await seed.init() - await seed.add({ data: 'a row with a plain field', type: NounType.Concept, metadata: { kind: 'report' } }) - await seed.flush() - await seed.close() - - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - - // Dress the metadata index as a provider that is rebuilding ITSELF: not - // serving, and honest about why. `init()` wires the real index first, so - // the hooks are installed on the instance as soon as it exists — the gate - // reads them by feature detection, exactly as it would a native provider's. - const rebuildStartedAt = Date.now() - const stillRebuilding = () => Date.now() - rebuildStartedAt < REBUILD_MS - let rebuildCalls = 0 - - const inner = brain as unknown as { - metadataIndex: Record - setupIndex?: unknown - } - // Install on the prototype-free instance right after construction by - // patching the property the moment init() assigns it. - const install = (target: Record) => { - const realRebuild = target.rebuild as () => Promise - target.rebuildInProgress = (): ProviderRebuildProgress | null => - stillRebuilding() - ? { phase: 'metadata shadow build', done: 4_096, total: 14_056, startedAt: rebuildStartedAt } - : null - target.healthReport = () => ({ - provider: 'metadata', - healthy: !stillRebuilding(), - serving: !stillRebuilding(), - generation: 1, - invariants: [], - unledgered: [] - }) - target.rebuild = async () => { - rebuildCalls++ - return realRebuild.call(target) - } - } - - // init() constructs the metadata index; patch as soon as it exists, before - // the gate consults it. A microtask hop after the index is assigned is - // enough because the gate runs later in the same init. - const initPromise = (async () => { - const originalEnsure = (brain as unknown as { setupIndex?: () => unknown }).setupIndex - void originalEnsure - return brain.init() - })() - // Patch on the first tick the index exists. - const patcher = setInterval(() => { - if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { - install(inner.metadataIndex) - } - }, 1) - const startedAt = Date.now() - try { - await initPromise - } finally { - clearInterval(patcher) - } - const openMs = Date.now() - startedAt - - // If the patch did not land before the gate ran, this test proves nothing — - // say so loudly rather than passing vacuously. - expect( - typeof inner.metadataIndex.rebuildInProgress, - 'the stub provider was never installed — the test is vacuous' - ).toBe('function') - - // 1. The open did not wait out the rebuild. - expect(openMs).toBeLessThan(REBUILD_MS) - // 2. And brainy did not start a rebuild of its own on top of the provider's. - expect(rebuildCalls).toBe(0) - - // 3. The family's door refuses BY NAME, carrying the provider's progress. - let refusal: Error | null = null - try { - await brain.find({ where: { kind: 'report' } } as never) - } catch (err) { - refusal = err as Error - } - expect(refusal, 'a not-serving metadata family must refuse, never serve empty').not.toBeNull() - expect(refusal!.message).toMatch(/metadata shadow build/i) - expect(refusal!.message).toMatch(/4,096\/14,056/) - expect(refusal!.message).toMatch(/no action is needed/i) - - // 4. Other families keep serving — the brain is open. - const all = await brain.getNouns?.({ pagination: { limit: 1 } } as never) - expect(all ?? true).toBeTruthy() - - // 5. When the provider reports itself serving, the door opens by itself. - await new Promise((r) => setTimeout(r, REBUILD_MS)) - ;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false - await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined() - }, 180_000) - - it('a rebuilding provider reporting 0 entries is not a CRITICAL, and gets no second rebuild', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-critical-')) - dirs.push(dir) - const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await seed.init() - await seed.add({ data: 'a stored entity', type: NounType.Concept }) - await seed.flush() - await seed.close() - - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - - let rebuildCalls = 0 - const errors: string[] = [] - const origError = console.error - console.error = ((...a: unknown[]) => { errors.push(a.map(String).join(' ')) }) as typeof console.error - - const inner = brain as unknown as { metadataIndex: Record } - const patcher = setInterval(() => { - if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { - const target = inner.metadataIndex - target.rebuildInProgress = () => ({ phase: 'online metadata rebuild', startedAt: Date.now() }) - target.healthReport = () => ({ - provider: 'metadata', healthy: false, serving: false, - generation: 1, invariants: [], unledgered: [] - }) - // The shape the native engine now has: the index reports NOTHING while - // its rebuild runs online behind refusing doors. - target.getStats = async () => ({ totalEntries: 0 }) - target.rebuild = async () => { rebuildCalls++ } - } - }, 1) - try { - await brain.init() - } finally { - clearInterval(patcher) - console.error = origError - } - - expect( - typeof inner.metadataIndex.rebuildInProgress, - 'the stub provider was never installed — the test is vacuous' - ).toBe('function') - expect(errors.filter((l) => /CRITICAL: Metadata index has 0 entries/.test(l))).toEqual([]) - expect(rebuildCalls).toBe(0) - }, 180_000) -}) diff --git a/tests/integration/open-narration.test.ts b/tests/integration/open-narration.test.ts deleted file mode 100644 index 95aba9f1..00000000 --- a/tests/integration/open-narration.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * @module tests/integration/open-narration - * @description THE OPEN IS NEVER SILENT. - * - * A production service opened a 16 GB store and logged nothing at all for - * three minutes before its first line of work. Two defects made that possible - * and both are pinned here: - * - * 1. The phase breakdown was written to `prodLog.warn`, which every - * environment that looks like production clamps away. The narration - * channel (`prodLog.narrate`) is always visible, like `error`. - * 2. Nothing spoke DURING a phase — only after the whole open finished, if - * at all. A heartbeat now names the phase currently running and its - * elapsed wall while the open is still happening. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' -import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' - -function makeTempDir(): string { - return mkdtempSync(join(tmpdir(), 'brainy-open-narration-')) -} - -/** Capture console.warn lines emitted while `fn` runs. */ -async function captureWarn(fn: () => Promise): Promise<{ result: T; lines: string[] }> { - const lines: string[] = [] - const orig = console.warn - console.warn = ((...args: unknown[]) => { - lines.push(args.map((a) => String(a)).join(' ')) - }) as typeof console.warn - try { - return { result: await fn(), lines } - } finally { - console.warn = orig - } -} - -describe('open narration', () => { - let dir: string - let brain: Brainy | null = null - - beforeEach(() => { dir = makeTempDir() }) - - afterEach(async () => { - if (brain) { - try { await brain.close() } catch { /* already closed */ } - brain = null - } - try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } - }) - - it('narrate() survives the production log clamp that silences warn()', async () => { - // Exactly what isProductionEnvironment() does to the logger: level ERROR. - configureLogger({ level: LogLevel.ERROR }) - try { - const { lines } = await captureWarn(async () => { - prodLog.warn('[Brainy] this line is chatter and may be clamped') - prodLog.narrate('[Brainy] this line is why the database is slow') - }) - expect(lines.some((l) => /why the database is slow/.test(l))).toBe(true) - expect(lines.some((l) => /chatter/.test(l))).toBe(false) - } finally { - configureLogger({ level: LogLevel.INFO }) - } - }) - - it('names a slow phase as it ends, and heartbeats while it is still running', async () => { - // Seed a store, then reopen it with a deliberately slow storage init so - // the first phase crosses both the heartbeat and the narrate thresholds. - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - await brain.add({ data: 'seed entity', type: NounType.Concept }) - await brain.flush() - await brain.close() - brain = null - - const realInit = FileSystemStorage.prototype.init - FileSystemStorage.prototype.init = async function slowInit(this: FileSystemStorage) { - await new Promise((r) => setTimeout(r, 6_500)) - return realInit.call(this) - } - // Clamped to ERROR for the whole open: the narration must survive it. - configureLogger({ level: LogLevel.ERROR }) - try { - const { result, lines } = await captureWarn(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - return next - }) - brain = result - - // The heartbeat spoke DURING the phase, naming the phase and its cause. - const heartbeats = lines.filter((l) => /open: still in phase 1\/5 "storage-init"/.test(l)) - expect(heartbeats.length).toBeGreaterThanOrEqual(1) - expect(heartbeats[0]).toMatch(/loading its count ledger/) - - // And the phase named its own wall as it ended. - const ended = lines.filter((l) => /open: phase 1\/5 "storage-init" finished in \d+ms/.test(l)) - expect(ended.length).toBe(1) - - // The whole-open breakdown is on the same always-visible channel. - expect(lines.some((l) => /slow open: \d+ms total \(.*storage-init=/.test(l))).toBe(true) - } finally { - FileSystemStorage.prototype.init = realInit - configureLogger({ level: LogLevel.INFO }) - } - }, 120_000) -}) diff --git a/tests/integration/read-gate-scope-and-no-reembed.test.ts b/tests/integration/read-gate-scope-and-no-reembed.test.ts deleted file mode 100644 index b1319249..00000000 --- a/tests/integration/read-gate-scope-and-no-reembed.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/** - * @module tests/integration/read-gate-scope-and-no-reembed - * @description Two cures from the pair's first production adoption: - * (1) THE READ GATE IS PER-FAMILY — a not-serving VECTOR leg refuses vector - * search only; a pure metadata find({ where }) and graph traversal keep - * serving. The brain-global gate refused a deployment's badge reads for a - * vector-leg verdict that had nothing to do with them. - * (2) NO RE-EMBED ON UNCHANGED DATA — an update() carrying the row's current - * data lands no vector, defers no embed, rewrites nothing. A host - * heartbeat re-writing an unchanged row fed a live index-row loop. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy, VectorIndexNotReadyError } from '../../src/index.js' - -describe('read gate scope + no re-embed on unchanged data', () => { - let dir: string - let brain: any - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-gate-scope-')) - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) - await brain.init() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('a not-serving VECTOR leg refuses vector search only — metadata and graph reads keep serving', async () => { - const a = await brain.add({ data: 'employee alpha', type: 'person', metadata: { status: 'active' } }) - const b = await brain.add({ data: 'employee beta', type: 'person', metadata: { status: 'active' } }) - await brain.relate({ from: a, to: b, type: 'relatedTo' }) - await brain.flush() - - // The vector provider says it is NOT serving (a rebuild-class failure). - brain.index.healthReport = () => ({ - provider: 'vector', healthy: false, serving: false, generation: 7, unledgered: [], - invariants: [{ name: 'node-coverage', holds: false, heal: 'rebuild', detail: 'posted 0 < canonical 2', source: 'ledger' }], - checkedAt: 1, durationMs: 1 - }) - try { - const byStatus = await brain.find({ where: { status: 'active' } }) - expect(byStatus.map((r: any) => r.id).sort(), 'metadata find serves').toEqual([a, b].sort()) - const rel = await brain.related(a) - expect(rel.length, 'graph traversal serves').toBe(1) - await expect(brain.find({ query: 'employee' }), 'vector search refuses typed').rejects.toBeInstanceOf(VectorIndexNotReadyError) - } finally { - delete brain.index.healthReport - } - }) - - it('update() with the row\'s current data re-embeds nothing; a real change re-embeds', async () => { - const id = await brain.add({ data: 'invoice 1042 pending', type: 'document', metadata: { n: 1 } }) - await brain.flush() - const before = (await brain.get(id, { includeVectors: true })).vector - const ledgerBefore = await brain.storage.getCanonicalCounts() - const logBefore = (await brain.transactionLog({ limit: 50 })).length - - // The heartbeat shape: same data, re-written, deferred. - for (let i = 0; i < 3; i++) { - await brain.update({ id, data: 'invoice 1042 pending', metadata: { n: 1, tick: i }, deferEmbedding: true }) - } - await brain.flush() - const after = (await brain.get(id, { includeVectors: true })).vector - const ledgerAfter = await brain.storage.getCanonicalCounts() - const log = await brain.transactionLog({ limit: 50 }) - expect(after, 'vector untouched by unchanged-data writes').toEqual(before) - expect(ledgerAfter.vectors.all, 'vectored ledger untouched').toBe(ledgerBefore.vectors.all) - expect(log.filter((e: any) => e.origin === 'system:embed-landing').length, 'no landing commit for unchanged data').toBe(0) - expect(log.length - logBefore, 'the metadata writes themselves still commit').toBe(3) - - // A REAL change re-embeds (deferred → the worker lands it). - await brain.update({ id, data: 'invoice 1042 PAID', deferEmbedding: true }) - await brain.flush() - const changed = (await brain.get(id, { includeVectors: true })).vector - expect(changed, 'a real data change re-embeds').not.toEqual(before) - expect((await brain.storage.getCanonicalCounts()).vectors.all, 'a re-embed of a vectored row never double-counts').toBe(ledgerBefore.vectors.all) - }) -}) diff --git a/tests/integration/readAfterWrite.test.ts b/tests/integration/readAfterWrite.test.ts index cf1dc9ec..e0ab5863 100644 --- a/tests/integration/readAfterWrite.test.ts +++ b/tests/integration/readAfterWrite.test.ts @@ -34,7 +34,13 @@ describe('Read-After-Write Consistency (v5.7.2 Bug Fix)', () => { testDir = join(tmpdir(), `brainy-consistency-${Date.now()}-${Math.random().toString(36).substring(7)}`) brain = new Brainy({ requireSubtype: false, - storage: { type: 'filesystem', path: testDir }, + storage: { + type: 'filesystem', + config: { + baseDir: testDir, + enableCompression: false // Faster tests + } + }, dimensions: 384 }) diff --git a/tests/integration/repair-narration.test.ts b/tests/integration/repair-narration.test.ts deleted file mode 100644 index 1fbe15e4..00000000 --- a/tests/integration/repair-narration.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -/** - * @module tests/integration/repair-narration - * @description A REPAIR NARRATES ITSELF, AND ITS RECEIPT SAYS WHERE THE TIME - * WENT. - * - * On a production store (14,647 nouns / 73,070 verbs) a `repairIndex()` ran - * for more than thirty minutes at roughly a full core with ZERO log lines - * between its start and its end, while the read doors kept serving. The - * operator could tell it was alive only from `top`, and could not tell which - * of its single-threaded walks it was inside. The law pinned here: - * - * - every phase announces itself BEFORE it works, naming what it is about - * to walk; - * - a heartbeat names the phase still running, at a bounded cadence, for as - * long as it runs; - * - every phase reports its own wall, and that wall is carried in the typed - * receipt (`RepairFamilyReport.durationMs`) — not only in a log line. - * - * All of it on the narration channel, which production's log clamp cannot - * silence (see tests/integration/open-narration.test.ts). - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' -import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' - -describe('repairIndex narration', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - configureLogger({ level: LogLevel.INFO }) - }) - - async function seededBrain(): Promise { - const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-narration-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - for (let i = 0; i < 5; i++) { - await brain.add({ data: `repair subject ${i}`, type: NounType.Concept }) - } - await brain.flush() - return brain - } - - it('announces every phase, reports its wall, and carries that wall in the receipt', async () => { - const brain = await seededBrain() - const narrateSpy = vi.spyOn(prodLog, 'narrate') - - const report = await brain.repairIndex() - - const lines = narrateSpy.mock.calls.map(([m]) => String(m)) - - // Every family that ran has BOTH a start line and a finish line naming it. - for (const family of report.families) { - const started = lines.filter((l) => l.includes(`"${family.family}" started —`)) - const finished = lines.filter((l) => - new RegExp(`"${family.family}" finished in \\d+ms`).test(l) - ) - expect(finished.length, `no finish line for ${family.family}`).toBeGreaterThanOrEqual(1) - // A skipped family may be recorded without a start line only if it never - // began; every family that began must have announced itself. - if (family.checked) { - expect(started.length, `no start line for ${family.family}`).toBeGreaterThanOrEqual(1) - } - // THE RECEIPT CARRIES THE WALL — not only the log. - expect(typeof family.durationMs, `${family.family} has no durationMs`).toBe('number') - expect(family.durationMs).toBeGreaterThanOrEqual(0) - } - - // The closing line accounts for the whole repair, per family. - const closing = lines.filter((l) => /repairIndex complete in \d+ms/.test(l)) - expect(closing.length).toBe(1) - expect(closing[0]).toMatch(/@\d+ms/) - }, 180_000) - - it('heartbeats while a single phase is still walking', async () => { - const brain = await seededBrain() - - // Make one phase long enough to cross the heartbeat cadence, exactly as a - // multi-minute canonical walk does on a real store. - const proto = FileSystemStorage.prototype as unknown as Record< - string, - (...args: unknown[]) => Promise - > - const realPrune = proto.pruneOrphanedEntities - proto.pruneOrphanedEntities = async function slow(this: unknown, ...args: unknown[]) { - await new Promise((r) => setTimeout(r, 6_500)) - return realPrune.apply(this, args) - } - // Clamped as production clamps it: the narration must survive. - configureLogger({ level: LogLevel.ERROR }) - const narrateSpy = vi.spyOn(prodLog, 'narrate') - try { - await brain.repairIndex() - } finally { - proto.pruneOrphanedEntities = realPrune - } - - const beats = narrateSpy.mock.calls - .map(([m]) => String(m)) - .filter((l) => /repairIndex: still in "orphaned-containers" after \d+s/.test(l)) - expect(beats.length).toBeGreaterThanOrEqual(1) - expect(beats[0]).toMatch(/ghost\/scar containers/) - }, 180_000) -}) diff --git a/tests/integration/repair-report.test.ts b/tests/integration/repair-report.test.ts index 28e0ad99..273eee2e 100644 --- a/tests/integration/repair-report.test.ts +++ b/tests/integration/repair-report.test.ts @@ -68,46 +68,4 @@ describe('repairIndex per-family receipt', () => { expect(orphans!.healed, 'the ghost was pruned and receipted').toBeGreaterThan(0) expect(report.healedTotal).toBeGreaterThan(0) }, 120000) - - - it("a heal:'repair' verdict routes to the provider's own repair(), and the re-read decides", async () => { - // A fake provider report: one failing invariant asking for the INCREMENTAL - // heal. repairIndex must call repair() (never rebuild()) and count the heal - // only when the post-repair re-read clears the same verdict. - const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-route-')) - const brain: any = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true }) - await brain.init() - brains.push(brain) - let repairCalls = 0 - let rebuildCalls = 0 - let healed = false - const failing = { - provider: 'vector', healthy: false, serving: true, - invariants: [{ name: 'node-coverage', holds: false, detail: 'short 3', heal: 'repair' as const }], - checkedAt: 1, durationMs: 1 - } - const clean = { - provider: 'vector', healthy: true, serving: true, - invariants: [{ name: 'node-coverage', holds: true, detail: 'ok', heal: 'none' as const }], - checkedAt: 2, durationMs: 1 - } - ;(brain.index as any).validateInvariants = async () => (healed ? clean : failing) - ;(brain.index as any).repair = async () => { repairCalls++; healed = true; return { repaired: 3 } } - const origRebuild = (brain.index as any).rebuild - ;(brain.index as any).rebuild = async () => { rebuildCalls++ } - try { - const report = await brain.repairIndex() - const row = report.families.find((f: any) => f.family === 'provider:vector') - expect(row, 'the provider family is in the receipt').toBeDefined() - expect(repairCalls, 'repair() ran exactly once').toBe(1) - expect(rebuildCalls, "a heal:'repair' verdict never runs rebuild()").toBe(0) - expect(row!.healed, 'the cleared verdict counts as healed').toBe(1) - expect(String(row!.detail)).toMatch(/incremental repair cleared: node-coverage/) - } finally { - delete (brain.index as any).validateInvariants - delete (brain.index as any).repair - ;(brain.index as any).rebuild = origRebuild - } - }) - }) diff --git a/tests/integration/vector-leg-open-build.test.ts b/tests/integration/vector-leg-open-build.test.ts deleted file mode 100644 index ea841545..00000000 --- a/tests/integration/vector-leg-open-build.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** - * @module tests/integration/vector-leg-open-build - * @description THE LAST RED of the two-engine release gate: a migrated - * store can hold canonical vectored nouns with NO derived vector index - * built. `open()` owns building the derived indexes (reads never build — - * see `rebuildIndexesIfNeeded`'s JSDoc); the defect this pins is the vector - * leg's decision silently skipping that build, so `find`/search served `[]` - * with no error and no narration. - * - * A downstream deployment measures this through a native vector provider - * whose own health report can legitimately say `serving: true` even while - * vector COVERAGE is honestly unledgered on its side (an unledgered - * invariant never flips `serving` — see `HealthReport`'s derivation laws). - * This repo ships only the JS engine, so the reproduction here uses the - * SAME plugin seam a native provider would (`brain.use({ activate: ctx => - * ctx.registerProvider('vector', factory) })`, the pattern - * `tests/unit/cold-open-rebuild-gate.test.ts` already established for this - * exact class of gate-decision bug) with a stub that WRAPS the real - * `JsHnswVectorIndex` — every method delegates to a genuine engine (so a - * successful rebuild restores REAL, searchable vectors), except `size()` - * (fakes 0 until rebuild runs — the "never built" posture) and - * `healthReport()` (always reports `serving: true`, `unledgered: - * ['vector-coverage']` — the "I don't track this yet" posture). This is - * "as close as the JS engine allows": the gap is reproduced at the exact - * decision the fix changes, not approximated by deleting files the JS - * engine's own cold-start heuristic already recovers from unaided (see the - * inverse pin below and cold-open-rebuild-gate.test.ts's already-pinned - * "isReady()===true, size()===0" contract, which this fix deliberately does - * NOT touch — bare isReady() has no unledgered concept to hide behind, and - * overriding it would reopen the 48-seconds-per-restart regression pinned - * there). - * - * Pins: - * (1) COVERAGE GAP FORCES THE BUILD: N vectored nouns, a provider that - * claims `serving: true` at `size()===0` — open() builds anyway (the - * ledger proves there is something to cover), and search returns real - * results, never `[]`. - * (2) THE INVERSE, HONEST EMPTY: 0 vectored nouns (every embed still - * deferred/unlanded) — open() does NOT attempt a rebuild (nothing to - * load; the old blunt "always rebuild when size()===0" heuristic wasted - * a full canonical walk here for zero benefit), and search honestly - * returns `[]` — no error, no false coverage-gap narration. - * - * SEARCH VERIFICATION NOTE: pin (1) verifies "search returns real results" - * via `find({ query: })` (semantic search — embeds the query, then - * searches), matching the pattern `tests/integration/hnsw-rebuild.test.ts` - * already uses for exactly this "post-rebuild search works" class of pin. - * A raw `find({ vector: })` / `index.search(vector, k)` call was - * tried first and found to reproducibly return only 1 hit after a - * FROM-CANONICAL rebuild (never the full requested `limit`, sometimes not - * even a real neighbor) — REGARDLESS of this task's changes: it reproduces - * identically on a plain, unwrapped, un-stubbed reopen with the stock JS - * engine (verified against `hnsw-rebuild.test.ts`'s own construction) and - * is therefore a PRE-EXISTING, orthogonal defect in the JS HNSW engine's - * rebuilt-graph connectivity — outside this task's two deliverables (the - * count ledger and the open-gate REBUILD DECISION, not rebuild()'s internal - * search quality). Left for a separate investigation; not touched here. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vector-leg-open-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - vi.restoreAllMocks() - for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) -}) - -const V = (seed: number) => - Array.from({ length: 384 }, (_, i) => Math.sin((seed + 1) * 7919 + i * 131) * 0.5 + 0.5) - -/** - * Build a store with N explicit-vector (non-deferred) nouns, flush, close. - * Each noun also carries embeddable text (`technology`/`science`, matching - * the query used below) so the semantic-search verification exercises real - * retrieval, not a coincidental match. The default JS engine builds a fully - * current store — the epoch marker is stamped current at this open's - * completion, so a later reopen's `_indexEpochStale` is honestly false and - * cannot mask the ledger-gap decision under test (nothing here manufactures - * epoch drift). - */ -async function buildVectoredStore(dir: string, n: number): Promise { - const brain: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - await brain.init() - const ids: string[] = [] - for (let i = 0; i < n; i++) { - ids.push( - await brain.add({ - data: `doc ${i} about ${i % 2 === 0 ? 'technology' : 'science'}`, - type: 'document', - vector: V(i) - }) - ) - } - await brain.flush() - await brain.close() - return ids -} - -describe('vector-leg open-build (two-engine gate, last red)', () => { - it('coverage gap: a provider reporting serving:true at size()===0 is overridden by the vectored-noun ledger — open() builds, search returns real results', async () => { - const dir = mkTmp() - const ids = await buildVectoredStore(dir, 12) - - // Wrap the REAL JS engine so a successful rebuild restores genuine, - // searchable vectors — only `size()` and `healthReport()` are faked, - // simulating a native provider that has never built its own coverage of - // an unledgered invariant. - const calls = { rebuild: 0 } - const brain: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - brain.use({ - name: 'fake-native-vector-unledgered-coverage', - activate: async (ctx: any) => { - ctx.registerProvider('vector', (config: any, distance: any, options: any) => { - const real = new JsHnswVectorIndex(config, distance, options) - let rebuilt = false - const originalRebuild = real.rebuild.bind(real) - ;(real as any).rebuild = async (...args: any[]) => { - const r = await originalRebuild(...args) - calls.rebuild++ - rebuilt = true - return r - } - const originalSize = real.size.bind(real) - ;(real as any).size = () => (rebuilt ? originalSize() : 0) - ;(real as any).healthReport = () => ({ - provider: 'vector', - healthy: true, - serving: true, - invariants: [], - checkedAt: Date.now(), - durationMs: 0, - generation: 1, - unledgered: ['vector-coverage'] - }) - return real - }) - return true - } - }) - await brain.init() - - // WITHOUT any find() first: open() itself must have built the leg. - expect(calls.rebuild, 'open() forced the rebuild despite serving:true').toBe(1) - const status = await brain.getIndexStatus() - expect(status.hnswIndex.size).toBeGreaterThanOrEqual(ids.length) - - // Real, searchable results — never [] (see the module doc's SEARCH - // VERIFICATION NOTE for why this is a semantic `query`, not a raw - // `vector`, call). - const results = await brain.find({ query: 'technology document', limit: 5 }) - expect(results.length).toBeGreaterThan(0) - expect(results.length).not.toBe(0) - - await brain.close() - }) - - it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => { - // ARCHITECTURAL NOTE (updated by the zero-norm root cure): every brainy - // store carries ONE permanent VFS root noun beyond user data - // (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, - // src/vfs/VirtualFileSystem.ts), created (or, on a pre-fix store, - // migrated) on every open — but it is deliberately UNVECTORED (vector - // `[]`), never a real all-zero placeholder: a zero-norm vector is not a - // vector and never crosses an engine boundary (see that file's - // doInitializeRoot() comment). It therefore contributes NOTHING to the - // vectored-noun ledger — a brand-new store's `vectors.all` floor is 0, - // not 1. This pin verifies the law the task names in the ACHIEVABLE - // form: nouns whose embed is still deferred/unlanded contribute NOTHING - // to the vectored-noun ledger either — the coverage-gap comparison sees - // exactly the baseline (the root, contributing 0), never - // baseline+deferred — and semantic search over deferred-only user - // content honestly returns `[]` (no error, no false "coverage restored" - // claim). - const dir = mkTmp() - - const build: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - await build.init() - const rootOnlyLedger = await build.storage.getCanonicalCounts() - // THE NEW LAW: the root is unvectored — a brand-new store's floor is 0. - expect(rootOnlyLedger.vectors.all).toBe(0) - // Block the embedder permanently so every add below stays deferred and - // unlanded for the rest of this test (a fast deterministic embedder - // could otherwise land it before we ever observe the "still 0 extra" - // state). - vi.spyOn(build, 'embed').mockImplementation(() => new Promise(() => {})) - for (let i = 0; i < 5; i++) { - await build.add({ data: `deferred ${i}`, type: 'document', deferEmbedding: true }) - } - await build.flush() - const ledgerWithDeferred = await build.storage.getCanonicalCounts() - // The five deferred adds contributed ZERO to the vectored-noun ledger. - expect(ledgerWithDeferred.vectors.all).toBe(rootOnlyLedger.vectors.all) - await build.close() - - // Reopen (default JS engine — no stub needed): the root is the ONLY - // thing the vector leg has to load; the deferred nouns are correctly - // invisible to it. Block the embedder again BEFORE init() — reopen - // recovers the durable pending-embed markers and kicks the worker as - // part of init() itself, and an unblocked deterministic embedder could - // land all five before this test observes the open-time ledger. - const brain: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) - await brain.init() - // The ledger is exactly the root — the five deferred, still-unlanded - // nouns (which the rebuild above DOES insert into the graph, each with - // its stub empty vector — `index.size()` counts EVERY canonical noun's - // graph node, deferred or not, so it is not the coverage metric) never - // inflate the VECTORED count. - const ledgerAfterReopen = await brain.storage.getCanonicalCounts() - expect(ledgerAfterReopen.vectors.all).toBe(rootOnlyLedger.vectors.all) - - const results = await brain.find({ vector: V(3), limit: 5 }) - expect(results).toEqual([]) - - await brain.close() - }) -}) diff --git a/tests/integration/verb-metadata-rows.test.ts b/tests/integration/verb-metadata-rows.test.ts deleted file mode 100644 index ff08132a..00000000 --- a/tests/integration/verb-metadata-rows.test.ts +++ /dev/null @@ -1,226 +0,0 @@ -/** - * @module tests/integration/verb-metadata-rows - * @description THE LIVE VERB PATH pins. Before this train, verb rows entered - * the metadata index ONLY via `MetadataIndexManager.rebuild()`'s canonical - * walk — every relate()/unrelate()/updateRelation() call, and every - * remove()-cascaded relationship, left the metadata index blind to verb - * writes until the next rebuild. This file pins that `relate()`, - * `unrelate()`, `updateRelation()`, `remove()`'s cascade, and their - * `transact()` mirrors now post/retract the SAME verb rows a rebuild would - * derive from canonical (ADR-007 A4: one mechanism for add/update, live and - * rebuilt). - */ -process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' -import type { MetadataIndexManager } from '../../src/utils/metadataIndex.js' - -/** The JS metadata-index manager backing a memory-storage brain in these - * tests (feature-detected in production code via `instanceof - * MetadataIndexManager`; a narrow test-only reach-in here, matching the - * existing idiom in tests/integration/find-where-zero.test.ts and - * tests/integration/level-field-shadow.test.ts). */ -function metadataIndexOf(brain: Brainy): MetadataIndexManager { - return (brain as unknown as { metadataIndex: MetadataIndexManager }).metadataIndex -} - -describe('verb metadata rows — the live path matches the rebuild walk', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - }) - - async function addPerson(label: string): Promise { - return brain.add({ - data: `person ${label}`, - type: NounType.Person, - metadata: { label } - }) - } - - it('(a) relate() posts a metadata-index-backed verb row a query can find', async () => { - const a = await addPerson('a') - const b = await addPerson('b') - const relId = await brain.relate({ - from: a, to: b, type: VerbType.WorksWith, metadata: { role: 'lead' } - }) - - // Read it back the SAME way a rebuild-sourced row is queried — the - // manager's own posting lookup, keyed on the custom field the caller wrote. - const index = metadataIndexOf(brain) - expect(await index.getIds('role', 'lead')).toEqual([relId]) - }) - - it('(b) unrelate() retracts the row', async () => { - const a = await addPerson('a') - const b = await addPerson('b') - const relId = await brain.relate({ - from: a, to: b, type: VerbType.WorksWith, metadata: { role: 'lead' } - }) - - const index = metadataIndexOf(brain) - expect(await index.getIds('role', 'lead')).toEqual([relId]) - - // Flush BEFORE retracting the field's only occurrence: this durably - // persists the 'role' column (a segment on disk/in the store), so the - // post-retraction query below reads "this field exists, zero live - // postings" (→ []) rather than "this field has never been written" - // (→ FIELD_NOT_INDEXED) — an orthogonal column-store characteristic - // (an unflushed field with its last live posting removed reverts to - // unknown), not a D2 behavior. - await brain.flush() - - await brain.unrelate(relId) - - expect(await index.getIds('role', 'lead')).toEqual([]) - }) - - it('(c) updateRelation({ metadata }) leaves exactly the new values', async () => { - const a = await addPerson('a') - const b = await addPerson('b') - const relId = await brain.relate({ - from: a, to: b, type: VerbType.WorksWith, metadata: { role: 'lead', team: 'core' } - }) - - const index = metadataIndexOf(brain) - expect(await index.getIds('role', 'lead')).toEqual([relId]) - - // Flush first — see (b)'s note: 'role'/'team' must be durably known - // fields before their only value is retracted, or the post-update - // "gone" checks below throw FIELD_NOT_INDEXED instead of returning []. - await brain.flush() - - await brain.updateRelation({ id: relId, metadata: { role: 'reviewer' }, merge: false }) - - // Stale values gone (the old shape AND the merge:false-dropped field)… - expect(await index.getIds('role', 'lead')).toEqual([]) - expect(await index.getIds('team', 'core')).toEqual([]) - // …only the new value serves. - expect(await index.getIds('role', 'reviewer')).toEqual([relId]) - }) - - it("(d) remove(entity) cascade retracts every incident relation's metadata row", async () => { - const a = await addPerson('a') - const b = await addPerson('b') - const c = await addPerson('c') - const rel1 = await brain.relate({ - from: a, to: b, type: VerbType.WorksWith, metadata: { tag: 'cascade-test' } - }) - const rel2 = await brain.relate({ - from: c, to: a, type: VerbType.WorksWith, metadata: { tag: 'cascade-test' } - }) - - const index = metadataIndexOf(brain) - expect((await index.getIds('tag', 'cascade-test')).sort()).toEqual([rel1, rel2].sort()) - - // Flush first — see (b)'s note. - await brain.flush() - - await brain.remove(a) // a is source of rel1, target of rel2 — both cascade - - expect(await index.getIds('tag', 'cascade-test')).toEqual([]) - }) - - it('(e) a rebuild() reproduces exactly the verb-row population the live path built', async () => { - const a = await addPerson('a') - const b = await addPerson('b') - const c = await addPerson('c') - await brain.relate({ from: a, to: b, type: VerbType.WorksWith, metadata: { tag: 'parity', label: 'ab' } }) - await brain.relate({ from: b, to: c, type: VerbType.RelatedTo, metadata: { tag: 'parity', label: 'bc' } }) - const relId3 = await brain.relate({ - from: c, to: a, type: VerbType.WorksWith, metadata: { tag: 'parity', label: 'ca' } - }) - await brain.unrelate(relId3) // exercise retraction too — the rebuild must NOT resurrect it - - const index = metadataIndexOf(brain) - const beforeIds = (await index.getIds('tag', 'parity')).slice().sort() - expect(beforeIds.length).toBe(2) - const beforeAb = await index.getIds('label', 'ab') - const beforeBc = await index.getIds('label', 'bc') - - await index.rebuild() - - const afterIds = (await index.getIds('tag', 'parity')).slice().sort() - expect(afterIds).toEqual(beforeIds) - expect(await index.getIds('label', 'ab')).toEqual(beforeAb) - expect(await index.getIds('label', 'bc')).toEqual(beforeBc) - expect(await index.getIds('label', 'ca')).toEqual([]) // the unrelated edge stays gone - }) - - it('(f) transact() relate/unrelate posts/retracts the same metadata-index rows as single-op', async () => { - const a = await addPerson('a') - const b = await addPerson('b') - const c = await addPerson('c') - const d = await addPerson('d') - - // Single-op baseline. - const singleOpId = await brain.relate({ - from: a, to: b, type: VerbType.WorksWith, metadata: { tag: 'parity-f' } - }) - - // transact() mirror. - const relateDb = await brain.transact([ - { op: 'relate', from: c, to: d, type: VerbType.WorksWith, metadata: { tag: 'parity-f' } } - ]) - const transactId = relateDb.receipt!.ids[0] - await relateDb.release() - - const index = metadataIndexOf(brain) - expect((await index.getIds('tag', 'parity-f')).sort()).toEqual([singleOpId, transactId].sort()) - - // Flush first — see (b)'s note: 'tag' must be durably known before its - // last live posting is retracted below. - await brain.flush() - - // Retract both ways — single-op unrelate() and transact() unrelate. - await brain.unrelate(singleOpId) - const unrelateDb = await brain.transact([{ op: 'unrelate', id: transactId }]) - await unrelateDb.release() - - expect(await index.getIds('tag', 'parity-f')).toEqual([]) - }) - - - it('the metadata crossing never carries BigInt endpoint ints — a cascade delete after graph resolution survives JSON', async () => { - // resolveVerbEndpointInts MIRRORS the resolved u64 ints onto the verb - // object as BigInt (verb.sourceInt/targetInt). A provider that JSON- - // serializes the metadata crossing dies on BigInt — found by the first - // joint pair gate. This pin drives the exact shape: relate (graph legs - // resolve ints), then remove the source entity (the cascade passes the - // SAME verb object to the retraction), through a provider shim that - // enforces the JSON-safety contract the way a native provider does. - const employee = await brain.add({ data: 'cascade employee', type: 'person' }) - const invoice = await brain.add({ data: 'cascade invoice', type: 'document' }) - await brain.relate({ from: employee, to: invoice, type: 'relatedTo' }) - const mgr: any = (brain as any).metadataIndex - const origRemove = mgr.removeFromIndex.bind(mgr) - const seen: unknown[] = [] - mgr.removeFromIndex = async (id: string, metadata?: unknown, generation?: bigint) => { - seen.push(metadata) - JSON.stringify(metadata) // the contract: throws on BigInt, exactly like a native crossing - return origRemove(id, metadata, generation) - } - try { - await brain.remove(employee) // cascades the relation's retraction - } finally { - mgr.removeFromIndex = origRemove - } - expect(seen.length).toBeGreaterThan(0) - for (const m of seen) { - if (m && typeof m === 'object') { - for (const [k, v] of Object.entries(m as Record)) { - expect(typeof v, `metadata key ${k} must be JSON-safe`).not.toBe('bigint') - } - } - } - }) - -}) diff --git a/tests/integration/vfs-root-sweep-once.test.ts b/tests/integration/vfs-root-sweep-once.test.ts deleted file mode 100644 index cac59b70..00000000 --- a/tests/integration/vfs-root-sweep-once.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -/** - * @module tests/integration/vfs-root-sweep-once - * @description THE OLD-ROOT SWEEP RUNS ONCE PER STORE, NOT ONCE PER OPEN. - * - * The VFS bootstrap ran a filtered `find()` over the whole store on EVERY - * open, hunting for root directories created before the fixed root id existed - * — duplicates a store has either always had or never will. MEASURED on a - * 14,056-noun / 72,679-verb store: the phase it dominates cost 43–53 SECONDS - * of every open, warm reopens included. - * - * The law: a migration sweep is caused by the store's state, not by the clock - * or the open count. It runs behind the doors, records that it ran, and a - * store carrying that record never sweeps again. - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync, existsSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' -import { prodLog } from '../../src/utils/logger.js' - -describe('the VFS old-root sweep', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - vi.restoreAllMocks() - }) - - async function open(dir: string): Promise { - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - return brain - } - - it('sweeps on the first open, records it, and never sweeps again', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-')) - dirs.push(dir) - - const sweepSpy = vi.spyOn( - VirtualFileSystem.prototype as unknown as { cleanupOldRoots: () => Promise }, - 'cleanupOldRoots' - ) - - const first = await open(dir) - await (first.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - expect(sweepSpy).toHaveBeenCalledTimes(1) - // The record is durable engine plumbing under _system/, like every other marker. - expect( - existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || - existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) - ).toBe(true) - - await first.add({ data: 'a row so the store is not trivially empty', type: NounType.Concept }) - await first.flush() - await first.close() - brains.splice(brains.indexOf(first), 1) - - sweepSpy.mockClear() - const second = await open(dir) - await (second.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - expect(sweepSpy).not.toHaveBeenCalled() - - await second.close() - brains.splice(brains.indexOf(second), 1) - - // ...and a third open, to prove it is the record and not a one-off. - sweepSpy.mockClear() - const third = await open(dir) - await (third.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - expect(sweepSpy).not.toHaveBeenCalled() - }, 180_000) - - it('a sweep that removes nothing on a fresh store says nothing', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-quiet-')) - dirs.push(dir) - - // The always-visible channel cannot be silenced by a log level, so a line - // on it has to earn its place. A fresh store's sweep finds no duplicate - // roots and costs a millisecond — it must do its work, record its marker, - // and stay quiet, or it trains operators to ignore the one channel that - // exists to be impossible to ignore. - const narrated: string[] = [] - const spy = vi.spyOn(prodLog, 'narrate').mockImplementation(((...args: unknown[]) => { - narrated.push(args.map((a) => String(a)).join(' ')) - }) as typeof prodLog.narrate) - - const brain = await open(dir) - await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - spy.mockRestore() - - expect(narrated.filter((l) => /old-root sweep/i.test(l))).toEqual([]) - // ...and it still did the work: the marker is recorded, so no future open sweeps. - expect( - existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || - existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) - ).toBe(true) - }, 180_000) - - it('the open does not wait for the sweep', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-')) - dirs.push(dir) - - const proto = VirtualFileSystem.prototype as unknown as Record< - string, - (...args: unknown[]) => Promise - > - const real = proto.cleanupOldRoots - proto.cleanupOldRoots = async function slow(this: unknown, ...args: unknown[]) { - await new Promise((r) => setTimeout(r, 4_000)) - return real.apply(this, args) - } - try { - const startedAt = Date.now() - const brain = await open(dir) - const openMs = Date.now() - startedAt - expect(openMs).toBeLessThan(3_000) - await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - } finally { - proto.cleanupOldRoots = real - } - }, 180_000) -}) diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts deleted file mode 100644 index 577ae7ee..00000000 --- a/tests/integration/vfs-root-zero-norm.test.ts +++ /dev/null @@ -1,216 +0,0 @@ -/** - * @module tests/integration/vfs-root-zero-norm - * @description THE ZERO-NORM ROOT CURE — a production incident traced 150+ - * darkened rows in a downstream engine's index to the VFS root's persisted - * ALL-ZERO placeholder vector: lawful inside brainy (`cosineDistance` - * treats a zero-norm operand as MAXIMUM distance, src/utils/distance.ts) - * but a "false attractor" for an engine serving squared-euclidean distance, - * which cannot tell a real all-zero vector apart from a legitimate origin - * point. THE LAW: a zero-norm vector is not a vector — it never crosses an - * engine boundary. - * - * Three legs pinned here: - * (a) the root persists NO zeros — a brand-new store creates it with - * vector `[]` (the "unvectored" shape), absent from the HNSW index, and - * the canonical vectored-noun ledger does not count it. - * (b) a ONE-TIME migration heals an existing (pre-fix) store: an old-shape - * root (a REAL all-zero vector, genuinely indexed and ledgered — the - * harness reproduces exactly what a pre-fix store looked like on disk) - * is rewritten to `[]` on the next `init()`, the ledger is decremented - * through the sanctioned path, and a second `init()` is a no-op. - * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up - * zero-norm/unvector-door fix): an entity added with an EXPLICIT - * all-zero vector (any dimension) is normalized to the "unvectored" - * `[]` shape BEFORE the canonical write, the ledger flag, and the index - * ops ever see it — the canonical write still succeeds, loudly, and the - * vector-index insert never happens (nothing to index). Supersedes the - * original "canonical keeps the zero vector, only the index refuses" - * shape: a downstream engine's health-report gate reads the canonical - * ledger directly, so leaving a zero-norm vector on the canonical side - * re-opened the exact false-attractor risk this whole fix closes. - * (d) the migrated root never surfaces in `find()` results (it was already - * hidden behind `visibility: 'system'` — this pin holds regardless). - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' -import { prodLog } from '../../src/utils/logger.js' - -const ROOT_ID = '00000000-0000-0000-0000-000000000000' - -process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vfs-root-zero-norm-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - vi.restoreAllMocks() - for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) -}) - -function openBrain(dir: string): any { - return new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) -} - -describe('VFS root zero-norm cure', () => { - it('(a) a brand-new store persists the root with vector [], absent from the HNSW index, and the canonical ledger counts it unvectored', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const root = await brain.get(ROOT_ID, { includeVectors: true }) - expect(root).not.toBeNull() - expect(root.vector).toEqual([]) - - const status = await brain.getIndexStatus() - expect(status.hnswIndex.size).toBe(0) - - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(0) - - await brain.close() - }) - - it('(b) an old-shape store (a real all-zero placeholder root) migrates to [] exactly once on init; the ledger is decremented through the sanctioned path; a second init is a no-op', async () => { - const dir = mkTmp() - - // SESSION 1 — build the store, then hand-rewrite the root to the LEGACY - // shape: a REAL all-zero 384-dim vector, genuinely inserted into the - // vector index and genuinely counted by the vectored-noun ledger — - // reproducing exactly what a pre-fix store's root looked like on disk - // (the pre-fix add() always indexed + counted it). `index.addItem` is - // called directly (bypassing AddToVectorIndexOperation's own zero-norm - // belt, added by this same fix) precisely because the pre-fix code path - // had no such belt — this harness must match history, not the cure. - let brain = openBrain(dir) - await brain.init() - const oldVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) - await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) - await brain.storage.noteVectorLanded(ROOT_ID) - await brain.storage.persistCounts() - await brain.flush() - - const ledgerBeforeMigration = await brain.storage.getCanonicalCounts() - expect(ledgerBeforeMigration.vectors.all).toBe(1) - await brain.close() - - // SESSION 2 — reopen: VFS init must detect the legacy shape and migrate. - // Spy on the sanctioned migration method itself (not console output — - // `silent: true` monkey-patches `console.log` to a no-op INSIDE init(), - // which would silently swallow any pre-installed console spy too). - brain = openBrain(dir) - const migrateSpy = vi.spyOn(brain, 'unvectorNounForRootMigration') - await brain.init() - - expect(migrateSpy).toHaveBeenCalledTimes(1) - expect(migrateSpy).toHaveBeenCalledWith(ROOT_ID) - await expect(migrateSpy.mock.results[0].value).resolves.toBe(true) - - const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) - expect(migratedRoot.vector).toEqual([]) - - const ledgerAfterMigration = await brain.storage.getCanonicalCounts() - expect(ledgerAfterMigration.vectors.all).toBe(0) - - const statusAfterMigration = await brain.getIndexStatus() - expect(statusAfterMigration.hnswIndex.size).toBe(0) - - await brain.flush() - await brain.close() - - // SESSION 3 — reopen again: the migration is a permanent no-op, not a - // one-time flag that silently re-drifts or re-fires. The zero-norm - // detection at the VFS init site never even calls the migration method - // again — the root's vector is already `[]`. - brain = openBrain(dir) - const migrateSpy2 = vi.spyOn(brain, 'unvectorNounForRootMigration') - await brain.init() - - expect(migrateSpy2).not.toHaveBeenCalled() - - const rootAfterSecondInit = await brain.get(ROOT_ID, { includeVectors: true }) - expect(rootAfterSecondInit.vector).toEqual([]) - - const ledgerAfterSecondInit = await brain.storage.getCanonicalCounts() - expect(ledgerAfterSecondInit.vectors.all).toBe(0) - - await brain.close() - }) - - it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const warnSpy = vi.spyOn(prodLog, 'warn') - - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - const ledgerBefore = await brain.storage.getCanonicalCounts() - const zeroVector = new Array(384).fill(0) - const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) - - // The canonical write succeeded — but the zero-norm vector was - // normalized to the "unvectored" `[]` shape BEFORE it was persisted - // (Leg A: a zero-norm vector is not a vector — it never crosses an - // engine boundary, canonical side included). - const entity = await brain.get(id, { includeVectors: true }) - expect(entity).not.toBeNull() - expect(entity.vector).toEqual([]) - - // Nothing to index — the vector-index size never moved, and the - // vectored-noun ledger never counted this row. - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore) - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - - // The normalization was LOUD and named the entity. - const loudCall = warnSpy.mock.calls.find( - (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') - ) - expect(loudCall).toBeDefined() - - await brain.close() - }) - - it('(d) find() over a store whose root has been migrated never returns the root (already hidden behind visibility: system — pinned anyway)', async () => { - const dir = mkTmp() - - // Build an old-shape store (same harness as pin (b)) and let it migrate. - let brain = openBrain(dir) - await brain.init() - const oldVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) - await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) - await brain.storage.noteVectorLanded(ROOT_ID) - await brain.storage.persistCounts() - await brain.add({ data: 'a document about technology', type: NounType.Document }) - await brain.flush() - await brain.close() - - brain = openBrain(dir) // migrates on init() - await brain.init() - - const results = await brain.find({ query: 'technology', limit: 10 }) - expect(results.some((r: any) => r.id === ROOT_ID)).toBe(false) - - // Even asking explicitly for system-tier entities must never surface the - // root as a semantic-search HIT (it carries no vector to match against). - const resultsIncludingSystem = await brain.find({ query: 'technology', limit: 10, includeSystem: true }) - expect(resultsIncludingSystem.some((r: any) => r.id === ROOT_ID)).toBe(false) - - await brain.close() - }) -}) diff --git a/tests/integration/writer-lock-clean-close.test.ts b/tests/integration/writer-lock-clean-close.test.ts deleted file mode 100644 index 7d9c59d6..00000000 --- a/tests/integration/writer-lock-clean-close.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** - * @module tests/integration/writer-lock-clean-close - * @description THE CLEAN-CLOSE CONTRACT for the writer lock. - * - * A production restart made this lane necessary: a service stopped with exit - * code 0, having awaited `close()` on every pooled brain, and its next boot - * announced `[brainy] Overwriting stale writer lock … appears dead` for every - * store it owned. "The pid is gone" is equally true of an orderly restart and - * of a crash, so the message could not tell an operator which one they had. - * - * The contract pinned here: - * 1. A completed close leaves NO lock file and DOES leave a clean-close - * record; the next open says nothing about staleness. - * 2. The next lock claim CONSUMES that record — it may never outlive the - * lock generation it describes, or a later crash would read as clean. - * 3. A close whose durable steps FAIL still releases the lock (and still - * rethrows the failure). - * 4. A killed process (SIGKILL, no close at all) leaves the lock behind with - * NO record, and the next open says exactly that — crash, recovery ahead. - * 5. A host application with its own SIGTERM handler is never force-exited - * out from under its own shutdown by Brainy's handler. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' -import { spawn } from 'node:child_process' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -const REPO_ROOT = process.cwd() -const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') - -function makeTempDir(): string { - return mkdtempSync(join(tmpdir(), 'brainy-clean-close-')) -} - -/** - * Write a child script to disk and start it under tsx. A file (not `tsx -e`) - * because the eval form compiles to CommonJS, which has no top-level await. - * The script imports Brainy by ABSOLUTE path, so its own dependency - * resolution still happens from inside the repository. - */ -function startChild(dir: string, body: string): ReturnType { - const scriptPath = join(dir, 'child-process.mts') - writeFileSync(scriptPath, body) - // `detached` puts the child in its own process GROUP: tsx runs the script in - // a grandchild process, and only a group-wide signal reaches the process - // that actually holds the writer lock. - return spawn(TSX, [scriptPath], { - cwd: REPO_ROOT, - stdio: ['ignore', 'pipe', 'pipe'], - detached: true - }) -} - -/** Capture every console.warn/error line emitted while `fn` runs. */ -async function captureConsole(fn: () => Promise): Promise<{ result: T; lines: string[] }> { - const lines: string[] = [] - const origWarn = console.warn - const origError = console.error - const sink = (...args: unknown[]) => { - lines.push(args.map((a) => String(a)).join(' ')) - } - console.warn = sink as typeof console.warn - console.error = sink as typeof console.error - try { - const result = await fn() - return { result, lines } - } finally { - console.warn = origWarn - console.error = origError - } -} - -/** - * Run a child process that opens `dir`, writes one row, prints `READY`, and - * then waits forever. Resolves with the child once READY is seen. - */ -function spawnHoldingChild(dir: string): Promise<{ - child: ReturnType - output: () => string -}> { - const script = ` - import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) - await brain.init() - await brain.add({ data: 'row from the child', type: 'concept' }) - await brain.flush() - console.log('READY') - setInterval(() => {}, 1000) - ` - const child = startChild(dir, script) - let out = '' - child.stdout.on('data', (d) => { out += String(d) }) - child.stderr.on('data', (d) => { out += String(d) }) - return new Promise((resolvePromise, rejectPromise) => { - const timer = setTimeout(() => rejectPromise(new Error(`child never became READY:\n${out}`)), 120_000) - child.stdout.on('data', () => { - if (out.includes('READY')) { - clearTimeout(timer) - resolvePromise({ child, output: () => out }) - } - }) - child.on('exit', (code) => { - clearTimeout(timer) - if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`)) - }) - }) -} - -describe('writer lock — the clean-close contract', () => { - let dir: string - let brain: Brainy | null = null - - beforeEach(() => { dir = makeTempDir() }) - - afterEach(async () => { - if (brain) { - try { await brain.close() } catch { /* may already be closed */ } - brain = null - } - try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } - }) - - const lockPath = () => join(dir, 'locks', '_writer.lock') - const recordPath = () => join(dir, 'locks', '_writer.close') - - it('a completed close leaves no lock, leaves a record, and the reopen is silent about staleness', async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - expect(existsSync(lockPath())).toBe(true) - - await brain.add({ data: 'seed entity', type: NounType.Concept }) - await brain.flush() - await brain.close() - brain = null - - // 1. The lock is gone and the release is RECORDED. - expect(existsSync(lockPath())).toBe(false) - expect(existsSync(recordPath())).toBe(true) - const record = JSON.parse(readFileSync(recordPath(), 'utf-8')) - expect(record.pid).toBe(process.pid) - expect(typeof record.closedAt).toBe('string') - expect(typeof record.startedAt).toBe('string') - - // 2. The reopen says nothing about a stale lock. - const { result: reopened, lines } = await captureConsole(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - return next - }) - brain = reopened - expect(lines.filter((l) => /stale writer lock|appears dead/i.test(l))).toEqual([]) - - // 3. The claim CONSUMED the record — it must not outlive its lock generation. - expect(existsSync(recordPath())).toBe(false) - expect(existsSync(lockPath())).toBe(true) - }, 120_000) - - it('releases the writer lock even when a durable close step fails — and still rethrows', async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - await brain.add({ data: 'seed entity', type: NounType.Concept }) - await brain.flush() - expect(existsSync(lockPath())).toBe(true) - - // Inject a failure into a durable close step (the counts flush). - const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage - const boom = new Error('injected: counts flush failed during close') - storage.flushCounts = async () => { throw boom } - - await expect(brain.close()).rejects.toThrow(/injected: counts flush failed/) - brain = null - - // The lock is released regardless: a process on its way out holds nothing. - expect(existsSync(lockPath())).toBe(false) - - // And the next writer opens without a stale-lock verdict. - const { lines } = await captureConsole(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - await next.close() - }) - expect(lines.filter((l) => /appears dead/i.test(l))).toEqual([]) - }, 120_000) - - it('a SIGKILLed writer leaves the lock with no record, and the next open names the crash', async () => { - const { child } = await spawnHoldingChild(dir) - expect(existsSync(lockPath())).toBe(true) - expect(existsSync(recordPath())).toBe(false) - - // Group-wide: the lock holder is tsx's grandchild, not the spawned pid. - process.kill(-(child.pid as number), 'SIGKILL') - await new Promise((r) => child.on('exit', () => r())) - // The grandchild's death is asynchronous with the wrapper's exit event. - await new Promise((r) => setTimeout(r, 500)) - - // The lock survives the kill — a dead process releases nothing. - expect(existsSync(lockPath())).toBe(true) - expect(existsSync(recordPath())).toBe(false) - - const { lines } = await captureConsole(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - await next.close() - }) - const verdict = lines.filter((l) => /Overwriting stale writer lock/i.test(l)) - expect(verdict.length).toBe(1) - // The verdict must name the ABSENT record and the recovery it implies — - // not merely that a pid is gone. - expect(verdict[0]).toMatch(/NO\s+clean-close record/i) - expect(verdict[0]).toMatch(/crash recovery/i) - }, 180_000) - - it("does not force-exit a host application that owns its own SIGTERM handler", async () => { - const script = ` - import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) - await brain.init() - await brain.add({ data: 'row from the host app', type: 'concept' }) - await brain.flush() - // The host application's OWN graceful shutdown, registered after Brainy's. - process.on('SIGTERM', async () => { - await new Promise((r) => setTimeout(r, 1500)) - console.log('APP-CLOSE-DONE') - process.exit(0) - }) - console.log('READY') - setInterval(() => {}, 1000) - ` - const child = startChild(dir, script) - let out = '' - child.stdout.on('data', (d) => { out += String(d) }) - child.stderr.on('data', (d) => { out += String(d) }) - await new Promise((r, reject) => { - const timer = setTimeout(() => reject(new Error(`child never became READY:\n${out}`)), 120_000) - child.stdout.on('data', () => { if (out.includes('READY')) { clearTimeout(timer); r() } }) - child.on('exit', () => { clearTimeout(timer); if (!out.includes('READY')) reject(new Error(`child died:\n${out}`)) }) - }) - - process.kill(-(child.pid as number), 'SIGTERM') - const code = await new Promise((r) => child.on('exit', (c) => r(c))) - expect(code).toBe(0) - // The host's own shutdown ran to completion — Brainy's handler did not - // exit the process out from under it. - expect(out).toContain('APP-CLOSE-DONE') - }, 180_000) -}) diff --git a/tests/integration/zero-norm-unvector-door.test.ts b/tests/integration/zero-norm-unvector-door.test.ts deleted file mode 100644 index d3290010..00000000 --- a/tests/integration/zero-norm-unvector-door.test.ts +++ /dev/null @@ -1,399 +0,0 @@ -/** - * @module tests/integration/zero-norm-unvector-door - * @description THE SEAM LAW, GENERALIZED: "a zero-norm vector is not a - * vector — it never crosses an engine boundary." `tests/integration/ - * vfs-root-zero-norm.test.ts` pins the VFS-root-specific cure; this file - * pins the follow-up that generalizes it to every write path plus the - * sanctioned door for shedding a vector on purpose. - * - * Four legs pinned here: - * (A) THE CANONICAL WRITE NORMALIZES ZERO-NORM TO `[]` — `add()` (single and - * `transact()`) persists an explicit real all-zero vector as the - * "unvectored" `[]` shape, loudly, before the ledger flag/dimension - * pin/index ops ever see it. The canonical write still succeeds. - * (B) THE LEGACY DERIVATION IS ZERO-NORM-AWARE — a lost/corrupted - * `counts.json`'s one-time re-derivation walk excludes a persisted - * zero-norm row from the vectored-noun scalar, matching the live - * ledger's definition of "vectored". - * (C) THE LEGACY VFS ROOT MIGRATES AT OPEN, BEFORE THE GATE, IN O(1) — a - * store whose ONLY vectored row is a legacy all-zero VFS root opens - * clean (no `VectorIndexNotReadyError`), via one fixed-path read, never - * a listing. - * (D) THE UNVECTOR DOOR — `update({ id, vector: [] })` (and the same op - * inside `transact()`) is the sanctioned, idempotent way to shed a - * vector on purpose: ledger decrement exactly once, index removal, no - * re-embed, and a pending deferred-embed marker is cleared rather than - * left to re-vectorize the row later. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' -import { prodLog } from '../../src/utils/logger.js' -import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' -import { BaseStorage } from '../../src/storage/baseStorage.js' - -const ROOT_ID = '00000000-0000-0000-0000-000000000000' - -process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-zero-norm-unvector-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - vi.restoreAllMocks() - for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) -}) - -function openBrain(dir: string): any { - return new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) -} - -const countsPath = (root: string) => path.join(root, '_system', 'counts.json') - -describe('zero-norm canonical write + the sanctioned unvector door', () => { - it('(A1) add() with an explicit all-zero vector persists [], warns loudly, never indexes, and the ledger is unchanged', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - const warnSpy = vi.spyOn(prodLog, 'warn') - - const zeroVector = new Array(384).fill(0) - const id = await brain.add({ data: 'zero-norm add', type: NounType.Document, vector: zeroVector }) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity).not.toBeNull() - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore) - - const loud = warnSpy.mock.calls.find( - (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') - ) - expect(loud).toBeDefined() - - await brain.close() - }) - - it('(A2) transact() add with an explicit all-zero vector — the same canonical normalization', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const warnSpy = vi.spyOn(prodLog, 'warn') - const zeroVector = new Array(384).fill(0) - const id = 'aaaaaaaa-0000-4000-8000-000000000001' - - await brain.transact([ - { op: 'add', id, type: NounType.Document, data: 'zero-norm transact add', vector: zeroVector } - ]) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity).not.toBeNull() - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - - const loud = warnSpy.mock.calls.find( - (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') - ) - expect(loud).toBeDefined() - - await brain.close() - }) - - it('(B) the legacy counts.json derivation excludes a persisted zero-norm row from the vectored-noun scalar', async () => { - const dir = mkTmp() - let brain = openBrain(dir) - await brain.init() - - // The VFS root alone (unvectored — []) — the floor. - const baseline = (await brain.storage.getCanonicalCounts()).vectors.all - - const realId = await brain.add({ data: 'a real vectored document', type: NounType.Document }) - - // Plant the legacy all-zero shape BY HAND: a genuine identity record - // (via add(), so it has real metadata) whose vector leg is then - // overwritten directly through the raw storage primitive — bypassing - // Leg A's canonical-write normalization entirely (brain.storage.saveNoun - // is not Brainy.add()/update()'s normalized path) — reproducing exactly - // what a pre-fix store could have persisted on disk. - const zeroId = await brain.add({ data: 'a legacy zero-norm document', type: NounType.Document }) - const zeroVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: zeroId, vector: zeroVector, connections: new Map(), level: 0 }) - - await brain.flush() - await brain.close() - - // Remove counts.json so the next open re-derives from scratch (the - // one-time legacy/lost-file derivation path — Leg B). - fs.rmSync(countsPath(dir), { force: true }) - - brain = openBrain(dir) - await brain.init() - const ledger = await brain.storage.getCanonicalCounts() - // Only realId counts; zeroId's persisted all-zero vector does not. - expect(ledger.vectors.all).toBe(baseline + 1) - - await brain.close() - }) - - it('(C) a legacy all-zero VFS root as the ONLY vectored row: open succeeds with no not-ready error, via an O(1) fixed-path read (no entities-tree readdir), and the ledger is 0 after open', async () => { - const dir = mkTmp() - - // SESSION 1 — build the legacy shape: the root is a REAL all-zero - // 384-dim vector, genuinely indexed and genuinely ledgered — exactly - // what a pre-fix store's root looked like on disk (see - // vfs-root-zero-norm.test.ts pin (b) for the identical harness). - // `index.addItem` is called directly (bypassing the transactional - // zero-norm belt) because the pre-fix code path had no such belt — this - // harness must match history, not the cure. No other entity is added, - // so the root is the store's ONLY vectored row. - let brain = openBrain(dir) - await brain.init() - const oldVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) - await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) - await brain.storage.noteVectorLanded(ROOT_ID) - await brain.storage.persistCounts() - await brain.flush() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(1) - await brain.close() - - // SESSION 2 — reopen with a FAKE native vector provider that claims - // `serving: true` at `size()===0` (the exact shape a downstream - // engine's own health report can legitimately carry — same technique as - // tests/integration/vector-leg-open-build.test.ts). This is the ONLY - // codepath where the vector-leg open gate's FAIL-TYPED throw - // (VectorIndexNotReadyError) can fire; the built-in JS engine alone - // never reaches it (the size-heuristic branch just rebuilds instead) — - // so this is the faithful reproduction of the incident Leg C closes. - const readdirCalls: string[] = [] - const originalReaddir = fs.promises.readdir.bind(fs.promises) - vi.spyOn(fs.promises, 'readdir').mockImplementation(((...args: any[]) => { - readdirCalls.push(String(args[0])) - return (originalReaddir as any)(...args) - }) as any) - - // Spy at the PROTOTYPE level (BaseStorage.getNoun) — the new brain's - // storage instance does not exist until init() runs, so an - // instance-level spy cannot be installed beforehand. Records the - // readdir-call delta across the FIRST call made with the root id — - // Leg C's own fixed-path read — proving it needs no directory listing. - let readdirDeltaDuringRootRead: number | null = null - const originalGetNoun = BaseStorage.prototype.getNoun - vi.spyOn(BaseStorage.prototype, 'getNoun').mockImplementation(async function ( - this: unknown, - id: string - ) { - const before = readdirCalls.length - const result = await originalGetNoun.call(this as BaseStorage, id) - if (id === ROOT_ID && readdirDeltaDuringRootRead === null) { - readdirDeltaDuringRootRead = readdirCalls.length - before - } - return result - }) - - brain = openBrain(dir) - brain.use({ - name: 'fake-native-vector-unledgered-coverage', - activate: async (ctx: any) => { - ctx.registerProvider('vector', (config: any, distance: any, options: any) => { - const real = new JsHnswVectorIndex(config, distance, options) - let rebuilt = false - const originalRebuild = real.rebuild.bind(real) - ;(real as any).rebuild = async (...args: any[]) => { - const r = await originalRebuild(...args) - rebuilt = true - return r - } - const originalSize = real.size.bind(real) - ;(real as any).size = () => (rebuilt ? originalSize() : 0) - ;(real as any).healthReport = () => ({ - provider: 'vector', - healthy: true, - serving: true, - invariants: [], - checkedAt: Date.now(), - durationMs: 0, - generation: 1, - unledgered: ['vector-coverage'] - }) - return real - }) - return true - } - }) - - // Must NOT throw VectorIndexNotReadyError (or anything else) — a - // near-empty store whose only vectored row is the zero-norm root must - // never go dark. - await brain.init() - - const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) - expect(migratedRoot.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(0) - - expect(readdirDeltaDuringRootRead).toBe(0) - - await brain.close() - }) - - describe('the sanctioned unvector door', () => { - it('(D1) update({ id, vector: [] }) unvectors a real vectored row — canonical [], removed from the index, ledger decremented by exactly 1, no embed call', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document', type: NounType.Document }) - await brain.flush() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - - const embedSpy = vi.spyOn(brain, 'embed') - await brain.update({ id, vector: [] }) - expect(embedSpy).not.toHaveBeenCalled() - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) - - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore - 1) - - await brain.close() - }) - - it('(D2) idempotent: a second update({ id, vector: [] }) on an already-unvectored row is a true no-op — no error, no further decrement', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document', type: NounType.Document }) - await brain.flush() - - await brain.update({ id, vector: [] }) - const ledgerAfterFirst = await brain.storage.getCanonicalCounts() - - await brain.update({ id, vector: [] }) - const ledgerAfterSecond = await brain.storage.getCanonicalCounts() - expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfterFirst.vectors.all) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - await brain.close() - }) - - it('(D3) a PENDING deferred-embed row: the unvector door clears the marker; awaitPendingEmbeds() then leaves it unvectored', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - // Prevent the background worker from ever actually running — it is - // fire-and-forget from add(), and a real run would race this test's - // own assertions (see tests/integration/vector-leg-open-build.test.ts - // for the same concern). This isolates exactly the marker-clearing - // behavior under test. - vi.spyOn(brain as any, 'kickEmbedWorker').mockImplementation(() => {}) - - const id = await brain.add({ - data: 'deferred content, never embedded', - type: NounType.Document, - deferEmbedding: true - }) - expect(brain.pendingEmbedCount()).toBe(1) - - const warnSpy = vi.spyOn(prodLog, 'warn') - await brain.update({ id, vector: [] }) - - expect(brain.pendingEmbedCount()).toBe(0) - const clearedWarn = warnSpy.mock.calls.find( - (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('pending') - ) - expect(clearedWarn).toBeDefined() - - // The barrier must not hang and must not re-vectorize the row — the - // worker (still mocked to a no-op) never runs again. - await brain.awaitPendingEmbeds() - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - await brain.close() - }) - - it('(D4) update({ vector: [], deferEmbedding: true }) is a typed refusal — the unvector door cannot be paired with a deferred embed', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document', type: NounType.Document }) - const before = await brain.get(id, { includeVectors: true }) - - await expect( - brain.update({ id, vector: [], deferEmbedding: true }) - ).rejects.toThrow(/unvector door/i) - - // Refused before any write — the row is untouched. - const after = await brain.get(id, { includeVectors: true }) - expect(after.vector).toEqual(before.vector) - - await brain.close() - }) - - it('(D5) the transact() twin of the unvector door decrements the ledger exactly once, and is idempotent on a second call', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document for transact unvector', type: NounType.Document }) - await brain.flush() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - - await brain.transact([{ op: 'update', id, vector: [] }]) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) - - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore - 1) - - // Idempotent through transact() too. - await brain.transact([{ op: 'update', id, vector: [] }]) - const ledgerAfterSecond = await brain.storage.getCanonicalCounts() - expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfter.vectors.all) - - await brain.close() - }) - }) -}) diff --git a/tests/lifecycle/README.md b/tests/lifecycle/README.md deleted file mode 100644 index ebe0e61d..00000000 --- a/tests/lifecycle/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# The Lifecycle Lane - -One brain, driven through founding, a working day, a clean restart, a -crash, a repair, and a second life, checked chapter by chapter against an -independent shadow-model referee (`biographyHarness.ts`). It catches -COMPOSITION regressions unit tests miss — a store fine in one process but -broken across a restart/crash/repair. Runs on the plain JS engine, so it -gates every commit. - -Run it: `npx vitest run tests/lifecycle --pool=forks` - -A red names the chapter label, the id, and expected-vs-actual — diagnosable -from the message alone. `biography.test.ts` is split into two `it` blocks -(Ch1-3, then Ch4-6) purely for reporting; it is still ONE fixed-order story. -Chapters must never be reordered, skipped, or made conditional, and a -failing chapter's assertion must never be weakened to force green. - -Lab notes (hard-won, keep): -- `git reset --hard` does NOT remove untracked files — a "clean" tree can still - carry stray test stores; use `git clean -fd tests/lifecycle-tmp` equivalents. -- `silent: true` patches `console` process-wide — never assert narration through - `console` spies in this lane; the engine's always-on channel is `prodLog`. diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts deleted file mode 100644 index 274f0ef0..00000000 --- a/tests/lifecycle/biography.test.ts +++ /dev/null @@ -1,418 +0,0 @@ -/** - * @module tests/lifecycle/biography - * @description THE LIFECYCLE LANE — see `tests/lifecycle/README.md` for what - * this proves and how to run it. One scenario, "the working store": a single - * brain driven through founding, a working day, a clean restart, a crash, a - * repair, and a second life, verified chapter by chapter against an - * independent shadow-model referee (`biographyHarness.ts`). - * - * Split into two `it` blocks so a currently-failing later chapter (see the - * second block's header comment — a live engine finding, not a defect in - * this lane) never hides the earlier chapters' passing coverage. The two - * blocks share one brain's directory and one shadow model, run in the SAME - * fixed order the single scenario always has (`describe.sequential` below - * exists to say so explicitly, though vitest's own default is sequential - * within a file) — this is a split for REPORTING clarity, not a reordering - * or conditional skip of any chapter. - */ -import { describe, it, expect } from 'vitest' -import * as fs from 'node:fs' -import { NounType, VerbType } from '../../src/types/graphTypes.js' -import type { Brainy } from '../../src/brainy.js' -import type { AddParams, RelateParams, UpdateParams, UpdateRelationParams } from '../../src/index.js' -import { abandonAsCrashed, makeTempDir, openBrain, uid } from '../helpers/durabilityKillMatrix.js' -import { - createModel, - getCanonicalCountsFor, - modelAdd, - modelDelete, - modelRelate, - modelUpdate, - modelUpdateRelation, - recordVfsFileWrite, - snapshotVfsBaseline, - verifyChapter, - type HubCheck, - type ShadowModel -} from './biographyHarness.js' - -const STATUSES = ['active', 'pending', 'closed', 'archived'] as const - -/** Cycle a status value to the next one in the fixed rotation — used so - * Ch2's 40 updates provably MOVE entities across find() buckets rather than - * risking a no-op reassignment of the same value. */ -function nextStatus(current: unknown): (typeof STATUSES)[number] { - const currentStr = typeof current === 'string' ? current : STATUSES[0] - const idx = STATUSES.indexOf(currentStr as (typeof STATUSES)[number]) - return STATUSES[(idx < 0 ? 0 : idx + 1) % STATUSES.length] -} - -// --------------------------------------------------------------------------- -// Shared biography state — set up by the first `it`, consumed by the second. -// The two blocks are one continuous story told in two named pieces; nothing -// here resets or diverges between them. -// --------------------------------------------------------------------------- -let dir: string -let model: ShadowModel -let brain: Brainy -let hubs: HubCheck[] -let employees: string[] -let customers: string[] -let invoices: string[] -let tasks: string[] -let projects: string[] -let nonHub: string[] - -// ---- Wrappers: every call to the real brain updates the shadow model in -// the same statement, so the two can never drift apart by construction. -// Defined once, closing over the `let` bindings above so both `it` blocks -// (and any future reopen inside them) operate on the current brain/model. -async function doAdd(label: string, params: Omit): Promise { - const id = uid(label) - await brain.add({ ...params, id }) - modelAdd(model, id, { - type: params.type, - subtype: params.subtype, - metadata: params.metadata ?? {}, - visibility: params.visibility - }) - return id -} - -async function doUpdate(id: string, patch: Omit): Promise { - await brain.update({ ...patch, id }) - modelUpdate(model, id, { metadata: patch.metadata, merge: patch.merge, visibility: patch.visibility }) -} - -async function doRemove(id: string): Promise { - await brain.remove(id) - modelDelete(model, id) -} - -async function doRelate(params: RelateParams): Promise { - const id = await brain.relate(params) - modelRelate(model, id, { - from: params.from, - to: params.to, - type: params.type, - subtype: params.subtype, - metadata: params.metadata - }) - return id -} - -async function doUpdateRelation(id: string, patch: Omit): Promise { - await brain.updateRelation({ ...patch, id }) - modelUpdateRelation(model, id, { metadata: patch.metadata, merge: patch.merge }) -} - -async function doVfsWrite(path: string, content: string): Promise { - await brain.vfs.writeFile(path, content) - recordVfsFileWrite(model) -} - -describe.sequential('lifecycle — the working store', () => { - it( - 'Ch1 FOUNDING -> Ch2 A WORKING DAY -> Ch3 CLEAN RESTART: every read serves truth', - async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = makeTempDir() - model = createModel() - - // logAuthority: 'adopt' from the first open, mirrored across every - // reopen — see write-flow-production-shape.test.ts, which the later - // crash chapter's at-ack law is pinned against. - brain = await openBrain(dir, { logAuthority: 'adopt' }) - - // ================================================================= - // CHAPTER 1 — FOUNDING - // ================================================================= - // Baseline MUST be snapshotted before any biography act — it is the - // VFS root's own system-tier footprint, measured, never hardcoded. - await snapshotVfsBaseline(brain, model) - - employees = [] - for (let i = 0; i < 20; i++) { - employees.push( - await doAdd(`emp-${i}`, { - data: `employee record ${i}`, - type: NounType.Person, - subtype: 'employee', - metadata: { status: STATUSES[i % STATUSES.length], department: ['engineering', 'sales', 'support'][i % 3] } - }) - ) - } - customers = [] - for (let i = 0; i < 20; i++) { - customers.push( - await doAdd(`cust-${i}`, { - data: `customer record ${i}`, - type: NounType.Person, - subtype: 'customer', - metadata: { status: STATUSES[i % STATUSES.length], tier: i % 2 === 0 ? 'gold' : 'standard' } - }) - ) - } - invoices = [] - for (let i = 0; i < 30; i++) { - invoices.push( - await doAdd(`inv-${i}`, { - data: `invoice record ${i}`, - type: NounType.Document, - subtype: 'invoice', - metadata: { status: STATUSES[i % STATUSES.length], amount: 100 + i * 17 } - }) - ) - } - tasks = [] - for (let i = 0; i < 25; i++) { - tasks.push( - await doAdd(`task-${i}`, { - data: `task record ${i}`, - type: NounType.Task, - subtype: 'milestone', - metadata: { status: STATUSES[i % STATUSES.length], priority: (i % 5) + 1 } - }) - ) - } - projects = [] - for (let i = 0; i < 25; i++) { - projects.push( - await doAdd(`proj-${i}`, { - data: `project record ${i}`, - type: NounType.Project, - metadata: { status: STATUSES[i % STATUSES.length], budget: 1000 * (i + 1) } - }) - ) - } - expect(employees.length + customers.length + invoices.length + tasks.length + projects.length).toBe(120) - - // Five hubs (proj-0..proj-4) fan out to tasks (Contains) and employees - // (WorksWith); a residual band of invoice->customer RelatedTo edges is - // unrelated to any hub. Hubs are never touched again for the rest of - // the biography, so they stay valid adjacency samples in every chapter. - const hubIds = projects.slice(0, 5) - for (let h = 0; h < 5; h++) { - for (let k = 0; k < 15; k++) { - const taskIdx = (h * 5 + k) % tasks.length - await doRelate({ from: hubIds[h], to: tasks[taskIdx], type: VerbType.Contains, subtype: 'delivers' }) - } - for (let k = 0; k < 10; k++) { - const empIdx = (h * 4 + k) % employees.length - await doRelate({ from: hubIds[h], to: employees[empIdx], type: VerbType.WorksWith }) - } - } - for (let j = 0; j < 25; j++) { - await doRelate({ from: invoices[j], to: customers[j % customers.length], type: VerbType.RelatedTo, subtype: 'billed-to' }) - } - expect(model.relations.size).toBe(150) - - // A handful of VFS files. - for (let i = 0; i < 5; i++) { - await doVfsWrite(`/report-${i}.txt`, `founding report ${i}`) - } - - await brain.flush() - - hubs = hubIds.map((id) => ({ id, typeFilters: [VerbType.Contains, VerbType.WorksWith] })) - await verifyChapter(brain, model, 'Ch1 FOUNDING', { hubs, bucketField: 'status' }) - - // ================================================================= - // CHAPTER 2 — A WORKING DAY - // ================================================================= - // Non-hub pool for every mutation below. - nonHub = [...employees, ...customers, ...invoices, ...tasks, ...projects.slice(5)] - - // 40 updates that provably MOVE entities across find() status buckets. - const updateTargets = nonHub.slice(0, 40) - for (const id of updateTargets) { - const current = model.entities.get(id)!.metadata.status - await doUpdate(id, { metadata: { status: nextStatus(current) } }) - } - - // 10 visibility flips (public -> internal). - const visibilityTargets = nonHub.slice(40, 50) - for (const id of visibilityTargets) { - await doUpdate(id, { visibility: 'internal' }) - } - - // 15 deletes — some hub members (their edges cascade away), 3 of them - // earmarked for Ch6's resurrection. - const resurrectIds = [tasks[0], tasks[1], employees[0]] - const otherDeletes = [ - tasks[2], tasks[3], tasks[4], tasks[5], tasks[6], - employees[1], employees[2], employees[3], - customers[0], customers[1], customers[2], customers[3] - ] - const ch2DeleteTargets = [...resurrectIds, ...otherDeletes] - expect(ch2DeleteTargets.length).toBe(15) - for (const id of ch2DeleteTargets) { - await doRemove(id) - } - - // 20 new adds. - const ch2NewTypes = [NounType.Person, NounType.Document, NounType.Task] - for (let i = 0; i < 20; i++) { - await doAdd(`ch2-new-${i}`, { - data: `working-day addition ${i}`, - type: ch2NewTypes[i % ch2NewTypes.length], - subtype: 'ad-hoc', - metadata: { status: STATUSES[i % STATUSES.length] } - }) - } - - // 10 updateRelation metadata patches — read AFTER the deletes above, - // so only relations the cascade left alive are ever targeted. - const survivingRelationIds = [...model.relations.keys()].slice(0, 10) - expect(survivingRelationIds.length).toBe(10) - for (const relId of survivingRelationIds) { - await doUpdateRelation(relId, { metadata: { reviewed: true } }) - } - - await brain.flush() - await verifyChapter(brain, model, 'Ch2 A WORKING DAY', { hubs, bucketField: 'status' }) - - // ================================================================= - // CHAPTER 3 — CLEAN RESTART - // ================================================================= - await brain.close() - brain = await openBrain(dir, { logAuthority: 'adopt' }) - await verifyChapter(brain, model, 'Ch3 CLEAN RESTART', { hubs, bucketField: 'status' }) - - // Leave the brain closed and the directory intact for the next `it` - // (the biography continues there) — do NOT remove `dir` here. - await brain.close() - }, - 300000 - ) - - it( - 'Ch4 CRASH -> Ch5 REPAIR -> Ch6 SECOND LIFE: continues the Ch3 store', - async () => { - try { - brain = await openBrain(dir, { logAuthority: 'adopt' }) - - // =============================================================== - // CHAPTER 4 — CRASH - // =============================================================== - const ch4Types = [NounType.Person, NounType.Document, NounType.Task, NounType.Project] - for (let i = 0; i < 10; i++) { - await doAdd(`ch4-new-${i}`, { - data: `crash-window addition ${i}`, - type: ch4Types[i % ch4Types.length], - metadata: { status: STATUSES[i % STATUSES.length] } - }) - } - const ch4UpdateTargets = nonHub.slice(50, 55) // invoices[10..14] — untouched so far - for (const id of ch4UpdateTargets) { - await doUpdate(id, { metadata: { status: 'active' } }) - } - // NO flush — abandon exactly the way process death would (the - // at-ack law: every write already awaited above must survive). - await abandonAsCrashed(brain) - brain = await openBrain(dir, { logAuthority: 'adopt' }) - await verifyChapter(brain, model, 'Ch4 CRASH', { hubs, bucketField: 'status' }) - - // =============================================================== - // CHAPTER 5 — REPAIR - // =============================================================== - const report = await brain.repairIndex() - for (const family of report.families) { - const accounted = - family.checked === true || (family.checked === false && typeof family.skipped === 'string' && family.skipped.length > 0) - expect( - accounted, - `[Ch5 REPAIR] family '${family.family}' must be checked or explicitly skipped with a reason; got ${JSON.stringify(family)}` - ).toBe(true) - } - // A healthy store: repair must change nothing the model doesn't - // already expect — verifyChapter against the UNCHANGED model proves it. - await verifyChapter(brain, model, 'Ch5 REPAIR', { hubs, bucketField: 'status' }) - - // =============================================================== - // CHAPTER 6 — SECOND LIFE - // =============================================================== - const ch6Types = [NounType.Person, NounType.Document, NounType.Task, NounType.Project] - for (let i = 0; i < 10; i++) { - await doAdd(`ch6-new-${i}`, { - data: `second-life addition ${i}`, - type: ch6Types[i % ch6Types.length], - metadata: { status: STATUSES[i % STATUSES.length] } - }) - } - const ch6UpdateTargets = nonHub.slice(55, 65) // invoices[15..24] — untouched so far - expect(ch6UpdateTargets.every((id) => model.entities.get(id)!.alive)).toBe(true) - for (const id of ch6UpdateTargets) { - await doUpdate(id, { metadata: { status: 'closed' } }) - } - const ch6DeleteTargets = nonHub - .slice(65, 90) // invoices[25..29] + tasks[0..19] (some already dead — filtered below) - .filter((id) => model.entities.get(id)!.alive) - .slice(0, 7) - expect(ch6DeleteTargets.length).toBe(7) - for (const id of ch6DeleteTargets) { - await doRemove(id) - } - - // Resurrection: the SAME three ids Ch2 deleted, reinserted with - // BRAND-NEW metadata — the model expects the new metadata only. - await doAdd('task-0', { data: 'resurrected task 0', type: NounType.Task, subtype: 'milestone', metadata: { status: 'active', resurrected: true } }) - await doAdd('task-1', { data: 'resurrected task 1', type: NounType.Task, subtype: 'milestone', metadata: { status: 'pending', resurrected: true } }) - await doAdd('emp-0', { data: 'resurrected employee 0', type: NounType.Person, subtype: 'employee', metadata: { status: 'active', resurrected: true } }) - expect(tasks[0]).toBe(uid('task-0')) // same id as Ch1/Ch2 — the resurrection-adjacent shape - - await brain.close() - brain = await openBrain(dir, { logAuthority: 'adopt' }) - await verifyChapter(brain, model, 'Ch6 SECOND LIFE', { hubs, bucketField: 'status' }) - - // Final, standalone getCanonicalCounts() exactness check (beyond - // verifyChapter's own (f) leg) — the whole ledger, in one shot. - const finalCounts = await getCanonicalCountsFor(brain) - const aliveEntities = [...model.entities.values()].filter((e) => e.alive) - const alivePublicEntities = aliveEntities.filter((e) => (e.visibility ?? 'public') === 'public') - const aliveVerbs = model.relations.size - expect(finalCounts, 'final getCanonicalCounts() exactness — Ch6 SECOND LIFE').toEqual({ - nouns: { - counted: alivePublicEntities.length + model.vfsFileNouns, - all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns - }, - verbs: { - counted: aliveVerbs + model.vfsContainsVerbs, - all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs - }, - // Every noun this biography ever adds carries an explicit/computed - // vector (the harness never defers an embed), so the vectored-noun - // scalar tracks nouns.all exactly EXCEPT for the VFS root counted - // in `vfsBaselineNouns`: the root is deliberately persisted with - // `vector: []` (the sanctioned "unvectored" shape — see - // VirtualFileSystem.doInitializeRoot()'s zero-norm-avoidance - // comment) so it never pays the WASM engine's cold-compile cost and - // never crosses an engine boundary as a false attractor. It is the - // ONE hidden-tier record `vfsBaselineNouns` represents (see - // biographyHarness's module header), so it is excluded here even - // though it counts toward `nouns.all`. - vectors: { - all: aliveEntities.length + model.vfsFileNouns - }, - suspect: false - }) - } finally { - await brain.close().catch(() => {}) - // Best-effort, retried: a still-draining background persistence - // write (e.g. count/index write-through) can race a single rmSync - // and leave a partial directory behind — retry a couple of times - // rather than let this temp dir leak. - for (let attempt = 0; attempt < 3; attempt++) { - try { - fs.rmSync(dir, { recursive: true, force: true }) - if (!fs.existsSync(dir)) break - } catch { - // ignore and retry - } - await new Promise((resolve) => setTimeout(resolve, 100)) - } - } - }, - 300000 - ) -}) diff --git a/tests/lifecycle/biographyHarness.ts b/tests/lifecycle/biographyHarness.ts deleted file mode 100644 index ca15b36a..00000000 --- a/tests/lifecycle/biographyHarness.ts +++ /dev/null @@ -1,389 +0,0 @@ -/** - * @module tests/lifecycle/biographyHarness - * @description The referee for the LIFECYCLE LANE (see `biography.test.ts`): - * a plain in-memory SHADOW MODEL of a brain's contents, updated by every act - * the biography performs (add/update/remove/relate/updateRelation/vfs writes), - * plus `verifyChapter()`, which asserts the live brain agrees with the model - * after every chapter. No engine code runs inside the model — it is an - * independent ledger, not a mirror of the implementation under test. - * - * COUNT SEMANTICS this harness encodes (verified against the live engine, - * not assumed — see the module-level comments below for how each was - * confirmed): - * - * - `getNounCount()` / `getVerbCount()` count PUBLIC-tier alive records only - * (visibility absent or `'public'`) — `'internal'` and `'system'` are both - * excluded. `storage.getCanonicalCounts()` mirrors that same PUBLIC-only - * scalar as `counted`, and additionally reports `all` — every tier, - * unfiltered — as the coverage-ledger denominator (see - * tests/integration/canonical-count-ledger.test.ts). - * - `brain.vfs.writeFile()` for a brand-new file at a path directly under the - * VFS root creates exactly ONE new File noun plus ONE new `Contains` verb - * (root -> file), and BOTH are ordinary PUBLIC records (no visibility - * field is set) — so they count toward `getNounCount()`/`getVerbCount()` - * as well as the canonical `all` scalars. Only the VFS ROOT entity itself - * is `'system'`-tier (created once, at `init()`, before any biography - * chapter runs) — that lone record is the only hidden-tier footprint the - * model does not construct explicitly, so it is captured empirically via - * `snapshotVfsBaseline()` immediately after `init()` rather than hardcoded. - * - `related()` filters edges by the RELATION's own visibility tier, not by - * the visibility of the entities the edge connects — flipping an entity to - * `'internal'` does not hide its edges from `related()`. This lane never - * sets relation visibility, so every relation the model tracks is exactly - * as reachable as its presence in `model.relations` implies. - * - `remove()` cascades: every relation touching the removed entity (as - * `from` or `to`) is hard-deleted along with it. The model mirrors this by - * deleting the relation entirely from `model.relations` (no relation - * "alive" flag — presence in the map IS aliveness). - */ -import { expect } from 'vitest' -import type { Brainy } from '../../src/brainy.js' -import type { NounType, VerbType } from '../../src/types/graphTypes.js' -import type { EntityVisibility, StorageAdapter } from '../../src/coreTypes.js' - -/** - * One entity's complete lifecycle-relevant state, as the biography's acts - * leave it. `alive: false` means the model believes the id has been removed - * — the entry is KEPT (never deleted from the map) so `verifyChapter` can - * assert the negative half of the contract: a dead id must read as `null`. - */ -export interface ShadowEntity { - type: NounType - subtype?: string - metadata: Record - visibility?: EntityVisibility - alive: boolean -} - -/** - * One relation's complete lifecycle-relevant state. There is no `alive` - * flag here — presence in {@link ShadowModel.relations} IS aliveness, - * mirroring the engine's hard delete of the canonical verb record on - * cascade (see the module header). - */ -export interface ShadowRelation { - from: string - to: string - type: VerbType - subtype?: string - metadata: Record -} - -/** - * The independent truth ledger the biography updates on every act it - * performs. `verifyChapter` checks the live brain against this — never the - * other way around. - */ -export interface ShadowModel { - entities: Map - relations: Map - /** - * `getCanonicalCounts()` nouns.all / verbs.all captured right after - * `init()`, before chapter 1 — the VFS root's own system-tier footprint. - * Set once via {@link snapshotVfsBaseline}; never hardcoded. - */ - vfsBaselineNouns: number - vfsBaselineVerbs: number - /** - * Public nouns/verbs created by `vfs.writeFile()` for a brand-new file at - * a flat top-level path: exactly one File noun + one Contains verb per - * call (see the module header). Bumped by {@link recordVfsFileWrite}. - */ - vfsFileNouns: number - vfsContainsVerbs: number -} - -/** A fresh, empty shadow model — call once before chapter 1. */ -export function createModel(): ShadowModel { - return { - entities: new Map(), - relations: new Map(), - vfsBaselineNouns: 0, - vfsBaselineVerbs: 0, - vfsFileNouns: 0, - vfsContainsVerbs: 0 - } -} - -/** Narrow, documented private-storage access (the same style already used by - * `tests/helpers/durabilityKillMatrix.ts`'s `storeOf()`), needed because - * `getCanonicalCounts()` lives on the storage adapter, not on `Brainy`. */ -function storageOf(brain: Brainy): StorageAdapter { - return (brain as unknown as { storage: StorageAdapter }).storage -} - -/** Public wrapper around the private-storage `getCanonicalCounts()` read, so - * callers never need their own private-access cast — used internally by - * {@link snapshotVfsBaseline} and {@link verifyChapter}, and by - * `biography.test.ts` for its final standalone exactness check. */ -export async function getCanonicalCountsFor(brain: Brainy): ReturnType> { - const storage = storageOf(brain) - if (!storage.getCanonicalCounts) { - throw new Error( - 'lifecycle lane: the storage adapter under test has no getCanonicalCounts() — the canonical-count-exactness leg of this lane is unrepresentable without it.' - ) - } - return storage.getCanonicalCounts() -} - -/** - * Snapshot the VFS root's own hidden-tier footprint. Call exactly once, - * immediately after `init()` and before chapter 1 does anything — this is - * the ONE baseline offset the model does not construct by hand (see the - * module header for why: the root is `'system'`-tier plumbing the biography - * never explicitly creates). - */ -export async function snapshotVfsBaseline(brain: Brainy, model: ShadowModel): Promise { - const counts = await getCanonicalCountsFor(brain) - model.vfsBaselineNouns = counts.nouns.all - model.vfsBaselineVerbs = counts.verbs.all -} - -/** - * Record one `brain.vfs.writeFile()` call for a brand-new file at a flat - * top-level path (no intermediate directories). Bumps both the noun and verb - * VFS counters by one, matching the engine's actual write path exactly (see - * the module header) — never call this for an overwrite of an existing path, - * a nested path (which would also vivify intermediate directory nouns/edges, - * a different, unmodeled shape), or the biography loses its exactness. - */ -export function recordVfsFileWrite(model: ShadowModel): void { - model.vfsFileNouns += 1 - model.vfsContainsVerbs += 1 -} - -/** Record a fresh `add()` (or a Ch6 resurrection — `Map.set` fully replaces - * whatever a prior dead entry held, which is exactly the "new metadata only" - * contract a resurrection must honor). */ -export function modelAdd( - model: ShadowModel, - id: string, - entity: { type: NounType; subtype?: string; metadata: Record; visibility?: EntityVisibility } -): void { - model.entities.set(id, { - type: entity.type, - subtype: entity.subtype, - metadata: { ...entity.metadata }, - visibility: entity.visibility, - alive: true - }) -} - -/** Record an `update()` — merges metadata by default, matching the engine's - * `merge: true` default; pass `merge: false` to mirror a full replace. */ -export function modelUpdate( - model: ShadowModel, - id: string, - patch: { metadata?: Record; merge?: boolean; visibility?: EntityVisibility } -): void { - const existing = model.entities.get(id) - if (!existing || !existing.alive) { - throw new Error(`shadow model: update() targeted ${id}, which the model does not have alive — biography sequencing bug`) - } - if (patch.metadata) { - existing.metadata = patch.merge === false ? { ...patch.metadata } : { ...existing.metadata, ...patch.metadata } - } - if (patch.visibility !== undefined) { - existing.visibility = patch.visibility - } -} - -/** Record a `remove()` — marks the entity dead (entry retained, per - * {@link ShadowEntity}) and cascades: every relation touching it, in either - * direction, is hard-deleted from the model too (matching the engine). */ -export function modelDelete(model: ShadowModel, id: string): void { - const existing = model.entities.get(id) - if (!existing || !existing.alive) { - throw new Error(`shadow model: remove() targeted ${id}, which the model does not have alive — biography sequencing bug`) - } - existing.alive = false - for (const [relId, rel] of model.relations) { - if (rel.from === id || rel.to === id) model.relations.delete(relId) - } -} - -/** Record a `relate()` — `id` is the relation id the real call returned. */ -export function modelRelate( - model: ShadowModel, - id: string, - relation: { from: string; to: string; type: VerbType; subtype?: string; metadata?: Record } -): void { - model.relations.set(id, { - from: relation.from, - to: relation.to, - type: relation.type, - subtype: relation.subtype, - metadata: { ...(relation.metadata ?? {}) } - }) -} - -/** Record an `updateRelation()` metadata patch — merges by default. */ -export function modelUpdateRelation( - model: ShadowModel, - id: string, - patch: { metadata?: Record; merge?: boolean } -): void { - const existing = model.relations.get(id) - if (!existing) { - throw new Error(`shadow model: updateRelation() targeted ${id}, which the model does not have — biography sequencing bug`) - } - if (patch.metadata) { - existing.metadata = patch.merge === false ? { ...patch.metadata } : { ...existing.metadata, ...patch.metadata } - } -} - -/** Order-independent structural equality for plain JSON-shaped metadata. */ -function deepEqual(a: unknown, b: unknown): boolean { - if (a === b) return true - if (typeof a !== typeof b) return false - if (a === null || b === null) return a === b - if (typeof a !== 'object') return false - const aKeys = Object.keys(a as Record) - const bKeys = Object.keys(b as Record) - if (aKeys.length !== bKeys.length) return false - for (const k of aKeys) { - if (!deepEqual((a as Record)[k], (b as Record)[k])) return false - } - return true -} - -/** One hub entity to sample for the `related()` adjacency check, plus the - * verb type(s) it is known (by biography construction) to have OUT-edges - * of, so the type-filtered variant is exercised too. */ -export interface HubCheck { - id: string - typeFilters: VerbType[] -} - -/** Options steering one `verifyChapter()` call. */ -export interface VerifyOptions { - /** Hub entities to sample for the `related()` adjacency check. */ - hubs: HubCheck[] - /** The metadata field `find()` bucket-checks against (a bare string field - * every alive entity may or may not carry — distinct values present among - * ALIVE model entities are discovered automatically each call, so a - * chapter that moves entities across buckets is re-checked exactly). */ - bucketField: string -} - -/** - * Assert the live brain agrees with the model, in full, after one chapter. - * Every failure message names the chapter `label`, the id (where - * applicable), and expected-vs-actual — a red here must be diagnosable from - * the assertion message alone, with no need to re-read this file. - */ -export async function verifyChapter(brain: Brainy, model: ShadowModel, label: string, opts: VerifyOptions): Promise { - // (a) + (b): every alive entity reads back exactly as modeled; every dead - // entity reads as null. - for (const [id, entity] of model.entities) { - const live = await brain.get(id) - if (entity.alive) { - expect(live, `[${label}] alive entity ${id} (type=${entity.type}) must be readable via get(), got null`).not.toBeNull() - const e = live! - expect(e.type, `[${label}] entity ${id} .type mismatch: expected ${entity.type}, got ${e.type}`).toBe(entity.type) - expect(e.subtype, `[${label}] entity ${id} .subtype mismatch: expected ${JSON.stringify(entity.subtype)}, got ${JSON.stringify(e.subtype)}`).toBe(entity.subtype) - expect( - e.visibility, - `[${label}] entity ${id} .visibility mismatch: expected ${JSON.stringify(entity.visibility)}, got ${JSON.stringify(e.visibility)}` - ).toBe(entity.visibility) - const metaMatches = deepEqual(e.metadata ?? {}, entity.metadata) - expect( - metaMatches, - `[${label}] entity ${id} .metadata mismatch: expected ${JSON.stringify(entity.metadata)}, got ${JSON.stringify(e.metadata)}` - ).toBe(true) - } else { - expect(live, `[${label}] dead entity ${id} (type=${entity.type}) must read as null, got ${JSON.stringify(live)}`).toBeNull() - } - } - - // (c) find({ where: { : value } }) returns exactly the - // model's matching alive set, per distinct value currently present. - const bucketValues = new Set() - for (const entity of model.entities.values()) { - if (!entity.alive) continue - const v = entity.metadata[opts.bucketField] - if (typeof v === 'string') bucketValues.add(v) - } - for (const value of bucketValues) { - const expectedIds = [...model.entities.entries()] - .filter(([, e]) => e.alive && e.metadata[opts.bucketField] === value) - .map(([id]) => id) - .sort() - const results = await brain.find({ - where: { [opts.bucketField]: value } as Record, - includeInternal: true, - limit: 100000 - }) - const actualIds = results.map((r) => r.id).sort() - expect( - actualIds, - `[${label}] find({ where: { ${opts.bucketField}: ${JSON.stringify(value)} } }) mismatch: expected ${expectedIds.length} ids ${JSON.stringify(expectedIds)}, got ${actualIds.length} ids ${JSON.stringify(actualIds)}` - ).toEqual(expectedIds) - } - - // (d) related(id) / related(id, { type }) for the hub sample matches the - // model's adjacency exactly (out-edges — related(id) is shorthand for - // { from: id }). - for (const hub of opts.hubs) { - const expectedAll = [...model.relations.entries()] - .filter(([, r]) => r.from === hub.id) - .map(([id]) => id) - .sort() - const liveAll = await brain.related({ from: hub.id, limit: 100000 }) - const actualAllIds = liveAll.map((r) => r.id).sort() - expect( - actualAllIds, - `[${label}] related(${hub.id}) mismatch: expected ${expectedAll.length} ids ${JSON.stringify(expectedAll)}, got ${actualAllIds.length} ids ${JSON.stringify(actualAllIds)}` - ).toEqual(expectedAll) - - for (const typeFilter of hub.typeFilters) { - const expectedTyped = [...model.relations.entries()] - .filter(([, r]) => r.from === hub.id && r.type === typeFilter) - .map(([id]) => id) - .sort() - const liveTyped = await brain.related({ from: hub.id, type: typeFilter, limit: 100000 }) - const actualTypedIds = liveTyped.map((r) => r.id).sort() - expect( - actualTypedIds, - `[${label}] related(${hub.id}, { type: '${typeFilter}' }) mismatch: expected ${expectedTyped.length} ids ${JSON.stringify(expectedTyped)}, got ${actualTypedIds.length} ids ${JSON.stringify(actualTypedIds)}` - ).toEqual(expectedTyped) - } - } - - // (e) getNounCount() / getVerbCount(): PUBLIC-tier alive records - // (visibility absent/'public'; 'internal' and 'system' both excluded — see - // the module header) plus the VFS's own public contributions. - const alivePublicNouns = [...model.entities.values()].filter((e) => e.alive && (e.visibility ?? 'public') === 'public').length - const aliveVerbs = model.relations.size - const expectedNounCount = alivePublicNouns + model.vfsFileNouns - const expectedVerbCount = aliveVerbs + model.vfsContainsVerbs - expect( - await brain.getNounCount(), - `[${label}] getNounCount() mismatch: expected ${expectedNounCount} (alive public entities ${alivePublicNouns} + vfs file nouns ${model.vfsFileNouns})` - ).toBe(expectedNounCount) - expect( - await brain.getVerbCount(), - `[${label}] getVerbCount() mismatch: expected ${expectedVerbCount} (alive relations ${aliveVerbs} + vfs contains verbs ${model.vfsContainsVerbs})` - ).toBe(expectedVerbCount) - - // (f) getCanonicalCounts(): ALL-visibility scalars (every tier) equal the - // model's alive totals including hidden tiers, plus the VFS's own - // contributions (both file nouns/verbs AND the once-measured root - // baseline). suspect must be false — every delete in this biography goes - // through brain.remove(), which always proves the record it decrements. - const ledger = await getCanonicalCountsFor(brain) - const aliveAllNouns = [...model.entities.values()].filter((e) => e.alive).length - const expectedNounsAll = aliveAllNouns + model.vfsFileNouns + model.vfsBaselineNouns - const expectedVerbsAll = aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs - expect( - ledger.nouns.all, - `[${label}] getCanonicalCounts().nouns.all mismatch: expected ${expectedNounsAll} (alive incl. internal ${aliveAllNouns} + vfs file nouns ${model.vfsFileNouns} + vfs root baseline ${model.vfsBaselineNouns})` - ).toBe(expectedNounsAll) - expect( - ledger.verbs.all, - `[${label}] getCanonicalCounts().verbs.all mismatch: expected ${expectedVerbsAll} (alive relations ${aliveVerbs} + vfs contains verbs ${model.vfsContainsVerbs} + vfs root baseline ${model.vfsBaselineVerbs})` - ).toBe(expectedVerbsAll) - expect(ledger.nouns.counted, `[${label}] getCanonicalCounts().nouns.counted mismatch (should mirror getNounCount())`).toBe(expectedNounCount) - expect(ledger.verbs.counted, `[${label}] getCanonicalCounts().verbs.counted mismatch (should mirror getVerbCount())`).toBe(expectedVerbCount) - expect(ledger.suspect, `[${label}] getCanonicalCounts().suspect must be false — every delete in this biography proves its record`).toBe(false) -} diff --git a/tests/unit/brainy/add.test.ts b/tests/unit/brainy/add.test.ts index 7203690c..10690e2f 100644 --- a/tests/unit/brainy/add.test.ts +++ b/tests/unit/brainy/add.test.ts @@ -335,24 +335,15 @@ describe('Brainy.add()', () => { }) describe('edge cases', () => { - it('should accept an empty string as real (empty) data', async () => { - // Arrange — '' is legitimate content (e.g. an empty file's first - // write), not a missing field. Only null/undefined data (with no - // vector either) is "missing" — see the separate - // 'data and vector are both missing' test above. + it('should reject empty string as data', async () => { + // Arrange const params = createAddParams({ data: '', type: 'thing' }) - - // Act - const id = await brain.add(params) - - // Assert — stored and readable back as empty, not rejected - expect(id).toBeDefined() - const entity = await brain.get(id) - expect(entity).not.toBeNull() - expect(entity!.data).toBe('') + + // Act & Assert - Empty string is not valid data + await expect(brain.add(params)).rejects.toThrow('Invalid add() parameters: Missing required field \'data\'') }) it('should handle very long text content', async () => { diff --git a/tests/unit/brainy/lazy-notready-honor.test.ts b/tests/unit/brainy/lazy-notready-honor.test.ts index e53be4a6..4cfc6857 100644 --- a/tests/unit/brainy/lazy-notready-honor.test.ts +++ b/tests/unit/brainy/lazy-notready-honor.test.ts @@ -1,49 +1,38 @@ /** * @module tests/unit/brainy/lazy-notready-honor * @description THE SILENT-EMPTY TRAP pin (found during a fleet adoption, - * SELF-ENGINE-PAIR-STANDARD): under `disableAutoRebuild: true`, the OLD lazy + * SELF-ENGINE-PAIR-STANDARD): under `disableAutoRebuild: true`, the lazy * first-query path (`ensureIndexesLoaded`) assessed ONLY the vector index's * readiness — a native METADATA provider reporting not-ready (its strand * report) never blocked the completion latch, so the promised lazy rebuild - * never fired and every `find()` silently returned `[]` on a populated store - * (measured: 52 entities durable-but-unqueryable, first query 0ms/0 rows). - * - * RE-POINTED to the health-gate law (a read never builds; a rebuild runs - * entirely at open): `ensureIndexesLoaded()` is now a pure CHECK. A not-ready - * report from ANY provider — metadata, vector, or graph — makes it THROW the - * matching typed `*NotReadyError` rather than silently letting the read - * proceed, and it NEVER calls `rebuildIndexesIfNeeded` (that is entirely - * open()'s job now — see the second describe block below). The spirit is - * unchanged: a not-ready report from any single provider can never be - * shadowed into a silent empty result. + * never fired and every `find()` silently returned `[]` on a populated + * store (measured: 52 entities durable-but-unqueryable, first query + * 0ms/0 rows). The law: a not-ready report from ANY provider falls through + * to the rebuild — never a silent empty. * * White-box provider-double pattern per tests/unit/brainy/migration-deference. */ import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy, MetadataIndexNotReadyError } from '../../../src/index.js' +import { Brainy } from '../../../src/index.js' import { NounType } from '../../../src/types/graphTypes.js' import { createTestConfig } from '../../helpers/test-factory.js' interface BrainInternals { index: { size(): number } metadataIndex: { isReady?: () => boolean } - ensureIndexesLoaded(): void + lazyRebuildCompleted: boolean + ensureIndexesLoaded(): Promise rebuildIndexesIfNeeded(force?: boolean): Promise } const brains: Brainy[] = [] -const dirs: string[] = [] afterEach(async () => { for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) vi.restoreAllMocks() }) -async function warmBrain(): Promise<{ brain: Brainy; internals: BrainInternals }> { +async function warmLazyBrain(): Promise<{ brain: Brainy; internals: BrainInternals }> { const brain = new Brainy(createTestConfig({ disableAutoRebuild: true })) await brain.init() brains.push(brain) @@ -51,59 +40,36 @@ async function warmBrain(): Promise<{ brain: Brainy; internals: BrainInternals } await brain.add({ data: `row ${i}`, type: NounType.Document, metadata: { i } }) } const internals = brain as unknown as BrainInternals + internals.lazyRebuildCompleted = false // simulate the cold first query return { brain, internals } } -describe('the read gate honors EVERY provider’s not-ready report', () => { - it('a not-ready METADATA provider refuses loudly — it never lets a read proceed, and it never rebuilds', async () => { - const { internals } = await warmBrain() +describe('lazy path honors EVERY provider’s not-ready report', () => { + it('a not-ready METADATA provider blocks the completion latch and fires the rebuild', async () => { + const { internals } = await warmLazyBrain() // The trap's shape: vector side looks fine (populated), metadata - // provider says NOT ready — the OLD gate silently latched complete here. - // The new gate refuses loudly instead; a read never triggers a rebuild. - internals.metadataIndex.isReady = () => false - const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + // provider says NOT ready — the old gate latched complete here. + ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => false + const rebuildSpy = vi + .spyOn(internals, 'rebuildIndexesIfNeeded') + .mockResolvedValue(undefined) - expect(() => internals.ensureIndexesLoaded()).toThrow(MetadataIndexNotReadyError) - expect(rebuildSpy, 'a read NEVER triggers a rebuild — building is entirely open()\'s job now').not.toHaveBeenCalled() + await internals.ensureIndexesLoaded() + + expect(rebuildSpy, 'not-ready metadata provider must fire the lazy rebuild').toHaveBeenCalledWith(true) }) - it('control: all providers ready/unknown+populated → the gate lets the read through, no rebuild', async () => { - const { internals } = await warmBrain() - internals.metadataIndex.isReady = () => true - const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + it('control: all providers ready/unknown+populated → latch completes, no rebuild', async () => { + const { internals } = await warmLazyBrain() + ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => true + const rebuildSpy = vi + .spyOn(internals, 'rebuildIndexesIfNeeded') + .mockResolvedValue(undefined) + + await internals.ensureIndexesLoaded() - expect(() => internals.ensureIndexesLoaded()).not.toThrow() expect(rebuildSpy).not.toHaveBeenCalled() + expect(internals.lazyRebuildCompleted).toBe(true) }) }) - -describe('the open-time build honors the same law: a needed rebuild runs at open, never deferred to a read', () => { - it('disableAutoRebuild:true does not defer a needed rebuild past open() on a reopened, populated store', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-lazy-notready-honor-')) - dirs.push(dir) - - const writer = new Brainy(createTestConfig({ disableAutoRebuild: true, storage: { type: 'filesystem', path: dir } })) - await writer.init() - for (let i = 0; i < 3; i++) { - await writer.add({ data: `row ${i}`, type: NounType.Document, metadata: { i } }) - } - await writer.flush() - await writer.close() - - // Fresh instance over the same store: its derived indexes start empty in - // memory, so open()'s rebuildIndexesIfNeeded MUST fire (and complete) - // before init() returns — even though disableAutoRebuild is true, there - // is no first-query lazy path left to defer to. - const reader = new Brainy(createTestConfig({ disableAutoRebuild: true, storage: { type: 'filesystem', path: dir } })) - const internals = reader as unknown as { rebuildIndexesIfNeeded(force?: boolean): Promise } - const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded') - - await reader.init() - brains.push(reader) - - expect(rebuildSpy).toHaveBeenCalledTimes(1) - const rows = await reader.find({ where: { i: 1 } }) - expect(rows.length).toBe(1) - }, 30000) -}) diff --git a/tests/unit/brainy/metadata-provider-contract.test.ts b/tests/unit/brainy/metadata-provider-contract.test.ts index 945c0670..7e690978 100644 --- a/tests/unit/brainy/metadata-provider-contract.test.ts +++ b/tests/unit/brainy/metadata-provider-contract.test.ts @@ -1,18 +1,15 @@ /** * @module tests/unit/brainy/metadata-provider-contract - * @description Brainy-side wiring of the metadata-provider contract. + * @description Brainy-side wiring of the two metadata-provider contract additions + * confirmed with cor for the lockstep: * - * `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED - * `find({ type, where, limit })` path so a native provider can early-stop. The JS - * index ignores `opts`. - * - * RETIRED (health-gate law): `probeConsistency()` / `ensureMetadataConsistencyProbed()` - * — a read-time consistency probe that launches `detectAndRepairCorruption()` on - * `false` was exactly the read-triggered dark rebuild the law forbids (a read must - * never start a store walk or a rebuild). The probe's diagnostic value lives on in - * `validateIndexConsistency()` / `repairIndex()`, which remain explicit, operator-invoked - * calls. The pin below confirms the retirement: `probeConsistency()` is never called by - * a read, even when a provider exposes it. + * 1. `probeConsistency()` — an OPTIONAL O(1) cold-open consistency sampler. On the + * first read, brainy calls it once; on `false` it self-heals via + * `detectAndRepairCorruption()` (the metadata counterpart of the graph cold-load + * guard). The native provider implements it; the JS index omits it (no-op). + * 2. `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED + * `find({ type, where, limit })` path so a native provider can early-stop. The JS + * index ignores `opts`. * * These are unit tests of brainy's CALL behaviour (the real end-to-end honoring is * exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS @@ -22,7 +19,7 @@ import { describe, it, expect, beforeEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' -describe('metadata-provider contract wiring (getIdsForFilter opts)', () => { +describe('metadata-provider contract wiring (probeConsistency + getIdsForFilter opts)', () => { let brain: Brainy let mi: any @@ -32,23 +29,47 @@ describe('metadata-provider contract wiring (getIdsForFilter opts)', () => { await brain.add({ data: 'a', type: NounType.Thing, metadata: { kind: 'x' } }) await brain.add({ data: 'b', type: NounType.Thing, metadata: { kind: 'y' } }) mi = (brain as any).metadataIndex + ;(brain as any)._metadataConsistencyProbed = false // reset the one-shot guard }) - it('RETIRED: a read never calls probeConsistency() / self-heals via detectAndRepairCorruption — that is the read-triggered dark rebuild the health-gate law forbids', async () => { + it('calls probeConsistency once on cold open and self-heals via detectAndRepairCorruption on false', async () => { let probes = 0 let repairs = 0 - mi.probeConsistency = async () => { probes++; return false } // would-be corrupt signal + mi.probeConsistency = async () => { probes++; return false } // corrupt → must repair const origRepair = mi.detectAndRepairCorruption.bind(mi) mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } await brain.find({ where: { kind: 'x' } }) + expect(probes).toBe(1) + expect(repairs).toBe(1) + + // Second read must NOT re-probe (once per brain). await brain.find({ where: { kind: 'y' } }) + expect(probes).toBe(1) + expect(repairs).toBe(1) + }) - expect(probes).toBe(0) // no read-time probe exists anymore - expect(repairs).toBe(0) // and therefore no read-triggered self-heal either + it('does NOT repair when the probe reports healthy', async () => { + let repairs = 0 + mi.probeConsistency = async () => true // clean + const origRepair = mi.detectAndRepairCorruption.bind(mi) + mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } - delete mi.probeConsistency - mi.detectAndRepairCorruption = origRepair + await brain.find({ where: { kind: 'x' } }) + expect(repairs).toBe(0) + }) + + it('a probe failure never breaks the read (best-effort, retried next time)', async () => { + let probes = 0 + mi.probeConsistency = async () => { probes++; throw new Error('probe boom') } + + // The read still succeeds despite the throwing probe. + const rows = await brain.find({ where: { kind: 'x' } }) + expect(rows.length).toBe(1) + expect(probes).toBe(1) + // Guard reset on failure → the next read retries the probe. + await brain.find({ where: { kind: 'y' } }) + expect(probes).toBe(2) }) it('passes a page bound to getIdsForFilter on the unsorted find path (offset 0, brainy re-windows)', async () => { diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index b5817c3d..5968c620 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -15,7 +15,7 @@ * - Hook 2: the public `brain.stampBrainFormat()` the provider calls once its * background migration has verified-and-swapped, authoring the shared * `_system/brain-format.json` marker. - * - Hook 3: the marker module is re-exported at `@soulcraftlabs/brainy/brain-format` + * - Hook 3: the marker module is re-exported at `@soulcraft/brainy/brain-format` * so cor reads the SAME `EXPECTED_INDEX_EPOCH` / `CURRENT_DATA_FORMAT` constants * (single source of truth, no duplicated value). * @@ -25,7 +25,7 @@ */ import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy, VectorIndexNotReadyError } from '../../../src/index.js' +import { Brainy } from '../../../src/index.js' import { NounType } from '../../../src/types/graphTypes.js' import { createTestConfig } from '../../helpers/test-factory.js' import { BaseStorage } from '../../../src/storage/baseStorage.js' @@ -43,8 +43,9 @@ interface BrainInternals { metadataIndex: { rebuild(...a: unknown[]): Promise } graphIndex: { size(): number; rebuild(...a: unknown[]): Promise } _indexEpochStale: boolean + lazyRebuildCompleted: boolean rebuildIndexesIfNeeded(force?: boolean): Promise - ensureIndexesLoaded(): void + ensureIndexesLoaded(): Promise storage: { readRawObject(p: string): Promise } } @@ -180,40 +181,40 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b expect(idxSpy).toHaveBeenCalledTimes(1) }) - // --- Hook 1: read-gate deference (RE-POINTED — the health-gate law retired - // the first-query lazy force-rebuild entirely: ensureIndexesLoaded() is now - // a pure CHECK that never calls rebuildIndexesIfNeeded, migrating or not. - // What survives from the original law is the DEFERENCE itself: a migrating - // provider's report is never judged by the gate — it neither throws nor - // rebuilds — while the exact same not-ready report on a NON-migrating - // provider throws the typed error instead of ever rebuilding.) ------------ + // --- Hook 1: large-path first-query lazy force-rebuild deference ---------- - it('the read gate defers to a migrating vector provider — a not-ready report neither throws nor rebuilds', async () => { + it('lazy first-query force-rebuild is SKIPPED when the vector provider isMigrating()', async () => { + // disableAutoRebuild routes first queries through ensureIndexesLoaded() (the + // large-brain lazy path that would otherwise force a blocking rebuild). const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) const internals = internalsOf(brain) const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - // Simulate a not-ready live vector index (cor is mid-swap, serving canonical). - ;(internals.index as unknown as { isReady?: () => boolean }).isReady = () => false + // Simulate a cold/empty live vector index (cor is mid-swap, serving canonical). + vi.spyOn(internals.index, 'size').mockReturnValue(0) + internals.lazyRebuildCompleted = false setMigrating(internals.index, true) - expect(() => internals.ensureIndexesLoaded()).not.toThrow() - // A query during cor's background swap must not trigger brainy's own - // rebuild — reads never rebuild in any case, migrating or not. + await internals.ensureIndexesLoaded() + + // A query during cor's background swap must not trigger brainy's blocking rebuild. expect(rebuildSpy).toHaveBeenCalledTimes(0) }) - it('the read gate THROWS for the same not-ready vector provider once migration clears (control)', async () => { + it('lazy first-query force-rebuild STILL fires when the vector provider is not migrating (control)', async () => { const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) const internals = internalsOf(brain) const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - ;(internals.index as unknown as { isReady?: () => boolean }).isReady = () => false + vi.spyOn(internals.index, 'size').mockReturnValue(0) + internals.lazyRebuildCompleted = false // No isMigrating → not deferring. - expect(() => internals.ensureIndexesLoaded()).toThrow(VectorIndexNotReadyError) - // Still never rebuilds — the gate refuses loudly instead. - expect(rebuildSpy).toHaveBeenCalledTimes(0) + await internals.ensureIndexesLoaded() + + // Without deference, the cold empty index drives the lazy force-rebuild. + expect(rebuildSpy).toHaveBeenCalledTimes(1) + expect(rebuildSpy).toHaveBeenCalledWith(true) }) // --- Hook 2: public stampBrainFormat() ----------------------------------- @@ -242,7 +243,7 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b // --- Hook 3: marker module export ---------------------------------------- it('the brain-format marker module exports the compiled epoch + data-format constants', () => { - // cor imports these from '@soulcraftlabs/brainy/brain-format' (Hook 3) so both + // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both // sides share ONE source of truth — no duplicated constant to drift. // Epoch 3: the namespace-law key split (bare user keys · literal // 'system.' scalars, 2026-08-03) — every brain rebuilds onto the diff --git a/tests/unit/brainy/open-path.test.ts b/tests/unit/brainy/open-path.test.ts deleted file mode 100644 index 3556d7a5..00000000 --- a/tests/unit/brainy/open-path.test.ts +++ /dev/null @@ -1,199 +0,0 @@ -/** - * OPEN-PATH tests: init() must never gate on the embedding model, the VFS - * root bootstrap must never touch the embedding engine, and a slow open - * must narrate its phases. - * - * Background: a production restart storm measured 90,017ms for a single - * brain init vs 1,117ms quiet — an ~80x contention multiplier — traced to - * every writer's init() eagerly awaiting the process-global WASM embedding - * engine before the VFS root even existed. See src/brainy.ts performInit() - * and src/vfs/VirtualFileSystem.ts doInitializeRoot(). - */ - -import { describe, it, expect, vi } from 'vitest' -import { Brainy } from '../../../src/brainy' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage' -import { embeddingManager } from '../../../src/embeddings/EmbeddingManager' -import { createTestConfig } from '../../helpers/test-factory' - -/** - * The four signals `isDeterministicEmbedMode()` checks (see - * src/embeddings/deterministicEmbedMode.ts). The global unit-test setup - * (tests/setup-unit.ts) sets some of these for the whole file/run in some - * vitest configurations; other configurations leave them unset and run the - * real WASM engine instead. The background-warm tests below need the - * "not unit-test mode" branch of performInit() to actually execute, so they - * save/clear/restore all four explicitly — deterministic regardless of - * which config invoked this file, never relying on ambient state. - */ -function withRealEmbedderBranch(fn: () => Promise): Promise { - const savedEnvDeterministic = process.env.BRAINY_DETERMINISTIC_EMBEDDINGS - const savedEnvUnitTest = process.env.BRAINY_UNIT_TEST - const g = globalThis as Record - const savedGlobalDeterministic = g.__BRAINY_DETERMINISTIC_EMBED__ - const savedGlobalUnitTest = g.__BRAINY_UNIT_TEST__ - - delete process.env.BRAINY_DETERMINISTIC_EMBEDDINGS - delete process.env.BRAINY_UNIT_TEST - delete g.__BRAINY_DETERMINISTIC_EMBED__ - delete g.__BRAINY_UNIT_TEST__ - - const restore = () => { - if (savedEnvDeterministic !== undefined) process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = savedEnvDeterministic - if (savedEnvUnitTest !== undefined) process.env.BRAINY_UNIT_TEST = savedEnvUnitTest - if (savedGlobalDeterministic !== undefined) g.__BRAINY_DETERMINISTIC_EMBED__ = savedGlobalDeterministic - if (savedGlobalUnitTest !== undefined) g.__BRAINY_UNIT_TEST__ = savedGlobalUnitTest - } - - return fn().finally(restore) -} - -/** - * A MemoryStorage whose init() takes an artificially long time — a - * controllable fake seam (not a wall-clock race) that reliably pushes - * performInit()'s "storage-init" phase (and therefore the total open time) - * past the 2000ms narration threshold, without touching the filesystem or - * relying on real contention. - */ -class SlowMemoryStorage extends MemoryStorage { - override async init(): Promise { - await new Promise((resolve) => setTimeout(resolve, 2200)) - await super.init() - } -} - -describe('OPEN-PATH: init() never gates on the embedding model', () => { - it('bootstrapping a fresh store never calls the embedding engine (VFS root add is engine-untouched)', async () => { - const embedSpy = vi.spyOn(embeddingManager, 'embed') - const brain = new Brainy(createTestConfig()) - try { - await brain.init() - - // The VFS root's add() must never have reached the embedding engine — - // it carries an explicit placeholder vector instead (see - // VirtualFileSystem.doInitializeRoot()). - expect(embedSpy).not.toHaveBeenCalled() - - // Sanity: the VFS is genuinely usable afterwards. - const files = await brain.vfs.readdir('/') - expect(files).toEqual([]) - } finally { - await brain.close() - embedSpy.mockRestore() - } - }) - - it('starts the embedding-engine warm in the BACKGROUND — init() resolves before the warm does', async () => { - await withRealEmbedderBranch(async () => { - const events: string[] = [] - let releaseWarm!: () => void - const warmGate = new Promise((resolve) => { - releaseWarm = resolve - }) - - const initSpy = vi.spyOn(embeddingManager, 'init').mockImplementation(async () => { - events.push('warm-start') - await warmGate - events.push('warm-resolve') - }) - - const brain = new Brainy(createTestConfig()) - try { - await brain.init() - events.push('init-resolved') - - // init() started the warm but returned WITHOUT waiting for it. - expect(initSpy).toHaveBeenCalledTimes(1) - expect(events).toEqual(['warm-start', 'init-resolved']) - - // Now let the fake warm finish and confirm it lands strictly after. - releaseWarm() - await new Promise((resolve) => setTimeout(resolve, 0)) - expect(events).toEqual(['warm-start', 'init-resolved', 'warm-resolve']) - } finally { - await brain.close() - initSpy.mockRestore() - } - }) - }) - - it('narrates a background warm FAILURE loudly instead of losing it silently', async () => { - await withRealEmbedderBranch(async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const initSpy = vi - .spyOn(embeddingManager, 'init') - .mockRejectedValue(new Error('simulated cold-compile failure')) - - const brain = new Brainy(createTestConfig()) - try { - // init() itself must still resolve — a failed background warm is - // never fatal to open(). - await expect(brain.init()).resolves.toBeUndefined() - - // Give the background .catch() a microtask/macrotask to run. - await new Promise((resolve) => setTimeout(resolve, 0)) - - const failureLine = warnSpy.mock.calls - .map((args) => args.map(String).join(' ')) - .find((line) => line.includes('background embedding-engine warm FAILED')) - expect(failureLine).toBeDefined() - expect(failureLine).toContain('simulated cold-compile failure') - } finally { - await brain.close() - initSpy.mockRestore() - warnSpy.mockRestore() - } - }) - }) - - it('eagerEmbeddings: false starts no warm at all', async () => { - await withRealEmbedderBranch(async () => { - const initSpy = vi.spyOn(embeddingManager, 'init') - const brain = new Brainy({ ...createTestConfig(), eagerEmbeddings: false }) - try { - await brain.init() - expect(initSpy).not.toHaveBeenCalled() - } finally { - await brain.close() - initSpy.mockRestore() - } - }) - }) - - it('narrates a slow open with a per-phase ms breakdown once total time exceeds 2000ms', async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const brain = new Brainy({ ...createTestConfig(), storage: new SlowMemoryStorage() }) - try { - await brain.init() - - const slowOpenLine = warnSpy.mock.calls - .map((args) => args.map(String).join(' ')) - .find((line) => line.includes('[Brainy] slow open:')) - - expect(slowOpenLine).toBeDefined() - expect(slowOpenLine).toContain('storage-init=') - expect(slowOpenLine).toContain('generation-store-open-fold=') - expect(slowOpenLine).toContain('index-init-gate=') - expect(slowOpenLine).toContain('vfs-bootstrap=') - expect(slowOpenLine).toContain('embedding-warm-started=') - } finally { - await brain.close() - warnSpy.mockRestore() - } - }, 20000) - - it('stays silent about phase timing when open is fast (under 2000ms)', async () => { - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) - const brain = new Brainy(createTestConfig()) - try { - await brain.init() - const slowOpenLine = warnSpy.mock.calls - .map((args) => args.map(String).join(' ')) - .find((line) => line.includes('[Brainy] slow open:')) - expect(slowOpenLine).toBeUndefined() - } finally { - await brain.close() - warnSpy.mockRestore() - } - }) -}) diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index f16e67b3..27ab85cb 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,119 +147,4 @@ describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) - - // ========================================================================== - // THE DENSITY LAW - // ========================================================================== - // - // A sealed segment declares a CONTIGUOUS range and every reader treats that - // range as containment. Folding a sparse batch therefore makes the segment - // claim generations it does not hold — and because `open()` merges declared - // ranges back into committedRanges, the hole is re-admitted as committed - // history and every later maintenance pass fails asking for a frame that was - // never written. That is the "generation N is inside sealed segment - // seg-....bgs's declared range but has no frame — packed history is damaged" - // narration seen on every run of the affected stores. - - it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => { - await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( - /not contiguous: 2 → 4 skips 1 generation/ - ) - // The refusal loses nothing: no segment was sealed, so the generations - // stay in the live tier and the next pass folds them correctly. - expect(store.segments()).toHaveLength(0) - expect(store.hasGeneration(1)).toBe(false) - }) - - it('a wider gap names how many generations it would have swallowed', async () => { - await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( - /not contiguous: 10 → 20 skips 9 generation\(s\)/ - ) - }) - - it('two contiguous runs folded separately declare honest ranges', async () => { - // What the caller now does instead of folding across the gap. - const a = await store.fold([gen(1), gen(2), gen(3)]) - const b = await store.fold([gen(7), gen(8)]) - expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) - expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) - // The gap is honestly outside the packed tier. - for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) - for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) - expect(await store.actualRanges()).toEqual([ - [1, 3], - [7, 8] - ]) - }) - - it('actualRanges() is exact and I/O-free for dense segments', async () => { - await store.fold([gen(1), gen(2)]) - await store.fold([gen(3), gen(4)]) - // Adjacent dense segments each contribute their declared range. - expect(await store.actualRanges()).toEqual([ - [1, 2], - [3, 4] - ]) - }) - - // ---- pre-existing damage: a store sealed by the old writer ---------------- - - /** - * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and - * sidecar for a contiguous run, then rewrite the manifest so the segment - * declares a wider range than the frames it holds. This reproduces on disk - * exactly what the affected stores carry, without needing the old code. - */ - const sealSparseSegment = async (): Promise => { - await store.fold([gen(1), gen(2), gen(3)]) - const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any - // Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become - // holes inside a sealed range. - manifest.segments[0].lastGeneration = 5 - await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) - } - - it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { - await sealSparseSegment() - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - - // The frames it really holds still serve, byte-faithfully. - expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) - expect(await reopened.readRecords(3)).toHaveLength(2) - - // The holes answer "not packed" instead of throwing. This is the fix for - // the wedge: the old reader threw here on EVERY maintenance pass. - expect(await reopened.readDelta(4)).toBeNull() - expect(await reopened.readRecords(5)).toBeNull() - }) - - it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { - await sealSparseSegment() - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - // Declared 1..5; actually holds 1..3. The store seeds committedRanges from - // THIS, so generations 4 and 5 never become committed history again. - expect(await reopened.actualRanges()).toEqual([[1, 3]]) - }) - - it('a DENSE segment missing a frame is still loud damage', async () => { - // The other side of the branch: when the manifest claims a complete span, - // a missing frame means the manifest and sidecar disagree — real damage, - // and it must not be quietly downgraded to "unpacked". - await store.fold([gen(1), gen(2), gen(3)]) - const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` - const raw = (await storage.readRawBytes(idxPath))! - const { decode, encode } = await import('@msgpack/msgpack') - const idx = decode(raw) as any - // Drop generation 2's entry while the manifest still declares 3 frames. - idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) - await storage.writeRawBytes(idxPath, encode(idx)) - - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - await expect(reopened.readDelta(2)).rejects.toThrow( - /manifest and the sidecar disagree; packed history is damaged/ - ) - }) }) diff --git a/tests/unit/metadata-cold-read-guard.test.ts b/tests/unit/metadata-cold-read-guard.test.ts index b4f82f15..40d37de6 100644 --- a/tests/unit/metadata-cold-read-guard.test.ts +++ b/tests/unit/metadata-cold-read-guard.test.ts @@ -3,14 +3,10 @@ * reported cold `find({ where })` returning a silent `[]` on a freshly-opened * brain (a native metadata index that reports data but has not loaded its field * postings). This guard, the field-index counterpart of verifyGraphAdjacencyLive, - * probes a known persisted value on the first filtered find(). - * - * RE-POINTED to the health-gate law: the guard NEVER rebuilds and NEVER walks - * the store from a read — a read-path rebuild is exactly the dark-rebuild - * failure mode the law retires (open() alone owns building). When the probe - * cannot serve the known value it raises a loud MetadataIndexNotReadyError - * IMMEDIATELY, with no rebuild attempt in between — never a silent empty - * result that misrepresents existing data. + * probes a known persisted value on the first filtered find(): if the index does + * not serve it, brainy rebuilds and re-probes, and raises a loud + * MetadataIndexNotReadyError only if the rebuild still can't serve — never a + * silent empty result that misrepresents existing data. * * The 8.0 JS index cold-loads correctly, so we simulate the cold native failure * mode by intercepting the provider's getIdsForFilter/rebuild. @@ -46,19 +42,37 @@ describe('Metadata cold-read guard (#venue silent-[])', () => { mi.rebuild = origRebuild }) - it('cold index: verifyMetadataLive REFUSES immediately — find({where}) throws MetadataIndexNotReadyError, NEVER a silent [], and NEVER a rebuild attempt', async () => { + it('cold index: verifyMetadataLive self-heals via rebuild — find({where}) is correct, NOT silent []', async () => { const mi = brain.metadataIndex const origGetIds = mi.getIdsForFilter.bind(mi) - let rebuilds = 0 const origRebuild = mi.rebuild.bind(mi) + let cold = true brain._metadataVerified = false // re-arm the one-shot for this scenario - mi.getIdsForFilter = async () => [] // cold: the known value never resolves - mi.rebuild = async () => { rebuilds++; return origRebuild() } + mi.getIdsForFilter = async (...a: any[]) => (cold ? [] : origGetIds(...a)) + mi.rebuild = async () => { + await origRebuild() + cold = false // the rebuild warms the postings + } + try { + const res = await brain.find({ where: { status: 'active' }, limit: 100 }) + expect(res.length).toBe(1) // self-healed — the known entity is returned + } finally { + mi.getIdsForFilter = origGetIds + mi.rebuild = origRebuild + } + }) + + it('unrecoverably cold index: find({where}) throws MetadataIndexNotReadyError — never a silent []', async () => { + const mi = brain.metadataIndex + const origGetIds = mi.getIdsForFilter.bind(mi) + const origRebuild = mi.rebuild.bind(mi) + brain._metadataVerified = false + mi.getIdsForFilter = async () => [] // always cold; rebuild can't fix it + mi.rebuild = async () => {} try { await expect(brain.find({ where: { status: 'active' }, limit: 100 })).rejects.toBeInstanceOf( MetadataIndexNotReadyError ) - expect(rebuilds).toBe(0) // the guard never rebuilds from a read — it refuses loudly instead } finally { mi.getIdsForFilter = origGetIds mi.rebuild = origRebuild diff --git a/tests/unit/plugin-activation-loudness.test.ts b/tests/unit/plugin-activation-loudness.test.ts deleted file mode 100644 index 23e50a50..00000000 --- a/tests/unit/plugin-activation-loudness.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * @module tests/unit/plugin-activation-loudness - * @description The plugin-activation swallow closes. Two laws: - * (1) THE NOT-INSTALLED FREE PASS IS EXACT — a resolution failure earns the - * silent skip ONLY when it names the probed package itself, terminated - * where the name ends. A missing platform-binary SIBLING package - * ("-linux-x64-gnu" — what a deploy replacing node_modules - * mid-restart leaves), an inner file path, or a dependency failure is a - * BROKEN install and must fail loud. A production storm ran 90s of - * throttled WASM behind this exact prefix-match hole. - * (2) A GRACEFUL DECLINE IS NARRATED ON THE ALWAYS-ON CHANNEL — activate() - * returning false warns via prodLog, which `silent: true` cannot patch - * away; a declined accelerator is never an invisible degrade. - */ -import { describe, it, expect, vi, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { prodLog } from '../../src/utils/logger.js' - -const isNotInstalled = (error: unknown, pkg: string): boolean => - (Brainy as unknown as { - isPackageNotInstalledError(e: unknown, p: string): boolean - }).isPackageNotInstalledError(error, pkg) - -const resolutionError = (message: string): Error => { - const e = new Error(message) as Error & { code?: string } - e.code = 'ERR_MODULE_NOT_FOUND' - return e -} - -describe('the not-installed free pass is exact', () => { - const PKG = '@soulcraft/cor' - - it('the package itself, quoted or bare → not-installed (the one free path)', () => { - expect(isNotInstalled(resolutionError(`Cannot find package '${PKG}' imported from /app/x.js`), PKG)).toBe(true) - expect(isNotInstalled(resolutionError(`Cannot find module ${PKG}`), PKG)).toBe(true) - }) - - it('a missing platform-binary SIBLING package is a broken install, never not-installed', () => { - expect(isNotInstalled(resolutionError(`Cannot find package '${PKG}-linux-x64-gnu' imported from /app`), PKG)).toBe(false) - expect(isNotInstalled(resolutionError(`Failed to resolve ${PKG}-darwin-arm64`), PKG)).toBe(false) - }) - - it('an inner file path or a non-resolution error is never not-installed', () => { - expect(isNotInstalled(resolutionError(`Cannot find module '/app/node_modules/${PKG}/native/b.node'`), PKG)).toBe(false) - expect(isNotInstalled(new Error(`dlopen failed: wrong ELF class in ${PKG}`), PKG)).toBe(false) - }) -}) - -describe('a graceful decline is narrated on the always-on channel', () => { - afterEach(() => vi.restoreAllMocks()) - - it('activate() → false warns via prodLog even under silent: true', async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - const warn = vi.spyOn(prodLog, 'warn') - const brain: any = new Brainy({ - requireSubtype: false, - storage: { type: 'memory' }, - silent: true, - dimensions: 384 - }) - brain.use({ name: 'declining-accelerator', activate: async () => false }) - await brain.init() - try { - expect( - warn.mock.calls.some((c) => String(c[0]).includes('"declining-accelerator" declined activation')) - ).toBe(true) - } finally { - await brain.close().catch(() => {}) - } - }) -}) diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts deleted file mode 100644 index 8bf9d357..00000000 --- a/tests/unit/release/wall-entry.test.ts +++ /dev/null @@ -1,395 +0,0 @@ -/** - * scripts/wall-entry.mjs — the mechanical releases-wall entry. - * - * The script's only real interface is its CLI (it has no importable - * exports by design — one door, no parallel API to drift from it), so - * these tests spawn it exactly as scripts/release.sh does: as a child - * process, against a fixture CHANGELOG and a throwaway local bare repo - * standing in for git@source.soulcraft.com:soulcraftlabs/releases.git - * (--remote) plus a throwaway cache directory (--cache-dir) standing in - * for ~/.cache/soulcraft-releases — never the real remote, never the - * real developer cache. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { execFileSync } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync, readFileSync, chmodSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' - -const SCRIPT = join(process.cwd(), 'scripts/wall-entry.mjs') - -/** Run the script and capture the outcome without throwing on a non-zero exit. */ -function run(args: string[], cwd: string): { status: number; stdout: string; stderr: string } { - try { - const stdout = execFileSync('node', [SCRIPT, ...args], { cwd, encoding: 'utf8' }) - return { status: 0, stdout, stderr: '' } - } catch (err: any) { - return { status: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' } - } -} - -function git(args: string[], cwd: string): string { - return execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim() -} - -const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' - -/** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ -function buildChangelog(entries: Array<{ version: string; date: string; bullets: string[] }>): string { - const body = entries - .map( - (e) => - `### [${e.version}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/vX...v${e.version}) (${e.date})\n\n` + - e.bullets.map((b) => `- ${b} (abc1234)`).join('\n') + - '\n', - ) - .join('\n') - return CHANGELOG_HEADER + '\n' + body -} - -function wallFile(product: string, entries: unknown[]): string { - return JSON.stringify({ product, entries }, null, 2) + '\n' -} - -const BASE_ENTRY = { - version: '10.4.11', - date: '2026-09-02', - headline: 'A faster open', - items: ['A faster open.'], - url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11', - thumb: null, -} - -/** A throwaway bare repo standing in for the real soulcraftlabs/releases remote. */ -function initBareRemote(): string { - const remoteDir = mkdtempSync(join(tmpdir(), 'wall-remote-')) - execFileSync('git', ['init', '--bare', '-b', 'main', remoteDir]) - return remoteDir -} - -/** Seed the bare remote with an initial .json, via a throwaway clone. */ -function seedRemote(remoteDir: string, product: string, entries: unknown[]): void { - const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-')) - execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) - git(['config', 'user.email', 'seed@example.com'], seedDir) - git(['config', 'user.name', 'Seed'], seedDir) - writeFileSync(join(seedDir, `${product}.json`), wallFile(product, entries)) - git(['add', `${product}.json`], seedDir) - git(['commit', '-m', 'seed'], seedDir) - git(['push', 'origin', 'main'], seedDir) - rmSync(seedDir, { recursive: true, force: true }) -} - -/** Read .json back out of the bare remote's main tip, via a throwaway clone. */ -function readRemote(remoteDir: string, product: string): any { - const readDir = mkdtempSync(join(tmpdir(), 'wall-read-')) - execFileSync('git', ['clone', remoteDir, readDir], { stdio: 'ignore' }) - const data = JSON.parse(readFileSync(join(readDir, `${product}.json`), 'utf8')) - rmSync(readDir, { recursive: true, force: true }) - return data -} - -/** Reject every push — stands in for any push failure (including a genuine - * non-fast-forward raced by a concurrent release rail), which this script - * treats identically: refuse loudly, name the cure, touch nothing further. */ -function makeRemoteRejectPushes(remoteDir: string): void { - const hookPath = join(remoteDir, 'hooks', 'pre-receive') - writeFileSync(hookPath, '#!/bin/sh\necho "remote: simulated push rejection" >&2\nexit 1\n') - chmodSync(hookPath, 0o755) -} - -let dir: string -let remoteDir: string -let cacheDir: string - -beforeEach(() => { - dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) - remoteDir = initBareRemote() - cacheDir = join(mkdtempSync(join(tmpdir(), 'wall-cache-')), 'soulcraft-releases') -}) - -afterEach(() => { - rmSync(dir, { recursive: true, force: true }) - rmSync(remoteDir, { recursive: true, force: true }) - rmSync(cacheDir, { recursive: true, force: true }) -}) - -describe('wall-entry.mjs — generate + publish', () => { - it('derives headline from the first bullet and items from every bullet, hashes stripped, and pushes it to the remote', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - writeFileSync( - join(dir, 'CHANGELOG.md'), - buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), - ) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/wrote v10\.4\.12.*pushed/i) - - const wall = readRemote(remoteDir, 'open-brainy') - expect(wall.entries).toHaveLength(2) - expect(wall.entries[0]).toEqual({ - version: '10.4.12', - date: '2026-09-03', - headline: 'fix(wall): mechanize the entry', - items: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'], - url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.12', - thumb: null, - }) - // the older entry stays put, still second - expect(wall.entries[1].version).toBe('10.4.11') - }) - - it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }])) - - run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) - - const wall = readRemote(remoteDir, 'open-brainy') - expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) - }) - - it('replaces an entry with the same version instead of duplicating it — idempotent re-runs', () => { - seedRemote(remoteDir, 'open-brainy', [ - { ...BASE_ENTRY, headline: 'stale headline, pre-fix' }, - { ...BASE_ENTRY, version: '10.4.10' }, - ]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: the corrected headline'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/replaced v10\.4\.11/i) - - const wall = readRemote(remoteDir, 'open-brainy') - expect(wall.entries).toHaveLength(2) // not 3 — replaced, not duplicated - expect(wall.entries[0].version).toBe('10.4.11') - expect(wall.entries[0].headline).toBe('fix: the corrected headline') - expect(wall.entries[1].version).toBe('10.4.10') - }) - - it('a re-run with byte-identical content commits nothing and still succeeds', () => { - // headline always equals items[0] for a derived entry, so this fixture - // (unlike BASE_ENTRY, whose headline/items intentionally diverge for the - // shape-only tests below) has to keep the two in lockstep to ever roundtrip. - const stableEntry = { ...BASE_ENTRY, headline: 'A faster open.', items: ['A faster open.'] } - seedRemote(remoteDir, 'open-brainy', [stableEntry]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['A faster open.'] }])) - const before = readRemote(remoteDir, 'open-brainy') - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/nothing to commit/i) - expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) - }) - - it('derives the public package-page permalink for the product engine (private repo, never null)', () => { - seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: 'https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.5' }]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) - - const result = run( - ['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - expect(result.status).toBe(0) - - const wall = readRemote(remoteDir, 'brainy') - expect(wall.entries[0].url).toBe('https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.6') - expect(wall.entries[0].thumb).toBeNull() - }) - - it('refuses a product with no permalink pattern, naming the cure', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['feat: first'] }])) - - const result = run(['--product', 'mystery', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) - expect(result.status).not.toBe(0) - expect(result.stderr).toMatch(/no permalink pattern for product "mystery"/) - expect(result.stderr).toMatch(/never carry url: null/) - }) - - it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { - seedRemote(remoteDir, 'open-brainy', []) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - const beforeSha = git(['rev-parse', 'main'], remoteDir) - - const result = run( - ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) - expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) - }) - - it('refuses by naming the cure when the remote cannot be cloned', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) - const noSuchRemote = join(tmpdir(), 'wall-remote-does-not-exist-' + Date.now()) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', noSuchRemote, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/cannot clone/i) - expect(result.stderr).toMatch(/cure:/i) - }) - - it('refuses by naming the cure, and touches no remote, when the fetched wall fails shape validation', () => { - const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-broken-')) - execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) - git(['config', 'user.email', 'seed@example.com'], seedDir) - git(['config', 'user.name', 'Seed'], seedDir) - writeFileSync( - join(seedDir, 'open-brainy.json'), - JSON.stringify({ product: 'open-brainy', entries: [{ version: '10.4.11', date: '2026-09-02', items: ['x'], url: null }] }, null, 2), - ) - git(['add', 'open-brainy.json'], seedDir) - git(['commit', '-m', 'seed broken'], seedDir) - git(['push', 'origin', 'main'], seedDir) - rmSync(seedDir, { recursive: true, force: true }) - const beforeSha = git(['rev-parse', 'main'], remoteDir) - - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/fails shape validation/i) - expect(result.stderr).toMatch(/missing key\(s\) headline/i) - expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) - }) - - it('refuses by naming the cure when the remote rejects the push (stands in for a raced non-fast-forward)', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - makeRemoteRejectPushes(remoteDir) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/push to .* failed/i) - expect(result.stderr).toMatch(/cure:/i) - }) - - it('refuses a cross-product write when the file\'s "product" field does not match --product', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-mismatch-')) - execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) - git(['config', 'user.email', 'seed@example.com'], seedDir) - git(['config', 'user.name', 'Seed'], seedDir) - const corrupted = JSON.parse(readFileSync(join(seedDir, 'open-brainy.json'), 'utf8')) - corrupted.product = 'brainy' - writeFileSync(join(seedDir, 'open-brainy.json'), JSON.stringify(corrupted, null, 2) + '\n') - git(['add', 'open-brainy.json'], seedDir) - git(['commit', '-m', 'corrupt product field'], seedDir) - git(['push', 'origin', 'main'], seedDir) - rmSync(seedDir, { recursive: true, force: true }) - - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) - - const result = run( - ['--product', 'open-brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/product "brainy".*--product "open-brainy"/i) - }) -}) - -describe('wall-entry.mjs — --dry-run', () => { - it('prints the entry and the target path, and touches neither the cache dir nor the remote', () => { - seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: a dry run'] }])) - const beforeSha = git(['rev-parse', 'main'], remoteDir) - - const result = run( - ['--dry-run', '--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], - dir, - ) - - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/would write to/i) - expect(result.stdout).toMatch(/"version": "10\.4\.12"/) - expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) - }) -}) - -describe('wall-entry.mjs — --check', () => { - it('passes a well-formed, newest-first file with no duplicates', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(0) - expect(result.stdout).toMatch(/OK/) - }) - - it('passes a file where "thumb" is entirely absent (optional per the HQ contract)', () => { - const { thumb, ...noThumb } = BASE_ENTRY as any - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [noThumb])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(0) - }) - - it('catches a missing entry key', () => { - const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'] } // no "url" - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/missing key\(s\) url/) - }) - - it('catches an unexpected top-level key (e.g. the retired "history" field)', () => { - const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) - raw.history = 'retired field' - writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/unexpected key\(s\) history/) - }) - - it('catches entries that are not newest-first', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, version: '10.4.10' }, BASE_ENTRY])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/not newest-first/) - }) - - it('catches a duplicate version even with identical entries', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/duplicate version 10\.4\.11/) - }) - - it('catches an empty items array', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, items: [] }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/"items" must be a non-empty array/) - }) - - it('catches a malformed date', () => { - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, date: '09/03/2026' }])) - const result = run(['--check', '--file', 'wall.json'], dir) - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/"date" must be a YYYY-MM-DD string/) - }) -}) diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index d4d268ac..c43b2cf2 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -63,9 +63,6 @@ function inGate(rel: string): boolean { return ( rel.startsWith('tests/unit/') || rel.startsWith('tests/integration/') || - // The lifecycle biography lane — included by the integration config - // ('tests/lifecycle/**/*.test.ts'; see tests/lifecycle/README.md). - rel.startsWith('tests/lifecycle/') || rel.endsWith('.unit.test.ts') || rel.endsWith('.integration.test.ts') ) diff --git a/tests/unit/utils/indexReadiness.test.ts b/tests/unit/utils/indexReadiness.test.ts deleted file mode 100644 index 75504c71..00000000 --- a/tests/unit/utils/indexReadiness.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -/** - * @module tests/unit/utils/indexReadiness - * @description Pins for the read-gate authority, {@link assessProviderHealth}, and - * its older sibling {@link assessIndexReadiness}. The health-gate law: a provider's - * NAMED, synchronous, O(1) health report — when exposed — REPLACES the `isReady()`/ - * size-heuristic fallback as the read gate's source of truth. A throw from - * `healthReport()` is a CONTRACT VIOLATION (never read as healthy, never swallowed - * into "unknown"); an `unledgered` family is UNKNOWN (never healthy, never broken — - * `serving` is always the provider's own verdict, verbatim). - */ -import { describe, it, expect } from 'vitest' -import { assessIndexReadiness, assessProviderHealth } from '../../../src/utils/indexReadiness.js' -import type { HealthReport, LedgerInvariantResult } from '../../../src/plugin.js' - -function invariant(overrides: Partial = {}): LedgerInvariantResult { - return { - name: 'manifest-residency', - holds: true, - detail: 'ok', - heal: 'none', - source: 'ledger', - ...overrides - } -} - -function report(overrides: Partial = {}): HealthReport { - return { - provider: 'vector', - healthy: true, - serving: true, - invariants: [], - // A FIXED stamp, never Date.now(): the pin at :98 compares two - // independently-built reports, and a live clock made them differ by 1ms - // whenever the millisecond ticked between the two calls — a plant-lane - // red that had nothing to do with the code under test. - checkedAt: 1_700_000_000_000, - durationMs: 1, - generation: 1, - unledgered: [], - ...overrides - } -} - -describe('assessIndexReadiness (legacy isReady() classifier)', () => { - it('unknown when the provider is null/undefined', () => { - expect(assessIndexReadiness(null)).toBe('unknown') - expect(assessIndexReadiness(undefined)).toBe('unknown') - }) - - it('unknown when isReady() is absent', () => { - expect(assessIndexReadiness({})).toBe('unknown') - }) - - it('ready / not-ready mirror isReady()', () => { - expect(assessIndexReadiness({ isReady: () => true })).toBe('ready') - expect(assessIndexReadiness({ isReady: () => false })).toBe('not-ready') - }) -}) - -describe('assessProviderHealth — the read-gate authority', () => { - it('via "none": no provider at all', () => { - const a = assessProviderHealth(null) - expect(a.via).toBe('none') - expect(a.readiness).toBe('unknown') - expect(a.report).toBeNull() - expect(a.reasons.length).toBeGreaterThan(0) - }) - - it('via "size-heuristic": provider exposes neither healthReport() nor isReady()', () => { - const a = assessProviderHealth({}) - expect(a.via).toBe('size-heuristic') - expect(a.readiness).toBe('unknown') - expect(a.report).toBeNull() - }) - - it('via "is-ready": provider exposes isReady() but no healthReport() — ready', () => { - const a = assessProviderHealth({ isReady: () => true }) - expect(a.via).toBe('is-ready') - expect(a.readiness).toBe('ready') - expect(a.reasons).toEqual([]) - }) - - it('via "is-ready": isReady() === false — not-ready with a reason', () => { - const a = assessProviderHealth({ isReady: () => false }) - expect(a.via).toBe('is-ready') - expect(a.readiness).toBe('not-ready') - expect(a.reasons.length).toBeGreaterThan(0) - }) - - it('healthReport() present REPLACES isReady() — serving:true wins even if isReady() lies false', () => { - const p = { isReady: () => false, healthReport: () => report({ serving: true }) } - const a = assessProviderHealth(p) - expect(a.via).toBe('health-report') - expect(a.readiness).toBe('ready') - }) - - it('serving:true, healthy:true, no invariants failing → ready, no reasons', () => { - const p = { healthReport: () => report({ serving: true, healthy: true }) } - const a = assessProviderHealth(p) - expect(a.readiness).toBe('ready') - expect(a.reasons).toEqual([]) - expect(a.report).toEqual(report({ serving: true, healthy: true })) - }) - - it('serving:false with a named heal:"rebuild" failing invariant → not-ready, reason names it', () => { - const failing = invariant({ name: 'posted-count-floor', holds: false, heal: 'rebuild', detail: 'posted 10 < canonical 20' }) - const p = { healthReport: () => report({ serving: false, healthy: false, invariants: [failing] }) } - const a = assessProviderHealth(p) - expect(a.readiness).toBe('not-ready') - expect(a.reasons.some((r) => r.includes('posted-count-floor') && r.includes('heal:rebuild') && r.includes('posted 10 < canonical 20'))).toBe(true) - }) - - it('unledgered-only report (serving:true, no failing invariant) → ready, reason names the unledgered family', () => { - const p = { healthReport: () => report({ serving: true, healthy: true, unledgered: ['canonical-verb-coverage'] }) } - const a = assessProviderHealth(p) - expect(a.readiness).toBe('ready') - expect(a.reasons.some((r) => r.includes('unledgered') && r.includes('canonical-verb-coverage'))).toBe(true) - }) - - it('UNLEDGERED IS UNKNOWN: an unledgered family never flips a NOT-serving provider to ready', () => { - const failing = invariant({ holds: false, heal: 'rebuild', name: 'x' }) - const p = { healthReport: () => report({ serving: false, healthy: false, invariants: [failing], unledgered: ['some-family'] }) } - const a = assessProviderHealth(p) - expect(a.readiness).toBe('not-ready') - }) - - it('serving:true, healthy:false with a heal:"repair" failure → still ready (degraded-but-serving)', () => { - const failing = invariant({ name: 'stale-counter', holds: false, heal: 'repair', detail: 'counter drift' }) - const p = { healthReport: () => report({ serving: true, healthy: false, invariants: [failing] }) } - const a = assessProviderHealth(p) - expect(a.readiness).toBe('ready') - expect(a.reasons.some((r) => r.includes('stale-counter') && r.includes('heal:repair'))).toBe(true) - }) - - it('healthReport() that THROWS is a CONTRACT VIOLATION: not-ready, via health-report, reason names the throw — never "unknown"', () => { - const p = { healthReport: () => { throw new Error('mmap window busy') } } - const a = assessProviderHealth(p) - expect(a.via).toBe('health-report') - expect(a.readiness).toBe('not-ready') - expect(a.report).toBeNull() - expect(a.reasons.some((r) => r.includes('mmap window busy'))).toBe(true) - expect(a.readiness).not.toBe('unknown') - }) - - it('healthReport() that throws a non-Error value still produces a named reason (String(err))', () => { - const p = { healthReport: () => { throw 'boom' } } - const a = assessProviderHealth(p) - expect(a.readiness).toBe('not-ready') - expect(a.reasons.some((r) => r.includes('boom'))).toBe(true) - }) - - it('the returned report carries the generation for narration dedup', () => { - const p = { healthReport: () => report({ generation: 42 }) } - const a = assessProviderHealth(p) - expect(a.report?.generation).toBe(42) - }) -}) diff --git a/tests/unit/utils/metadataIndex-watermark.test.ts b/tests/unit/utils/metadataIndex-watermark.test.ts index 6d6f6e28..6c195b35 100644 --- a/tests/unit/utils/metadataIndex-watermark.test.ts +++ b/tests/unit/utils/metadataIndex-watermark.test.ts @@ -11,11 +11,8 @@ * Same rule, same verdict names as the shipped aggregation machinery * (AggregationIndex.stateAdoptionVerdict). * - * The verdict is computed at init and consumed via - * {@link MetadataIndexManager.applyWatermarkCatchup} — the coordinator - * (`Brainy.performInit`) calls it right after `init()`, with an open fact - * scan when the verdict is `'catchup'`. This file pins both halves: the - * verdict computation (above) and the fold/no-op/demotion behavior below. + * The verdict is COMPUTED AND EXPOSED only — these pins assert no rebuild + * trigger changed; acting on 'catchup' lands with the coordinator's wiring. */ import { describe, it, expect, vi, afterEach } from 'vitest' import { v4 as uuidv4 } from 'uuid' @@ -25,46 +22,6 @@ import { } from '../../../src/utils/metadataIndex.js' import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' import { prodLog } from '../../../src/utils/logger.js' -import type { CommitFact, FactScanBatch, FactScanHandle } from '../../../src/db/factLog.js' - -/** A fact scan handle over an in-memory list of facts — batches them one - * fact at a time (batch size is irrelevant to the fold, which reads - * `batch.facts` only). */ -function fakeScan(facts: CommitFact[]): FactScanHandle { - return { - headGeneration: facts.length > 0 ? facts[facts.length - 1].generation : 0, - segmentCount: 1, - approxFactCount: facts.length, - async *batches(): AsyncGenerator { - for (const fact of facts) { - yield { - facts: [fact], - firstGeneration: fact.generation, - lastGeneration: fact.generation, - factCount: 1, - byteSize: 0, - segmentId: 'fake' - } - } - }, - summary: () => ({ factsYielded: facts.length, segmentsRead: 1 }) - } -} - -/** One noun after-image fact — the flat-record shape (no nested `metadata` - * key), matching this file's existing `writeArtifact` convention. */ -function nounAdd(generation: number, id: string, metadata: Record): CommitFact { - return { - generation, - timestamp: Date.now(), - ops: [{ kind: 'noun', id, record: { metadata, vector: null } }] - } -} - -/** One noun tombstone fact. */ -function nounDelete(generation: number, id: string): CommitFact { - return { generation, timestamp: Date.now(), ops: [{ kind: 'noun', id, record: null }] } -} /** Fresh storage with a controllable committed generation. */ async function makeStorage(committed: number | null): Promise { @@ -212,106 +169,3 @@ describe('metadata index — watermark stamp + three-way load verdict', () => { expect(await storage.getMetadata(METADATA_INDEX_STAMP_KEY)).toBeNull() }) }) - -describe('metadata index — applyWatermarkCatchup (the coordinator door)', () => { - it("an 'adopt' verdict performs zero index writes", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('adopt') - - const addSpy = vi.spyOn(index, 'addToIndex') - const removeSpy = vi.spyOn(index, 'removeFromIndex') - - const result = await index.applyWatermarkCatchup(null) - - expect(result).toEqual({ action: 'noop' }) - expect(addSpy).not.toHaveBeenCalled() - expect(removeSpy).not.toHaveBeenCalled() - }) - - it('a catchup window folding an add, an update (same id twice), and a delete → the index serves exactly the final state', async () => { - const storage = await makeStorage(5) - - // Session 1: two pre-existing entities, stamped at generation 5. - const survivorId = uuidv4() - const deletedId = uuidv4() - { - const index = new MetadataIndexManager(storage) - await index.init() - await index.addToIndex(survivorId, { status: 'active' }) - await index.addToIndex(deletedId, { status: 'active' }) - index.stampWatermark(5) - await index.flush() - } - - // The store advanced to generation 8 without another metadata flush — - // the exact shape a crash-then-adopt-reopen leaves behind. - setCommitted(storage, 8) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('catchup') - expect(index.watermarkGap()).toEqual({ from: 5, to: 8 }) - - const addedId = uuidv4() - const scan = fakeScan([ - nounAdd(6, addedId, { status: 'new' }), // add - nounAdd(7, addedId, { status: 'updated' }), // update — same id twice - nounDelete(8, deletedId) // delete - ]) - - const result = await index.applyWatermarkCatchup(scan) - - expect(result.action).toBe('caught-up') - expect(result.window).toEqual({ from: 5, to: 8 }) - expect(result.factsApplied).toBe(3) - expect(result.nounsApplied).toBe(3) - expect(result.verbsApplied).toBe(0) - - // Final state: the added/updated id serves ONLY its final value... - expect(await index.getIds('status', 'updated')).toEqual([addedId]) - expect(await index.getIds('status', 'new')).toEqual([]) // stale value gone - // ...the deleted id is gone... - expect(await index.getIds('status', 'active')).toEqual([survivorId]) - // ...and the untouched survivor is unaffected. - expect(await index.getIds('status', 'active')).toContain(survivorId) - - // The window is certified: watermark stamped at `to`, and a fresh - // reopen now verdicts 'adopt'. - expect(index.watermark()).toBe(8) - const reopened = await reopen(storage) - expect(reopened.watermarkVerdict()).toBe('adopt') - }) - - it("a 'rescan' verdict runs the existing rebuild path instead of folding", async () => { - const storage = await makeStorage(9) - await writeArtifact(storage, 9) - setCommitted(storage, 4) // a truncated log pulled the watermark back — stamp ABOVE committed → rescan - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('rescan') - - const rebuildSpy = vi.spyOn(index, 'rebuild') - const result = await index.applyWatermarkCatchup(null) - - expect(result.action).toBe('rescan') - expect(result.reason).toBeTruthy() - expect(rebuildSpy).toHaveBeenCalledTimes(1) - }) - - it("a 'catchup' verdict with no fact log available demotes to rebuild, narrated", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - setCommitted(storage, 8) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('catchup') - - const rebuildSpy = vi.spyOn(index, 'rebuild') - const result = await index.applyWatermarkCatchup(null) // no scan — no fact log - - expect(result.action).toBe('rescan') - expect(result.reason).toContain('no fact log') - expect(rebuildSpy).toHaveBeenCalledTimes(1) - }) -}) diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts index 805dd40d..7e5212b8 100644 --- a/tests/unit/utils/paramValidation.test.ts +++ b/tests/unit/utils/paramValidation.test.ts @@ -149,33 +149,7 @@ describe('Zero-Config Parameter Validation', () => { type: NounType.Document } as AddParams)).toThrow('Invalid add() parameters: Missing required field \'data\'') }) - - it('should accept an empty string as real data — only null/undefined is "missing"', () => { - // A legitimate empty file's first write: '' is content, not absence. - expect(() => validateAddParams({ - data: '', - type: NounType.Document - })).not.toThrow() - - // null/undefined (with no vector) is still the genuine missing-field case. - expect(() => validateAddParams({ - data: null as any, - type: NounType.Document - })).toThrow('Invalid add() parameters: Missing required field \'data\'') - expect(() => validateAddParams({ - data: undefined, - type: NounType.Document - })).toThrow('Invalid add() parameters: Missing required field \'data\'') - }) - - it('deferEmbedding accepts empty-string data (real content, not absence)', () => { - expect(() => validateAddParams({ - data: '', - type: NounType.Document, - deferEmbedding: true - } as AddParams)).not.toThrow() - }) - + it('should validate NounType', () => { expect(() => validateAddParams({ data: 'test', @@ -216,22 +190,7 @@ describe('Zero-Config Parameter Validation', () => { id: 'test-id' })).toThrow('must specify at least one field to update') }) - - it('empty-string data counts as a real field to update (truncating content)', () => { - expect(() => validateUpdateParams({ - id: 'test-id', - data: '' - })).not.toThrow() - }) - - it('deferEmbedding accepts empty-string data on update', () => { - expect(() => validateUpdateParams({ - id: 'test-id', - data: '', - deferEmbedding: true - } as UpdateParams)).not.toThrow() - }) - + it('should validate NounType if changing', () => { expect(() => validateUpdateParams({ id: 'test-id', diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts index 0905f298..49ca6426 100644 --- a/tests/unit/vector-cold-read-guard.test.ts +++ b/tests/unit/vector-cold-read-guard.test.ts @@ -3,14 +3,9 @@ * @description Pattern-A / Finding 1: a pure semantic find({ query }) has no * filter, so verifyMetadataLive never fires — nothing guarded the vector index. * A cold native vector index that loaded its COUNT but not its serving structure - * returned a silent []. verifyVectorLive() closes that: the health-report/isReady() - * authority first, else a known-vector self-match probe. - * - * RE-POINTED to the health-gate law: the guard NEVER rebuilds and NEVER walks - * the store from a read — a read-path rebuild is exactly the dark-rebuild - * failure mode the law retires (open() alone owns building). A not-serving - * signal (from either strategy) THROWS VectorIndexNotReadyError immediately, - * with no rebuild attempt in between — never a silent empty result. + * returned a silent []. verifyVectorLive() closes that: honest isReady() first, + * else a known-vector self-match probe; self-heal (rebuild) or throw + * VectorIndexNotReadyError — never a silent empty result. */ import { describe, it, expect, beforeEach } from 'vitest' import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js' @@ -39,37 +34,50 @@ describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semant vi.rebuild = origRebuild }) - it('cold index (no isReady()): verifyVectorLive REFUSES immediately — throws VectorIndexNotReadyError, NEVER rebuilds', async () => { + it('cold index: verifyVectorLive self-heals via rebuild — semantic find is correct, NOT silent []', async () => { const vi = brain.index const origSearch = vi.search.bind(vi) - let rebuilds = 0 const origRebuild = vi.rebuild.bind(vi) + let cold = true brain._vectorVerified = false - // size()>0 (count present) but search never returns a hit for the known vector. - vi.search = async () => [] - vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } + // size()>0 (count present) but search returns nothing until a rebuild warms it. + vi.search = async (...a: any[]) => (cold ? [] : origSearch(...a)) + vi.rebuild = async (...a: any[]) => { await origRebuild(...a); cold = false } try { - await expect( - brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - ).rejects.toBeInstanceOf(VectorIndexNotReadyError) - expect(rebuilds).toBe(0) // the guard never rebuilds from a read — it refuses loudly instead + const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) + expect(res.length).toBeGreaterThan(0) // self-healed } finally { vi.search = origSearch; vi.rebuild = origRebuild } }) - it('native provider reporting isReady()===false THROWS immediately — never rebuilds', async () => { + it('unrecoverably cold index: semantic find throws VectorIndexNotReadyError', async () => { const vi = brain.index - let rebuilds = 0 + const origSearch = vi.search.bind(vi) const origRebuild = vi.rebuild.bind(vi) brain._vectorVerified = false - vi.isReady = () => false - vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } + vi.search = async () => [] // always cold; rebuild can't fix it + vi.rebuild = async () => {} try { await expect( brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) ).rejects.toBeInstanceOf(VectorIndexNotReadyError) - expect(rebuilds).toBe(0) // a not-ready report throws immediately — it is never a rebuild trigger + } finally { + vi.search = origSearch; vi.rebuild = origRebuild + } + }) + + it('native provider reporting isReady()===false rebuilds, then serves', async () => { + const vi = brain.index + const origRebuild = vi.rebuild.bind(vi) + let ready = false + brain._vectorVerified = false + vi.isReady = () => ready + vi.rebuild = async (...a: any[]) => { await origRebuild(...a); ready = true } + try { + const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) + expect(ready).toBe(true) // rebuild ran because isReady() was false + expect(res).toBeDefined() } finally { delete vi.isReady; vi.rebuild = origRebuild } diff --git a/tests/unit/vfs-readdir-recursive.test.ts b/tests/unit/vfs-readdir-recursive.test.ts deleted file mode 100644 index 2ee8a775..00000000 --- a/tests/unit/vfs-readdir-recursive.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * vfs.readdir()'s `recursive` option: typed since 7.30 but never read, so it - * silently behaved exactly like `recursive: false`. This pins the real, - * documented contract: a recursive listing returns every descendant (files - * AND directories, all depths) as paths RELATIVE TO THE QUERIED DIRECTORY — - * the same convention Node's `fs.readdir(dir, { recursive: true })` uses — - * for both the plain string-array form and the `withFileTypes` VFSDirent - * form (whose `name` carries that same relative path when recursive). - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import type { VFSDirent } from '../../src/vfs/types.js' - -describe('vfs.readdir() recursive option', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ - requireSubtype: false, - storage: { type: 'memory' }, - silent: true - }) - await brain.init() - - // Build: - // /a/b.txt - // /a/sub/c.txt - // /a/sub/deeper/d.txt - // /a/sub2/ (empty directory) - await brain.vfs.writeFile('/a/b.txt', 'B') - await brain.vfs.writeFile('/a/sub/c.txt', 'C') - await brain.vfs.writeFile('/a/sub/deeper/d.txt', 'D') - await brain.vfs.mkdir('/a/sub2', { recursive: true }) - }) - - afterEach(async () => { - await brain.close() - }) - - it('non-recursive (default) still returns only direct children, by basename', async () => { - const entries = await brain.vfs.readdir('/a') as string[] - expect([...entries].sort()).toEqual(['b.txt', 'sub', 'sub2']) - }) - - it('recursive: true returns every descendant as a path relative to the queried directory', async () => { - const entries = await brain.vfs.readdir('/a', { recursive: true }) as string[] - expect([...entries].sort()).toEqual([ - 'b.txt', - 'sub', - 'sub/c.txt', - 'sub/deeper', - 'sub/deeper/d.txt', - 'sub2' - ]) - }) - - it('recursive: true at the root has no leading slash on relative entries', async () => { - const entries = await brain.vfs.readdir('/', { recursive: true }) as string[] - expect(entries).toContain('a') - expect(entries).toContain('a/b.txt') - expect(entries).toContain('a/sub/deeper/d.txt') - for (const entry of entries) { - expect(entry.startsWith('/')).toBe(false) - } - }) - - it('recursive + withFileTypes: VFSDirent.name is the relative path, .path stays absolute', async () => { - const entries = await brain.vfs.readdir('/a', { - recursive: true, - withFileTypes: true - }) as VFSDirent[] - - const byName = new Map(entries.map((e) => [e.name, e])) - - const nested = byName.get('sub/deeper/d.txt') - expect(nested).toBeDefined() - expect(nested!.path).toBe('/a/sub/deeper/d.txt') - expect(nested!.type).toBe('file') - - const nestedDir = byName.get('sub/deeper') - expect(nestedDir).toBeDefined() - expect(nestedDir!.path).toBe('/a/sub/deeper') - expect(nestedDir!.type).toBe('directory') - - // Non-recursive VFSDirent behavior is unchanged: name is the basename. - const direct = await brain.vfs.readdir('/a', { withFileTypes: true }) as VFSDirent[] - const directEntry = direct.find((e) => e.path === '/a/b.txt') - expect(directEntry?.name).toBe('b.txt') - }) - - it('recursive + filter composes: only files survive a type filter', async () => { - const entries = await brain.vfs.readdir('/a', { - recursive: true, - filter: { type: 'file' } - }) as string[] - expect([...entries].sort()).toEqual(['b.txt', 'sub/c.txt', 'sub/deeper/d.txt']) - }) -}) diff --git a/tests/vfs/vfs.unit.test.ts b/tests/vfs/vfs.unit.test.ts index 4b4ba8d2..5ea79377 100644 --- a/tests/vfs/vfs.unit.test.ts +++ b/tests/vfs/vfs.unit.test.ts @@ -53,35 +53,6 @@ describe('VirtualFileSystem - Production Tests', () => { expect(exists).toBe(true) }) - it('should write and read an empty (0-byte) file end-to-end', async () => { - // Pin: validateAddParams() used to treat '' as a missing 'data' field - // (falsy check), so a legitimate empty file's FIRST write threw - // "Missing required field 'data'". '' is real content, not an absent - // field — only null/undefined is absent. - const path = '/empty.txt' - - await vfs.writeFile(path, '') - - const result = await vfs.readFile(path) - expect(result.toString()).toBe('') - - const exists = await vfs.exists(path) - expect(exists).toBe(true) - - const stats = await vfs.stat(path) - expect(stats.size).toBe(0) - expect(stats.isFile()).toBe(true) - - // The file lists like any other. - const entries = await vfs.readdir('/') as string[] - expect(entries).toContain('empty.txt') - - // Overwriting it back to empty (truncate) must also succeed. - await vfs.writeFile(path, 'not empty anymore') - await vfs.writeFile(path, '') - expect((await vfs.readFile(path)).toString()).toBe('') - }) - it('should handle binary files', async () => { const binaryData = Buffer.from([0x00, 0x01, 0x02, 0xFF]) const path = '/binary.dat'