diff --git a/CHANGELOG.md b/CHANGELOG.md index a54d609e..56757f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,48 +2,19 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) - -- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) -- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25) -- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40) -- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2) -- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927) -- docs: measurements in public history carry numbers, not provenance (02c61636) -- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010) -- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950) -- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9) -- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5) -- perf(open): answer "are there any entities?" with one directory read (417ddb51) -- perf(generations): discover generations by directory name, not by walking the log (9dd39921) -- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc) -- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca) -- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f) -- chore: keep the generated neural stamps at main's values (c1f09723) -- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3) -- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02) -- feat(open): open never waits for a provider that is rebuilding itself (131daa08) -- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f) -- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6) -- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b) -- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f) -- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c) -- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b) - - -### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) +### [10.4.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2...v10.4.3) (2026-08-27) - Merge branch 'next/open-brainy-rename' (a58372f0) - chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) - docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) -### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) +### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) - docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) -### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) +### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) - fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) @@ -60,23 +31,23 @@ All notable changes to this project will be documented in this file. See [standa - chore(release): 10.4.1-rc.1 (7870dc40) -### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26) +### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) - docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) -### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) +### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) - docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) -### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) +### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) - feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) -### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) +### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) - fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) - Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) @@ -85,7 +56,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) -### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) +### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) - test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) - feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) @@ -96,7 +67,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) -### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) +### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) - ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) - chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) @@ -109,13 +80,13 @@ All notable changes to this project will be documented in this file. See [standa - ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) -### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) +### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18) - docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) - fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) -### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) +### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) - fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) @@ -124,14 +95,14 @@ All notable changes to this project will be documented in this file. See [standa - feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) -### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) +### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17) - docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) - ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) - fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) -### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) +### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) - fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) @@ -140,7 +111,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) -### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) +### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12) - fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) - fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) @@ -172,7 +143,7 @@ All notable changes to this project will be documented in this file. See [standa - test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) -### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) +### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04) - docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) - fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) @@ -207,7 +178,7 @@ All notable changes to this project will be documented in this file. See [standa - feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) -### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) +### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27) - docs: the last two archived-host links point home (91ef1c8b) - feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) @@ -216,19 +187,19 @@ All notable changes to this project will be documented in this file. See [standa - ci: run the pipeline on the forge (999d0ebb) -### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) +### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03) - docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) - fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) -### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) +### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) - fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) -### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) +### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) - fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54d4f784..d277091d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ may find elsewhere in the repo's history. ## Where the project lives -The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. +The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**. It's anonymously readable and cloneable — no account needed to browse, clone, or build. @@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework. ## Development setup ```bash -git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git +git clone https://source.soulcraft.com/soulcraft/brainy.git cd brainy npm install npm run build @@ -57,17 +57,6 @@ see `package.json` for `test:integration`, `test:coverage`, and friends. description states a number, cite the benchmark that produced it (see [docs/performance-envelopes.md](docs/performance-envelopes.md) for the pattern). Don't state an estimate as if it were measured. -- **Measurements carry numbers, not provenance.** Public commit messages and - docs give the SHAPE a number was taken at and never where it was taken: no - hostnames, no store or deployment identities, no operational anecdotes about - someone's running system. "A 14,056-noun / 72,679-verb production-shaped - store, measured solo under an exclusive lock" tells a reader everything the - number depends on; the machine it ran on and whose data it was tell them - nothing except where somebody's infrastructure lives. -- **Documents that answer or reference a confidential specification never enter - this repository, even summarized.** The public docs describe THIS engine and - the published contract, and nothing else — a summary of a private document is - still that document's contents. ## License diff --git a/README.md b/README.md index 762c9ec3..47a9123a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

Package on The Source Repository - CI + CI Documentation MIT License TypeScript diff --git a/RELEASES.md b/RELEASES.md index e8833b80..b89beba0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,7 +1,7 @@ # @soulcraft/brainy — Release Notes for Consumers This file is the **quick reference for downstream sessions** tracking Brainy changes. -Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases +Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases **How to use:** Brainy is the underlying data engine for downstream applications. Read this when: - Upgrading `@soulcraft/brainy` in your application @@ -31,115 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v10.4.4 — 2026-08-28 - -**A correctness and observability release.** The headline is not speed: it is that a -restart now tells you the truth about itself, a store stops lying about how much it -holds, and the engine stops doing work nobody asked for. There is a performance -improvement and it is modest; it is stated exactly below rather than rounded up. - -### The dark restart — fixed at the root - -A service could stop cleanly, exit 0, having awaited `close()` on every store it held, -and its next boot would announce `Overwriting stale writer lock … appears dead` for -every one of them. Nothing had crashed. Two deployments hit this; the same defect also -made those boots pay a crash-recovery fold they did not owe. - -The cause was not the lock. `close()` released it correctly — when it got there. A -failure part-way through close skipped both the release AND the clean-shutdown marker, -and "the recorded pid is gone" reads identically for an orderly restart and a crash. - -- `close()` is now two parts and the second is unconditional: the flush-request watcher, - the **writer lock**, the VFS timers and the terminal `closed` flag are released whether - the durable steps succeeded or not. The original failure is narrated with what it costs - the next open, then rethrown. -- Releasing the lock writes a **clean-close record** naming the lock generation it gave - up. The next open reads that record instead of guessing: recorded → nothing to recover; - absent → it says so, and names the recovery it is about to run. This also ends two - long-standing false alarms — a recycled pid locking a store out of its own reopen, and - `Re-acquiring writer lock … this is a bug` after a perfectly clean close. -- The signal path stopped failing in a batch. One store's failing flush used to strand - every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the - generation store's close (the marker) is part of shutdown, the lock goes in a `finally`, - and the handler no longer calls `process.exit()` when the host application has its own - signal handler, a race that truncated the host's own shutdown mid-flight. - -### The count ledger stops lying, and `counts.json` is written atomically - -The all-tier scalars are the denominator a coverage check subtracts against. A ledger -derived under the old rule — one entity per id DIRECTORY — counted ghost and scar -containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for -the life of the store. Two copies of one archive could disagree, and a downstream index -heal reported remaining work that did not exist. - -- Such a ledger now derives itself honestly **in the background** after the open, counting - identity records, and persists the correction stamped. Nothing waits for it, because no - read is served from a denominator. -- A derivation that raced a write refuses to stamp its number: one retry on a quiet store, - then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under - a barrier. -- `counts.json` is written temp+rename. A truncating write left a window in which a - concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open - down the full-rescan path, so the cheapest file in the store was buying the most - expensive recovery. - -### An open and a repair narrate themselves — on a channel a log level cannot silence - -A store could open for three minutes and print nothing at all. The phase timings existed; -they were written to a channel that every production-looking environment clamps away. - -- Narration moved to an always-visible channel. An open now heartbeats the phase it is in, - names each phase as it ends with what it was paying for, and names the expensive STEP - inside a phase. `repairIndex()` does the same and its receipt carries a per-family - `durationMs` — a repair that ran for half an hour with no output could only be watched - through `top`. -- A brain nobody has written to now does nothing: a flush over a clean store is a no-op - and says nothing, the graph index's auto-flush asks before it acts, and the - cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a - directory every 500 ms per store forever, with a slow safety sweep behind it and a - narrated fall back to polling where a filesystem cannot be watched. -- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()` - does not wait for it, every other family serves, and that family's doors refuse **by - name, carrying the provider's own progress**, saying plainly that they open by - themselves and no action is needed. Health narration dedupes by content, so an unchanged - verdict is silent however a provider's generation counter moves. - -### For operators — one behaviour change - -**Four `where` operators that previously returned an empty page now raise -`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range -posting index cannot evaluate a substring, a pattern or an array length without reading -every row, and it now refuses by name instead of answering with an empty result that -looks like an answer. - -**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and -`excludes`. All 25 accepted operator tokens now agree between this engine and its -accelerated counterpart. - -### Performance — stated exactly - -Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under -an exclusive lock: - -- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is - one fix — generation discovery reads directory NAMES instead of recursively walking the - entire generation log (−9.2 s, and it scales with history rather than row count). The - VFS phase is **unchanged**. -- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving - off the critical path accounts for storage-init dropping 5,941 ms → 25 ms. -- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own - init is under 2 s of that phase. It is the log-authority adoption and/or the - pending-embed log recovery, both now instrumented so the next measurement names the - culprit outright. - -Continuing work, named so nobody has to rediscover it: that ~38 s term; making the -generation store's committed-range set lazy; the hydration path that substitutes -`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix -filter built with a `$startsWith` spelling no operator set accepts, so -`searchFiles({ path })` throws today. - ---- - ## v10.4.3 — 2026-08-27 (Open Brainy's first release) **`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for diff --git a/docs/api-contract.json b/docs/api-contract.json deleted file mode 100644 index aafd838a..00000000 --- a/docs/api-contract.json +++ /dev/null @@ -1,1544 +0,0 @@ -{ - "contractVersion": 1, - "engine": "@soulcraftlabs/brainy", - "compatibility": { - "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", - "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" - }, - "doors": [ - { - "name": "adaptiveHistoryBudgetBytes", - "kind": "method", - "arity": 1 - }, - { - "name": "add", - "kind": "method", - "arity": 1 - }, - { - "name": "addMany", - "kind": "method", - "arity": 1 - }, - { - "name": "adoptLogAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "adoptLogAuthorityInner", - "kind": "method", - "arity": 0 - }, - { - "name": "aggViewFromEntity", - "kind": "method", - "arity": 1 - }, - { - "name": "anyProviderMigrating", - "kind": "method", - "arity": 0 - }, - { - "name": "applyFusionScoring", - "kind": "method", - "arity": 2 - }, - { - "name": "applyGraphConstraints", - "kind": "method", - "arity": 2 - }, - { - "name": "armIdleFlushTimer", - "kind": "method", - "arity": 2 - }, - { - "name": "asOf", - "kind": "method", - "arity": 2 - }, - { - "name": "assertGenerationStoreReady", - "kind": "method", - "arity": 1 - }, - { - "name": "assertWritable", - "kind": "method", - "arity": 1 - }, - { - "name": "audit", - "kind": "method", - "arity": 0 - }, - { - "name": "auditGraph", - "kind": "method", - "arity": 0 - }, - { - "name": "autoAdoptLegacyVfsBlobsIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "autoAlpha", - "kind": "method", - "arity": 1 - }, - { - "name": "autoCompactHistory", - "kind": "method", - "arity": 0 - }, - { - "name": "awaitMigrationLock", - "kind": "method", - "arity": 1 - }, - { - "name": "awaitPendingEmbeds", - "kind": "method", - "arity": 0 - }, - { - "name": "backfillAggregateIfNeeded", - "kind": "method", - "arity": 1 - }, - { - "name": "batchGet", - "kind": "method", - "arity": 2 - }, - { - "name": "brainWideStrictRequiresSubtype", - "kind": "method", - "arity": 1 - }, - { - "name": "bridgeLegacyPendingEmbedSidecars", - "kind": "method", - "arity": 0 - }, - { - "name": "buildAtGenerationVectors", - "kind": "method", - "arity": 2 - }, - { - "name": "buildGraphView", - "kind": "method", - "arity": 4 - }, - { - "name": "buildMetadataFilter", - "kind": "method", - "arity": 1 - }, - { - "name": "buildMigrationUpdate", - "kind": "method", - "arity": 5 - }, - { - "name": "buildRelationMigrationUpdate", - "kind": "method", - "arity": 5 - }, - { - "name": "cacheVerbInt", - "kind": "method", - "arity": 2 - }, - { - "name": "canServeVectorAtGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "checkHealth", - "kind": "method", - "arity": 0 - }, - { - "name": "checkMigrations", - "kind": "method", - "arity": 0 - }, - { - "name": "clear", - "kind": "method", - "arity": 0 - }, - { - "name": "clearPendingEmbed", - "kind": "method", - "arity": 1 - }, - { - "name": "close", - "kind": "method", - "arity": 0 - }, - { - "name": "closeDurableSteps", - "kind": "method", - "arity": 0 - }, - { - "name": "cluster", - "kind": "method", - "arity": 1 - }, - { - "name": "collectProviderInvariants", - "kind": "method", - "arity": 0 - }, - { - "name": "compactHistory", - "kind": "method", - "arity": 1 - }, - { - "name": "consumeMetadataWatermarkVerdict", - "kind": "method", - "arity": 1 - }, - { - "name": "convertMetadataToEntity", - "kind": "method", - "arity": 2 - }, - { - "name": "convertNounToEntity", - "kind": "method", - "arity": 1 - }, - { - "name": "counts", - "kind": "accessor" - }, - { - "name": "createIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "createMigrationBackupIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "createPinnedDb", - "kind": "method", - "arity": 1 - }, - { - "name": "createResult", - "kind": "method", - "arity": 4 - }, - { - "name": "dbFinalizationRegistry", - "kind": "accessor" - }, - { - "name": "dbHost", - "kind": "accessor" - }, - { - "name": "defineAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "detectIdKind", - "kind": "method", - "arity": 3 - }, - { - "name": "diagnostics", - "kind": "method", - "arity": 0 - }, - { - "name": "diff", - "kind": "method", - "arity": 2 - }, - { - "name": "embed", - "kind": "method", - "arity": 1 - }, - { - "name": "embedBatch", - "kind": "method", - "arity": 2 - }, - { - "name": "emitCommitted", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceSubtypeOnAdd", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceSubtypeOnRelate", - "kind": "method", - "arity": 4 - }, - { - "name": "enforceTrackedFieldValues", - "kind": "method", - "arity": 2 - }, - { - "name": "enhanceNLPResult", - "kind": "method", - "arity": 2 - }, - { - "name": "enqueuePendingEmbed", - "kind": "method", - "arity": 1 - }, - { - "name": "ensureAggregationIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "ensureIndexesLoaded", - "kind": "method", - "arity": 0 - }, - { - "name": "ensureInitialized", - "kind": "method", - "arity": 1 - }, - { - "name": "entityForAggFromRawRecord", - "kind": "method", - "arity": 1 - }, - { - "name": "entityFromGenerationRecord", - "kind": "method", - "arity": 3 - }, - { - "name": "entityIntsToUuids", - "kind": "method", - "arity": 1 - }, - { - "name": "entityViewFromRawRecord", - "kind": "method", - "arity": 2 - }, - { - "name": "excludedVisibilityTiers", - "kind": "method", - "arity": 1 - }, - { - "name": "executeGraphSearch", - "kind": "method", - "arity": 2 - }, - { - "name": "executeProximitySearch", - "kind": "method", - "arity": 1 - }, - { - "name": "executeTextSearch", - "kind": "method", - "arity": 2 - }, - { - "name": "executeVectorSearch", - "kind": "method", - "arity": 3 - }, - { - "name": "explain", - "kind": "method", - "arity": 1 - }, - { - "name": "export", - "kind": "method", - "arity": 0 - }, - { - "name": "extract", - "kind": "method", - "arity": 2 - }, - { - "name": "extractConcepts", - "kind": "method", - "arity": 2 - }, - { - "name": "extractEntities", - "kind": "method", - "arity": 2 - }, - { - "name": "factSegmentPaths", - "kind": "method", - "arity": 1 - }, - { - "name": "fieldCountsAggregateName", - "kind": "method", - "arity": 1 - }, - { - "name": "fillSubtypes", - "kind": "method", - "arity": 1 - }, - { - "name": "filterIdsBelted", - "kind": "method", - "arity": 2 - }, - { - "name": "find", - "kind": "method", - "arity": 1 - }, - { - "name": "findAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "findDuplicates", - "kind": "method", - "arity": 1 - }, - { - "name": "findMatchingWords", - "kind": "method", - "arity": 3 - }, - { - "name": "flush", - "kind": "method", - "arity": 0 - }, - { - "name": "formatInfo", - "kind": "method", - "arity": 0 - }, - { - "name": "formatSubtypeError", - "kind": "method", - "arity": 1 - }, - { - "name": "generation", - "kind": "method", - "arity": 0 - }, - { - "name": "generationDigest", - "kind": "method", - "arity": 1 - }, - { - "name": "get", - "kind": "method", - "arity": 2 - }, - { - "name": "getActivePlugins", - "kind": "method", - "arity": 0 - }, - { - "name": "getAvailableFields", - "kind": "method", - "arity": 0 - }, - { - "name": "getBackgroundDeduplicator", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldsForType", - "kind": "method", - "arity": 1 - }, - { - "name": "getFieldStatistics", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldsWithCardinality", - "kind": "method", - "arity": 0 - }, - { - "name": "getFieldValues", - "kind": "method", - "arity": 1 - }, - { - "name": "getIndexStats", - "kind": "method", - "arity": 0 - }, - { - "name": "getIndexStatus", - "kind": "method", - "arity": 0 - }, - { - "name": "getMemoryStats", - "kind": "method", - "arity": 0 - }, - { - "name": "getNeighborUuids", - "kind": "method", - "arity": 2 - }, - { - "name": "getNounCount", - "kind": "method", - "arity": 0 - }, - { - "name": "getOptimalQueryPlan", - "kind": "method", - "arity": 1 - }, - { - "name": "getStats", - "kind": "method", - "arity": 1 - }, - { - "name": "getStorageType", - "kind": "method", - "arity": 0 - }, - { - "name": "getSubtypeRule", - "kind": "method", - "arity": 1 - }, - { - "name": "getTripleIntelligence", - "kind": "method", - "arity": 0 - }, - { - "name": "getTypedNeighbors", - "kind": "method", - "arity": 4 - }, - { - "name": "getVerbCount", - "kind": "method", - "arity": 0 - }, - { - "name": "graph", - "kind": "accessor" - }, - { - "name": "graphAccelerationProvider", - "kind": "method", - "arity": 0 - }, - { - "name": "graphCommunities", - "kind": "method", - "arity": 1 - }, - { - "name": "graphCommunitiesFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphCommunitiesNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphEntityInt", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExport", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExportFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphExportNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphPath", - "kind": "method", - "arity": 3 - }, - { - "name": "graphPathFallback", - "kind": "method", - "arity": 3 - }, - { - "name": "graphPathNative", - "kind": "method", - "arity": 4 - }, - { - "name": "graphRank", - "kind": "method", - "arity": 1 - }, - { - "name": "graphRankFallback", - "kind": "method", - "arity": 1 - }, - { - "name": "graphRankNative", - "kind": "method", - "arity": 2 - }, - { - "name": "graphSubgraph", - "kind": "method", - "arity": 2 - }, - { - "name": "graphSubgraphFallback", - "kind": "method", - "arity": 4 - }, - { - "name": "graphSubgraphFromQuery", - "kind": "method", - "arity": 5 - }, - { - "name": "graphSubgraphNative", - "kind": "method", - "arity": 5 - }, - { - "name": "groupByLabel", - "kind": "method", - "arity": 2 - }, - { - "name": "hasStorageMethod", - "kind": "method", - "arity": 1 - }, - { - "name": "hasVectorOrTextCriteria", - "kind": "method", - "arity": 1 - }, - { - "name": "health", - "kind": "method", - "arity": 0 - }, - { - "name": "highlight", - "kind": "method", - "arity": 1 - }, - { - "name": "highlightSemanticPhase", - "kind": "method", - "arity": 5 - }, - { - "name": "history", - "kind": "method", - "arity": 2 - }, - { - "name": "historyStats", - "kind": "method", - "arity": 0 - }, - { - "name": "hub", - "kind": "accessor" - }, - { - "name": "hydrateIdMapperForGraphRebuild", - "kind": "method", - "arity": 0 - }, - { - "name": "hydrateNativeSubgraph", - "kind": "method", - "arity": 2 - }, - { - "name": "import", - "kind": "method", - "arity": 2 - }, - { - "name": "importPluginPackage", - "kind": "method", - "arity": 1 - }, - { - "name": "incidentEdges", - "kind": "method", - "arity": 3 - }, - { - "name": "indexStats", - "kind": "method", - "arity": 0 - }, - { - "name": "init", - "kind": "method", - "arity": 1 - }, - { - "name": "insights", - "kind": "method", - "arity": 0 - }, - { - "name": "isEmbeddingReady", - "kind": "method", - "arity": 0 - }, - { - "name": "isInfrastructureWrite", - "kind": "method", - "arity": 1 - }, - { - "name": "isInitialized", - "kind": "accessor" - }, - { - "name": "isReadOnly", - "kind": "accessor" - }, - { - "name": "kickBackgroundFlush", - "kind": "method", - "arity": 1 - }, - { - "name": "kickEmbedWorker", - "kind": "method", - "arity": 0 - }, - { - "name": "legacyLayoutMigrationPhase", - "kind": "method", - "arity": 0 - }, - { - "name": "loadAnalyticsGraph", - "kind": "method", - "arity": 1 - }, - { - "name": "loadPlugins", - "kind": "method", - "arity": 0 - }, - { - "name": "logAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "maintenanceDebt", - "kind": "method", - "arity": 0 - }, - { - "name": "materializeAtGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "metadataIndexRetractionOp", - "kind": "method", - "arity": 3 - }, - { - "name": "migrate", - "kind": "method", - "arity": 1 - }, - { - "name": "migrateField", - "kind": "method", - "arity": 1 - }, - { - "name": "migrateInternal", - "kind": "method", - "arity": 2 - }, - { - "name": "migrateLegacyZeroNormVfsRootIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "migrationSnapshot", - "kind": "method", - "arity": 0 - }, - { - "name": "neededFamiliesMigrating", - "kind": "method", - "arity": 1 - }, - { - "name": "neighbors", - "kind": "method", - "arity": 2 - }, - { - "name": "newId", - "kind": "method", - "arity": 0 - }, - { - "name": "nlp", - "kind": "method", - "arity": 0 - }, - { - "name": "normalizeConfig", - "kind": "method", - "arity": 1 - }, - { - "name": "noteWriteForPersistence", - "kind": "method", - "arity": 0 - }, - { - "name": "now", - "kind": "method", - "arity": 0 - }, - { - "name": "onChange", - "kind": "method", - "arity": 1 - }, - { - "name": "pagination", - "kind": "accessor" - }, - { - "name": "parseMigrationPath", - "kind": "method", - "arity": 1 - }, - { - "name": "parseNaturalQuery", - "kind": "method", - "arity": 1 - }, - { - "name": "pathExists", - "kind": "method", - "arity": 2 - }, - { - "name": "pendingEmbedCount", - "kind": "method", - "arity": 0 - }, - { - "name": "performInit", - "kind": "method", - "arity": 1 - }, - { - "name": "persistPinnedGeneration", - "kind": "method", - "arity": 2 - }, - { - "name": "persistSingleOp", - "kind": "method", - "arity": 6 - }, - { - "name": "pickMetadataProbe", - "kind": "method", - "arity": 1 - }, - { - "name": "pickVectorProbe", - "kind": "method", - "arity": 0 - }, - { - "name": "pinGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "planGetEntity", - "kind": "method", - "arity": 3 - }, - { - "name": "planTransact", - "kind": "method", - "arity": 1 - }, - { - "name": "planTxAdd", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxRelate", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxRemove", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxUnrelate", - "kind": "method", - "arity": 3 - }, - { - "name": "planTxUpdate", - "kind": "method", - "arity": 3 - }, - { - "name": "projectionGauges", - "kind": "method", - "arity": 0 - }, - { - "name": "providerForFamily", - "kind": "method", - "arity": 1 - }, - { - "name": "providerIsMigrating", - "kind": "method", - "arity": 1 - }, - { - "name": "providerMigrationStatus", - "kind": "method", - "arity": 0 - }, - { - "name": "queryAggregate", - "kind": "method", - "arity": 2 - }, - { - "name": "queryIndexFamilies", - "kind": "method", - "arity": 1 - }, - { - "name": "readPath", - "kind": "method", - "arity": 2 - }, - { - "name": "ready", - "kind": "accessor" - }, - { - "name": "rebuildIndexesIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "rebuildMetadataIndexOnline", - "kind": "method", - "arity": 0 - }, - { - "name": "reconcileLogDivergence", - "kind": "method", - "arity": 2 - }, - { - "name": "reconstructPath", - "kind": "method", - "arity": 4 - }, - { - "name": "recordStateAt", - "kind": "method", - "arity": 3 - }, - { - "name": "recoverPendingEmbedsFromLog", - "kind": "method", - "arity": 0 - }, - { - "name": "registerShutdownHooks", - "kind": "method", - "arity": 0 - }, - { - "name": "relate", - "kind": "method", - "arity": 1 - }, - { - "name": "related", - "kind": "method", - "arity": 1 - }, - { - "name": "relateMany", - "kind": "method", - "arity": 1 - }, - { - "name": "relationFromGenerationRecord", - "kind": "method", - "arity": 2 - }, - { - "name": "relationshipSubtypesOf", - "kind": "method", - "arity": 1 - }, - { - "name": "releaseGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "remove", - "kind": "method", - "arity": 1 - }, - { - "name": "removeAggregate", - "kind": "method", - "arity": 1 - }, - { - "name": "removeMany", - "kind": "method", - "arity": 1 - }, - { - "name": "removeMigrationBackupSafe", - "kind": "method", - "arity": 0 - }, - { - "name": "repackHistory", - "kind": "method", - "arity": 1 - }, - { - "name": "repairIndex", - "kind": "method", - "arity": 1 - }, - { - "name": "requestFlush", - "kind": "method", - "arity": 1 - }, - { - "name": "requireProviders", - "kind": "method", - "arity": 1 - }, - { - "name": "requireSubtype", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveAsOfGeneration", - "kind": "method", - "arity": 2 - }, - { - "name": "resolveDiffEndpoint", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveHiddenIds", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveHNSWPersistMode", - "kind": "method", - "arity": 0 - }, - { - "name": "resolveRawGeneration", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveRetentionPolicy", - "kind": "method", - "arity": 0 - }, - { - "name": "resolveVerbEndpointInts", - "kind": "method", - "arity": 1 - }, - { - "name": "resolveVerbIntsToIds", - "kind": "method", - "arity": 1 - }, - { - "name": "restore", - "kind": "method", - "arity": 2 - }, - { - "name": "rrfFusion", - "kind": "method", - "arity": 4 - }, - { - "name": "runAggregationBackfillWalk", - "kind": "method", - "arity": 0 - }, - { - "name": "runAggregationCatchUp", - "kind": "method", - "arity": 0 - }, - { - "name": "runEmbedWorker", - "kind": "method", - "arity": 0 - }, - { - "name": "runOracle", - "kind": "method", - "arity": 1 - }, - { - "name": "runRepairIndexPhases", - "kind": "method", - "arity": 5 - }, - { - "name": "scanFacts", - "kind": "method", - "arity": 1 - }, - { - "name": "seedIdsToInts", - "kind": "method", - "arity": 1 - }, - { - "name": "selectorToSeedIds", - "kind": "method", - "arity": 1 - }, - { - "name": "setRetentionBudget", - "kind": "method", - "arity": 1 - }, - { - "name": "setupEmbedder", - "kind": "method", - "arity": 0 - }, - { - "name": "setupIndex", - "kind": "method", - "arity": 0 - }, - { - "name": "setupStorage", - "kind": "method", - "arity": 0 - }, - { - "name": "similar", - "kind": "method", - "arity": 1 - }, - { - "name": "similarity", - "kind": "method", - "arity": 2 - }, - { - "name": "splitForHighlighting", - "kind": "method", - "arity": 2 - }, - { - "name": "stampBrainFormat", - "kind": "method", - "arity": 0 - }, - { - "name": "stampBrainFormatIfNeeded", - "kind": "method", - "arity": 0 - }, - { - "name": "stampEntityTree", - "kind": "method", - "arity": 0 - }, - { - "name": "stampProjectionWatermarks", - "kind": "method", - "arity": 0 - }, - { - "name": "stats", - "kind": "method", - "arity": 0 - }, - { - "name": "storageAdapter", - "kind": "accessor" - }, - { - "name": "stream", - "kind": "method", - "arity": 0 - }, - { - "name": "streaming", - "kind": "accessor" - }, - { - "name": "subtypesOf", - "kind": "method", - "arity": 1 - }, - { - "name": "trackField", - "kind": "method", - "arity": 1 - }, - { - "name": "transact", - "kind": "method", - "arity": 2 - }, - { - "name": "transactionLog", - "kind": "method", - "arity": 1 - }, - { - "name": "unrelate", - "kind": "method", - "arity": 1 - }, - { - "name": "unvectorNounForRootMigration", - "kind": "method", - "arity": 1 - }, - { - "name": "update", - "kind": "method", - "arity": 1 - }, - { - "name": "updateMany", - "kind": "method", - "arity": 1 - }, - { - "name": "updateRelation", - "kind": "method", - "arity": 1 - }, - { - "name": "upsertMergeParams", - "kind": "method", - "arity": 2 - }, - { - "name": "use", - "kind": "method", - "arity": 1 - }, - { - "name": "usesDefaultWasmEmbedder", - "kind": "method", - "arity": 0 - }, - { - "name": "validateIndexConsistency", - "kind": "method", - "arity": 0 - }, - { - "name": "vectorSearchAtGeneration", - "kind": "method", - "arity": 4 - }, - { - "name": "verbsToRelations", - "kind": "method", - "arity": 1 - }, - { - "name": "verbToRelationLike", - "kind": "method", - "arity": 1 - }, - { - "name": "verifyEntityTreeStamp", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyGraphAdjacencyLive", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyLogAuthority", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyMetadataLive", - "kind": "method", - "arity": 0 - }, - { - "name": "verifyVectorLive", - "kind": "method", - "arity": 0 - }, - { - "name": "versionedIndexProviders", - "kind": "method", - "arity": 0 - }, - { - "name": "vfs", - "kind": "accessor" - }, - { - "name": "waitForIndexed", - "kind": "method", - "arity": 2 - }, - { - "name": "warm", - "kind": "method", - "arity": 0 - }, - { - "name": "warmupEmbeddings", - "kind": "method", - "arity": 0 - }, - { - "name": "warnIfReadsDegraded", - "kind": "method", - "arity": 1 - }, - { - "name": "wireConnectionsCodec", - "kind": "method", - "arity": 0 - }, - { - "name": "wireGraphIdResolver", - "kind": "method", - "arity": 0 - } - ], - "errors": [ - "BrainyError", - "DerivedArtifactMissingError", - "GraphIndexNotReadyError", - "MetadataIndexNotReadyError", - "MigrationInProgressError", - "ProtectedArtifactError", - "VectorIndexNotReadyError" - ], - "operators": { - "accepted": [ - "between", - "contains", - "endsWith", - "eq", - "equals", - "excludes", - "exists", - "greaterThan", - "greaterThanOrEqual", - "gt", - "gte", - "hasAll", - "in", - "length", - "lessThan", - "lessThanOrEqual", - "lt", - "lte", - "matches", - "missing", - "ne", - "noneOf", - "notEquals", - "oneOf", - "startsWith" - ], - "servedOnIndexPath": [ - "between", - "contains", - "eq", - "equals", - "excludes", - "exists", - "greaterThan", - "greaterThanOrEqual", - "gt", - "gte", - "hasAll", - "in", - "lessThan", - "lessThanOrEqual", - "lt", - "lte", - "missing", - "ne", - "noneOf", - "notEquals", - "oneOf" - ], - "refusedByIndexPath": [ - "endsWith", - "length", - "matches", - "startsWith" - ], - "combinators": [ - "allOf", - "anyOf", - "not" - ] - }, - "fieldAddressing": { - "systemKeyPrefix": "system.", - "systemEntityScalars": [ - "confidence", - "createdAt", - "createdBy", - "id", - "service", - "subtype", - "type", - "updatedAt", - "visibility", - "weight" - ], - "systemRelationScalars": [ - "confidence", - "createdAt", - "createdBy", - "service", - "sourceId", - "subtype", - "targetId", - "updatedAt", - "verb", - "visibility", - "weight" - ], - "plumbingFields": [ - "_rev", - "connections", - "data", - "level", - "vector" - ] - }, - "health": { - "verdicts": [ - "pass", - "warn", - "fail" - ], - "healKinds": [ - "none", - "repair", - "rebuild" - ], - "servingWithholdingInvariants": [ - "index-initialized", - "durable-state-present", - "manifest-residency", - "replay-clean", - "strand-latch" - ] - } -} diff --git a/package-lock.json b/package-lock.json index c4f66561..4d247780 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.3", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 06ce0253..bb6b5a47 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", - "brainyContract": 1, + "version": "10.4.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3–4× faster imports", - "items": [ - "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", - "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine — Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place — no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 582f4847..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs deleted file mode 100644 index be73d4ca..00000000 --- a/scripts/emit-contract-manifest.mjs +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env node -/** - * Emit this build's API-contract manifest to docs/api-contract.json. - * - * WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the - * code the first time somebody adds one. This reads the surface the build - * actually exposes — the prototype's own methods and accessors, the exported - * error classes, the `where` operator sets, the field-addressing vocabulary, - * the health verdicts — so a diff between two engines' manifests is a diff - * between two engines, never between two authors. - * - * Requirement marking (required / optional per door) is NOT derivable from the - * surface — it is a commitment, recorded with the contract's owner rather than - * here. This manifest carries the surface; the promise lives with the contract. - * - * Usage: node scripts/emit-contract-manifest.mjs [--check] - * --check exits non-zero when the committed manifest is stale. - */ - -import { writeFileSync, readFileSync, existsSync } from 'node:fs' -import { join, dirname } from 'node:path' -import { fileURLToPath } from 'node:url' - -const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..') -const OUT = join(ROOT, 'docs', 'api-contract.json') - -const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js')) -const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js')) -const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js')) -const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js')) - -/** Every own method and accessor on the class's prototype, minus the private ones. */ -function surfaceOf(ctor) { - const doors = [] - for (const name of Object.getOwnPropertyNames(ctor.prototype)) { - if (name === 'constructor' || name.startsWith('_')) continue - const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name) - if (!descriptor) continue - if (typeof descriptor.value === 'function') { - doors.push({ name, kind: 'method', arity: descriptor.value.length }) - } else if (descriptor.get) { - doors.push({ name, kind: 'accessor' }) - } - } - return doors.sort((a, b) => a.name.localeCompare(b.name)) -} - -const errors = Object.entries(errorsModule) - .filter(([name, value]) => typeof value === 'function' && /Error$/.test(name)) - .map(([name]) => name) - .sort() - -// The operator sets, read from the engine's own refusal message so the -// manifest can never disagree with the validator. -const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8') -const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set\(\[([\s\S]*?)\]\)/) -if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess') -const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() - -const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8') -const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) => - // Proven by the refusal path: these are the tokens with no case in the - // index's operator switch, so they fall to its default and are refused. - !new RegExp(`case '${op}':`).test(indexSource) -) -const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) - -const manifest = { - contractVersion: versionModule.contractVersion(), - engine: '@soulcraftlabs/brainy', - compatibility: { - minor: - 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', - major: - 'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused' - }, - doors: surfaceOf(Brainy), - errors, - operators: { - accepted, - servedOnIndexPath: servedOnIndex, - refusedByIndexPath: refusedByIndex, - combinators: ['allOf', 'anyOf', 'not'] - }, - fieldAddressing: { - systemKeyPrefix: 'system.', - systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(), - systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(), - plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort() - }, - health: { - verdicts: ['pass', 'warn', 'fail'], - healKinds: ['none', 'repair', 'rebuild'], - servingWithholdingInvariants: [ - 'index-initialized', - 'durable-state-present', - 'manifest-residency', - 'replay-clean', - 'strand-latch' - ] - } -} - -const rendered = `${JSON.stringify(manifest, null, 2)}\n` - -if (process.argv.includes('--check')) { - if (!existsSync(OUT)) { - console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`) - process.exit(1) - } - if (readFileSync(OUT, 'utf-8') !== rendered) { - console.error( - `docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` + - `the addition (minor = additive; a removal is a contract major).` - ) - process.exit(1) - } - console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`) - process.exit(0) -} - -writeFileSync(OUT, rendered) -console.log( - `Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` + - `${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` + - `${manifest.operators.accepted.length} operators ` + - `(${manifest.operators.refusedByIndexPath.length} refused by the index path).` -) diff --git a/scripts/release.sh b/scripts/release.sh index 5d434320..be1d6d6b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,7 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) ${COMMITS} " @@ -237,7 +237,7 @@ fi # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then - if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then echo -e "${GREEN}✅ Release page created on The Source${NC}\n" @@ -248,15 +248,20 @@ else echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): -# soulcraft.com/docs carries the paid product's documentation only. This -# engine's documentation home is THIS repository — README and docs/ — and the -# site serves 301s for the slugs this rail used to push. The push script stays -# in the tree for history; the rail no longer calls it. -echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" +# Step 12: Push public docs to the soulcraft.com docs ingest door +# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when +# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — +# that already happened) when a push errors, so the docs site never +# silently trails npm. +echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" +if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" +else + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" +fi echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}" +echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" diff --git a/src/brainy.ts b/src/brainy.ts index da04577e..966d3188 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -198,12 +198,7 @@ import { import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { - assessIndexReadiness, - assessProviderHealth, - assessProviderRebuild, - describeRebuildProgress -} from './utils/indexReadiness.js' +import { assessIndexReadiness, assessProviderHealth } from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { @@ -745,24 +740,6 @@ export class Brainy implements BrainyInterface { // Write acks NEVER await it; a failed background flush is LOUD and re-armed. private _persistDirtyWrites = 0 private _persistLastFlushAt = Date.now() - /** - * Whether a write has been committed since the last flush that ran. THE - * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written - * to has nothing to make durable, and a flush over it must cost nothing and - * say nothing. Before this, a flush called every provider, stamped the - * watermarks, persisted the generation counter and re-stamped the entity - * tree whether or not anything had changed — roughly 28 writes for a store - * that had not moved. - * - * WHAT THIS DOES NOT EXPLAIN, stated so nobody reads it as solved: a - * production process holding 21 brains printed "All indexes flushed to disk - * in 216-601ms" per brain every ~35s and idled at 1.26 cores with no writes - * for ten minutes. This engine's cadence is WRITE-DRIVEN — every trigger - * runs through noteWriteForPersistence, which only a committed write calls — - * so something was calling flush() on those brains, and this gate makes such - * a call free rather than accounting for it. The caller is still unidentified. - */ - private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null private _persistBackgroundFlight: Promise | null = null @@ -857,18 +834,7 @@ export class Brainy implements BrainyInterface { // Read-gate narration dedup: a degraded-but-serving or not-ready health // report narrates via prodLog.warn ONCE per (provider, report.generation) — // never once per read. Keyed on the provider instance itself. - /** - * The last health narration emitted per provider, keyed by its CONTENT. - * - * This used to dedupe on the provider's `generation` counter, which bumps on - * every ledger mutation and every rebuild boundary — so a provider that - * bumps its generation on routine work re-emitted the same unchanged health - * line on every read that consulted it, and a provider that never bumped - * could suppress a line whose reasons had genuinely changed. The dedupe key - * is now what the line SAYS: an unchanged verdict is silent however the - * generation moves, and a changed verdict is always heard. - */ - private _lastNarratedHealth = new Map() + private _lastNarratedHealthGeneration = new Map() constructor(config?: BrainyConfig) { // The reserved-field write policy died with the field-addressing law: @@ -1132,84 +1098,21 @@ export class Brainy implements BrainyInterface { configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging } - // OPEN-PATH NARRATION: phase timing across the five named stretches of - // init — storage init / generation-store open+fold / index init+gate / - // VFS bootstrap / embedding-warm-started. Each `markPhase()` call records - // elapsed ms SINCE THE PREVIOUS checkpoint, so the buckets always sum to - // the pre-integration/warmOnOpen total. - // - // THE LAW THIS ENFORCES: an open is never silent for more than - // OPEN_HEARTBEAT_MS. A production service opening a 16 GB store logged - // NOTHING for three minutes and then began work — the operator could not - // tell a slow open from a hung one, and restarted into the same wall. - // Two mechanisms, both on the always-visible narration channel (the old - // breakdown used `prodLog.warn`, which production clamps away — that is - // why the three minutes were silent): - // - a heartbeat that names the phase currently running and its elapsed - // wall, every OPEN_HEARTBEAT_MS, for as long as the open lasts; - // - one line per phase AS IT ENDS, naming its wall and its cause, for - // any phase over OPEN_PHASE_NARRATE_MS. - // The heartbeat is unref'd and cleared in the `finally` below, so it can - // neither hold the process open nor outlive a failed init. It cannot fire - // inside a phase that blocks the event loop synchronously; such a phase - // must narrate its own progress (the generation-log fold does). - const OPEN_HEARTBEAT_MS = 5_000 - const OPEN_PHASE_NARRATE_MS = 2_000 - /** Phase order + what each one is paying for, quoted in its narration. */ - const OPEN_PHASES: ReadonlyArray<{ name: string; cause: string }> = [ - { name: 'storage-init', cause: 'opening the store and loading its count ledger' }, - { - name: 'generation-store-open-fold', - cause: 'opening the generation store: crash-recovery replay/fold, derived-family registration, format handshake' - }, - { name: 'index-init-gate', cause: 'constructing the derived indexes and gating them for serving' }, - { name: 'vfs-bootstrap', cause: 'bootstrapping the virtual filesystem' }, - { name: 'embedding-warm-started', cause: 'starting the background embedding warm' } - ] + // OPEN-PATH NARRATION: lightweight phase timing across the five + // named stretches of init — storage init / generation-store open+fold / + // index init+gate / VFS bootstrap / embedding-warm-started. Each + // `markPhase()` call records elapsed ms SINCE THE PREVIOUS checkpoint, + // so the buckets always sum to the pre-integration/warmOnOpen total. + // Silent under 2s; one `prodLog.warn` line naming every phase's ms + // above it, so the operator's next restart storm names its own slow + // phase instead of re-deriving it from a stack of raw timestamps. const initStart = Date.now() let lastPhaseCheckpoint = initStart - let currentPhaseIndex = 0 const phaseTimingsMs: Record = {} - const openHeartbeat: ReturnType = setInterval(() => { - const phase = OPEN_PHASES[currentPhaseIndex] - if (!phase) return - prodLog.narrate( - `[Brainy] open: still in phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + - `"${phase.name}" after ${Math.round((Date.now() - lastPhaseCheckpoint) / 1000)}s ` + - `(${Math.round((Date.now() - initStart) / 1000)}s into the open) — ${phase.cause}` - ) - }, OPEN_HEARTBEAT_MS) - if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref() - /** - * Narrate one STEP inside a phase when it turns out to be expensive. - * A phase that costs a minute and names only itself tells an operator - * where to look but not what to look at; this names the step. Silent - * under OPEN_PHASE_NARRATE_MS, so a fast open says nothing extra. - */ - const step = async (name: string, cause: string, run: () => Promise): Promise => { - const startedAt = Date.now() - try { - return await run() - } finally { - const elapsed = Date.now() - startedAt - if (elapsed >= OPEN_PHASE_NARRATE_MS) { - prodLog.narrate(`[Brainy] open: step "${name}" took ${elapsed}ms — ${cause}`) - } - } - } const markPhase = (name: string): void => { const now = Date.now() - const elapsed = now - lastPhaseCheckpoint - phaseTimingsMs[name] = elapsed + phaseTimingsMs[name] = now - lastPhaseCheckpoint lastPhaseCheckpoint = now - const finished = OPEN_PHASES[currentPhaseIndex] - if (elapsed >= OPEN_PHASE_NARRATE_MS && finished && finished.name === name) { - prodLog.narrate( - `[Brainy] open: phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + - `"${name}" finished in ${elapsed}ms — ${finished.cause}` - ) - } - currentPhaseIndex++ } try { @@ -1297,12 +1200,9 @@ export class Brainy implements BrainyInterface { // instances skip recovery (readers never write; the next writer // repairs). this.generationStore = new GenerationStore(this.storage) - const generationOpenResult = await step( - 'generation-store.open', - 'reading the generation manifest and committed ranges, opening the fact log and the ' + - 'packed segment tier, and folding any crash-recovery replay', - () => this.generationStore.open({ readOnly: this.config.mode === 'reader' }) - ) + const generationOpenResult = await this.generationStore.open({ + readOnly: this.config.mode === 'reader' + }) // The generation fact log is CANONICAL state, not a derived index — no // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare @@ -1340,11 +1240,7 @@ export class Brainy implements BrainyInterface { // rollup invariants against the log head + live counters. Loud on // genuine incoherence (repairIndex heals), silent on absent/coherent, // benign-behind refreshes at the next flush. Never blocks open. - await step( - 'verify-entity-tree-stamp', - 'comparing the entity tree\'s stamped generation and rollups against the store', - () => this.verifyEntityTreeStamp() - ) + await this.verifyEntityTreeStamp() // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — @@ -1356,11 +1252,7 @@ export class Brainy implements BrainyInterface { // them from the canonical records and then re-stamps the marker AFTER the // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / // absent marker, so the next open idempotently re-rebuilds). - this._brainFormat = await step( - 'read-brain-format', - 'reading the on-disk format marker that decides whether the derived indexes are stale', - () => readBrainFormat(this.storage) - ) + this._brainFormat = await readBrainFormat(this.storage) this._indexEpochStale = this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH @@ -1371,11 +1263,7 @@ export class Brainy implements BrainyInterface { // upgrade verifies + stamps; retained on failure. No-op for a reader, for // non-filesystem storage, or for a brain with no persisted data. if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { - await step( - 'pre-upgrade-backup', - 'snapshotting the brain directory before a one-time format rebuild (migrationBackup)', - () => this.createMigrationBackupIfNeeded() - ) + await this.createMigrationBackupIfNeeded() } // PHASE 2 of 5 — "generation-store open+fold": GenerationStore @@ -1538,27 +1426,10 @@ export class Brainy implements BrainyInterface { `[Brainy] Rebuilding indexes after crash recovery rolled back ` + `${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)` ) - // SELF-REBUILD DEFERENCE, same law as the open gate: a provider that - // is already rebuilding itself from canonical is doing exactly this - // work. Kicking a second rebuild on top of it is redundant at best. - // Safe by ordering: the crash-recovery fold ran in the generation - // store's open, BEFORE any provider was constructed, so a provider - // rebuilding now is reading the repaired canonical records. - const kick = async (leg: string, provider: { rebuild: () => Promise }) => { - const rebuilding = assessProviderRebuild(provider) - if (rebuilding) { - prodLog.narrate( - `[Brainy] crash-recovery rebuild: the ${leg} provider is already ` + - `${describeRebuildProgress(rebuilding)} from canonical — not kicking a second one.` - ) - return - } - await provider.rebuild() - } await Promise.all([ - kick('metadata', this.metadataIndex), - kick('vector', this.index as unknown as { rebuild: () => Promise }), - kick('graph', this.graphIndex) + this.metadataIndex.rebuild(), + this.index.rebuild(), + this.graphIndex.rebuild() ]) } @@ -1655,11 +1526,7 @@ export class Brainy implements BrainyInterface { // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true // once this open-time step has run. - await step( - 'rebuild-indexes-if-needed', - 'the derived-index gate: each family\'s readiness verdict, and any build it asks for', - () => this.rebuildIndexesIfNeeded() - ) + await this.rebuildIndexesIfNeeded() this.lazyRebuildCompleted = true // Check for pending data migrations @@ -1732,11 +1599,7 @@ export class Brainy implements BrainyInterface { // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) - await step( - 'vfs.init', - 'creating or adopting the VFS root and wiring the path resolver', - () => this._vfs!.init() - ) + await this._vfs.init() this._vfsInitialized = true // Mark VFS as fully initialized // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the @@ -1760,11 +1623,7 @@ export class Brainy implements BrainyInterface { const storedArtifact = await this.storage .readRawObject(LOG_AUTHORITY_PATH) .catch(() => null) - const authority = await step( - 'read-log-authority', - 'reading the stored storage-authority artifact', - () => readLogAuthority(this.storage) - ) + const authority = await readLogAuthority(this.storage) this._logAuthority = authority if (authority.authority === 'log') { this.generationStore.setLogDurability('at-ack') @@ -1775,12 +1634,7 @@ export class Brainy implements BrainyInterface { this.generationStore.getFactLog() !== null ) { try { - await step( - 'adopt-log-authority', - 'the adoption oracle: verifying the log against canonical before flipping this ' + - 'brain to durable-at-ack, and backfilling any curable divergence', - () => this.adoptLogAuthority() - ) + await this.adoptLogAuthority() prodLog.info( '[Brainy] storage authority adopted at open: generation log ' + '(fleet default; oracle green; durable-at-ack enabled)' @@ -1820,16 +1674,8 @@ export class Brainy implements BrainyInterface { // this is where it lands. if (!this.isReadOnly) { try { - await step( - 'bridge-pending-embed-sidecars', - 'migrating any pre-log deferred-embed marker files into the generation log', - () => this.bridgeLegacyPendingEmbedSidecars() - ) - await step( - 'recover-pending-embeds', - 'folding the generation log\'s deferred-embed markers back into the pending set', - () => this.recoverPendingEmbedsFromLog() - ) + await this.bridgeLegacyPendingEmbedSidecars() + await this.recoverPendingEmbedsFromLog() if (this._pendingEmbedIds.size > 0) { prodLog.info( `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + @@ -1931,7 +1777,7 @@ export class Brainy implements BrainyInterface { const phaseList = Object.entries(phaseTimingsMs) .map(([name, ms]) => `${name}=${ms}ms`) .join(', ') - prodLog.narrate( + prodLog.warn( `[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` + `phase breakdown above to find which one to investigate first` ) @@ -1993,10 +1839,6 @@ export class Brainy implements BrainyInterface { // log — a plain string interpolation discards both stack and cause. const message = error instanceof Error ? error.message : String(error) throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error }) - } finally { - // The open is over — succeeded or failed. Stop the heartbeat here so a - // failed init never leaves a timer narrating a phase nobody is running. - clearInterval(openHeartbeat) } } @@ -2014,112 +1856,76 @@ export class Brainy implements BrainyInterface { * NOTE: Registers globally (once for all instances) to avoid MaxListenersExceededWarning */ private registerShutdownHooks(): void { - /** - * The signal-path shutdown. THREE LAWS, each written by a production - * shutdown that looked clean and wasn't: - * - * 1. PER-INSTANCE ISOLATION. This used to be one `try` around a loop over - * every open brain: the first instance whose flush rejected aborted the - * loop, so every remaining brain kept its writer lock and its unwritten - * markers — and the process still exited 0. A pool of brains failed in - * a batch, not one at a time. - * 2. THE MARKER IS PART OF SHUTDOWN. Flushing the indexes without closing - * the generation store leaves the clean-shutdown marker unwritten, so - * the NEXT open reads the store as crashed and folds the whole - * generation log — measured in tens of seconds on a real store, paid on - * every restart, after a shutdown the operator saw exit 0. - * 3. THE LOCK IS ALWAYS GIVEN UP. In a `finally`, per instance: a process - * on its way out holds nothing. - */ const flushOnShutdown = async () => { console.log('Shutdown signal received - flushing pending data...') - let flushedCount = 0 - let failedCount = 0 - // Snapshot: close() splices Brainy.instances while we iterate. - for (const instance of [...Brainy.instances]) { - if (!instance.initialized) continue - try { - // Flush all buffered data (parallel across components, this brain only). - await Promise.all([ - (async () => { - if (instance.storage && typeof instance.storage.flushCounts === 'function') { - await instance.storage.flushCounts() - } - })(), - (async () => { - if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { - await instance.metadataIndex.flush() - } - })(), - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { - await instance.graphIndex.flush() - } - })(), - (async () => { - if (instance.index && typeof instance.index.flush === 'function') { - await instance.index.flush() - } - })() - ]) - - // Close the generation store: persists the counter, advances the - // fold checkpoint, and stamps the clean-shutdown marker LAST — the - // one step that decides whether the next open adopts or folds. Law 2. - if (instance.generationStore && !instance.isReadOnly) { - await instance.generationStore.close() - } - - // Close components to stop timers that would prevent clean process exit - await Promise.all([ - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { - await instance.graphIndex.close() - } - })(), - (async () => { - const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && typeof index.close === 'function') { - await index.close() - } - })(), - (async () => { - const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && typeof metadataIndex.close === 'function') { - await metadataIndex.close() - } - })() - ]) - flushedCount++ - } catch (error) { - failedCount++ - console.error('Failed to flush one Brainy instance on shutdown:', error) - } finally { - // Law 3 — the lock and the watcher go regardless. - try { - if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { - instance.storage.stopFlushRequestWatcher() - } - } catch (error) { - console.error('Failed to stop the flush-request watcher on shutdown:', error) - } - try { - if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { - await instance.storage.releaseWriterLock() - } - } catch (error) { - console.error('Failed to release the writer lock on shutdown:', error) + try { + let flushedCount = 0 + for (const instance of Brainy.instances) { + if (instance.initialized) { + // Flush all buffered data, then close to release resources (timers, handles) + await Promise.all([ + (async () => { + if (instance.storage && typeof instance.storage.flushCounts === 'function') { + await instance.storage.flushCounts() + } + })(), + (async () => { + if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { + await instance.metadataIndex.flush() + } + })(), + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { + await instance.graphIndex.flush() + } + })(), + (async () => { + if (instance.index && typeof instance.index.flush === 'function') { + await instance.index.flush() + } + })() + ]) + // Close components to stop timers that would prevent clean process exit + await Promise.all([ + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { + await instance.graphIndex.close() + } + })(), + (async () => { + const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks + if (index && typeof index.close === 'function') { + await index.close() + } + })(), + (async () => { + const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks + if (metadataIndex && typeof metadataIndex.close === 'function') { + await metadataIndex.close() + } + })(), + // Release the writer lock so a successor process can take over. + // No-op for readers and for backends without locking. + (async () => { + if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { + await instance.storage.releaseWriterLock() + } + })(), + // Stop the flush-request watcher to release its interval timer. + (async () => { + if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { + instance.storage.stopFlushRequestWatcher() + } + })(), + ]) + flushedCount++ } } - } - if (flushedCount > 0) { - console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) - } - if (failedCount > 0) { - console.error( - `${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` + - `their writer locks were released, but their next open will run crash recovery.` - ) + if (flushedCount > 0) { + console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) + } + } catch (error) { + console.error('Failed to flush on shutdown:', error) } } @@ -2127,32 +1933,13 @@ export class Brainy implements BrainyInterface { // kept as statics so the last live instance's close() can deregister them // — the signal handles they hold are ref'd and would otherwise keep the // process alive forever after every brain is closed. - /** - * Exit the process ONLY when Brainy is the sole handler for this signal. - * - * Registering a signal listener suppresses Node's default terminate - * behaviour, so a library that attaches one must either exit or be sure - * someone else will. Brainy attaching one AND exiting was the wrong half - * of that choice for every host application with its own graceful - * shutdown: both handlers run concurrently, and whichever finishes first - * wins — a library flush finishing before an application's close() - * terminated that close mid-flight, at exit code 0, with locks and - * markers unwritten. When the host has its own handler (listener count - * above our own), the host owns the exit; Brainy only makes its data - * durable and steps aside. - */ - const exitIfSoleShutdownOwner = (signal: 'SIGTERM' | 'SIGINT'): void => { - if (process.listenerCount(signal) <= 1) { - process.exit(0) - } - } Brainy.sigtermListener = async () => { await flushOnShutdown() - exitIfSoleShutdownOwner('SIGTERM') + process.exit(0) } Brainy.sigintListener = async () => { await flushOnShutdown() - exitIfSoleShutdownOwner('SIGINT') + process.exit(0) } Brainy.beforeExitListener = async () => { // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- @@ -2776,12 +2563,6 @@ export class Brainy implements BrainyInterface { * engine's own cadence (callers never call flush() in hot paths). */ private noteWriteForPersistence(): void { - // THE DIRTY WITNESS. Set on every committed write — both commit paths - // (single-op and transaction) end here, and the deferred-embed worker - // lands its vectors through the single-op path — BEFORE the policy check, - // so a `'manual'` consumer's explicit flush() is never skipped either. - // Cleared by a flush that actually runs; see flush(). - this._dirtySinceLastFlush = true const cfg = this.config.persistence if (this.isReadOnly || cfg?.policy === 'manual') return this._persistDirtyWrites++ @@ -4636,19 +4417,6 @@ export class Brainy implements BrainyInterface { this._graphAdjacencyVerified = true return 'live' } - // A provider that is REBUILDING ITSELF gets a refusal that says so, - // with its own progress: open deliberately did not wait for it (see - // rebuildIndexesIfNeeded), so this door is temporarily closed and will - // open on its own. Anything else is a broken index needing a repair. - const rebuilding = assessProviderRebuild(this.graphIndex) - if (rebuilding) { - throw new GraphIndexNotReadyError( - `Graph adjacency index is ${describeRebuildProgress(rebuilding)} and is not serving ` + - `yet. find({ connected }), neighbors() and related() refuse rather than serve an ` + - `empty result. The brain is open and every other family is serving; this door opens ` + - `by itself when the provider reports serving — no action is needed.` - ) - } throw new GraphIndexNotReadyError( `Graph adjacency index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` + @@ -4752,15 +4520,6 @@ export class Brainy implements BrainyInterface { this._metadataVerified = true return 'live' } - const rebuilding = assessProviderRebuild(this.metadataIndex) - if (rebuilding) { - throw new MetadataIndexNotReadyError( - `Metadata field index is ${describeRebuildProgress(rebuilding)} and is not serving ` + - `yet. find({ where }) and other filtered reads refuse rather than serve an empty ` + - `result. The brain is open and every other family is serving; this door opens by ` + - `itself when the provider reports serving — no action is needed.` - ) - } throw new MetadataIndexNotReadyError( `Metadata field index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` + @@ -4890,15 +4649,6 @@ export class Brainy implements BrainyInterface { this._vectorVerified = true return 'live' } - const rebuilding = assessProviderRebuild(this.index) - if (rebuilding) { - throw new VectorIndexNotReadyError( - `Vector index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + - `Semantic find({ query }) and proximity search refuse rather than serve an empty ` + - `result. The brain is open and every other family is serving; this door opens by ` + - `itself when the provider reports serving — no action is needed.` - ) - } throw new VectorIndexNotReadyError( `Vector index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` + @@ -8557,11 +8307,6 @@ export class Brainy implements BrainyInterface { */ async clear(): Promise { await this.ensureInitialized() - // A clear mutates durable state without going through a commit path, so - // it must set the dirty witness itself — otherwise a `clear()` followed by - // `flush()` would find the brain "clean" and skip the entity-tree stamp, - // leaving a stamp that describes the population this call just removed. - this._dirtySinceLastFlush = true // Clear storage await this.storage.clear() @@ -12396,27 +12141,6 @@ export class Brainy implements BrainyInterface { return } - // A CLEAN BRAIN FLUSHES NOTHING, AND SAYS NOTHING. No write has been - // committed since the last flush, so every step below would re-persist - // state identical to what is already on disk — provider flushes, the - // watermark stamps, the generation counter, the entity-tree stamp — and - // print two lines announcing it. The witness is set by every committed - // write (see noteWriteForPersistence) and cleared here; a write landing - // DURING this flush sets it again, so it is never lost — the next flush - // does that write's work. This makes an unexplained flush FREE; it does - // not explain one (see _dirtySinceLastFlush). - if (!this._dirtySinceLastFlush) { - return - } - this._dirtySinceLastFlush = false - // An explicit flush IS a flush: tell the cadence so, or the very next - // write sees "30s since the last flush" (the cadence only counted its - // own) and kicks a background flush that has nothing left to do, and the - // idle timer fires two seconds later over writes this flush already - // persisted. - this._persistLastFlushAt = Date.now() - this._persistDirtyWrites = 0 - console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() @@ -12497,18 +12221,6 @@ export class Brainy implements BrainyInterface { * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. - * - * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is - * the ALLOCATED counter — a number a write in flight has claimed and may - * never commit. Stamping it made the stamp's generation label a claim about - * counts it was not taken at, and every crash inside a write window then - * produced a spurious verdict at the next open: either `sourceGeneration N - * is ahead of the log head N-1` (the allocated generation died with the - * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the - * recovery fold folded facts the stamp's counts predate). MEASURED on the - * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on - * a coherent store raised one of those two verdicts, each of them naming - * `repairIndex()` — a whole-store recount — as the cure for nothing. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return @@ -12519,7 +12231,7 @@ export class Brainy implements BrainyInterface { ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', - sourceGeneration: this.generationStore.committedGeneration(), + sourceGeneration: this.generationStore.generation(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { @@ -12532,24 +12244,16 @@ export class Brainy implements BrainyInterface { /** * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the store's COMMITTED generation and - * the stamped rollup invariants against the live counters. Verdicts: + * compare `sourceGeneration` against the log head and the stamped rollup + * invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) → silent. * - `behind` → benign for the tree (it is written BY the commit; only the * stamp is stale — a crash landed between commit and flush). Refreshed at * the next flush. - * - `torn` → a TORN GENERATION-LOG TAIL, handled by - * {@link demoteTornEntityTreeStamp}: terminal, never a wait. * - `incoherent` → LOUD: the tree or its counters diverged from what was * stamped — `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. - * - * THE COMPARISON IS AGAINST `committedGeneration()`, matching what - * {@link stampEntityTree} writes and what every other open-time watermark in - * this class already reasons about (the fact-scan capability, the metadata / - * graph / HNSW watermark verdicts). Comparing against the allocated counter - * was the one place that disagreed, and disagreeing was the whole defect. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null @@ -12566,16 +12270,11 @@ export class Brainy implements BrainyInterface { this.storage.getNounCount(), this.storage.getVerbCount() ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), { + const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { nounCount, verbCount }) - if (verdict.state === 'torn') { - await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, { - nounCount, - verbCount - }) - } else if (verdict.state === 'incoherent') { + if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state — run ` + @@ -12589,92 +12288,6 @@ export class Brainy implements BrainyInterface { } } - /** - * @description THE TERMINAL VERDICT for a torn generation-log tail. - * - * A stamp whose `sourceGeneration` sits ABOVE the store's committed - * watermark witnesses a generation that is not in the log: the stamp's fsync - * outlived the tail's. By the time this runs, log-authority recovery has - * already folded every intact fact above the manifest and advanced the - * watermark to cover them — so if the stamp is STILL ahead, the generation - * it names is not merely late, it is GONE. There is nothing to wait for. - * - * That is the whole point of this method. A field report of this class - * (single-process store, abrupt termination mid-fold) described a reopen - * that narrated the tear and then held 100% CPU with zero log growth for - * eight minutes before an operator wiped the directory. A recovery that - * cannot say what it is waiting for has no business spinning; the honest - * answer here is a verdict, taken now, at O(1) cost. - * - * WHAT THE VERDICT DOES — the stamped surface is UNUSABLE, so it is - * discarded rather than believed: the stamped counts describe a generation - * that never became durable, and comparing them against live counters can - * only produce noise. The tree itself is not in question (it IS canonical — - * every commit writes it, and the fold re-applied every after-image the log - * still holds), so the demotion is a re-derivation of this family's verified - * surface at the generation the store can actually show: - * - * - WRITER open → re-stamp at `committedGeneration()` from the live - * counters — exactly what the next flush would write, taken now so the - * tear cannot re-narrate on every subsequent open. Both count sets are - * logged so an operator can see whether anything really moved. - * - READER open → a reader cannot re-stamp. Narrate the same terminal - * verdict with the named cure and carry on serving; a read-only inspector - * is never locked out of a store, and never left waiting either. - * - * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any - * external progress signal — the two counter reads and one stamp write are - * the entire cost, and none of them scales with the store. - */ - private async demoteTornEntityTreeStamp( - stamp: FamilyStamp, - stampSource: number, - head: number, - observed: { nounCount: number; verbCount: number } - ): Promise { - const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {} - const detail = - `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` + - `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` + - `generation is ${head} after crash recovery — the stamp's fsync outlived the log tail's, ` + - `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` + - `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.` - - if (this.isReadOnly) { - prodLog.warn( - `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` + - `family stays UNVERIFIED for this session (reads are unaffected — the canonical tree ` + - `is the truth this stamp only describes). Cure: open the store with a writer, or run ` + - `brain.repairIndex() there, to recount from canonical and re-stamp.` - ) - return - } - - const startedAt = Date.now() - try { - await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { - family: 'entity-tree', - sourceGeneration: head, - members: { - mode: 'rollup', - invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount } - } - }) - prodLog.warn( - `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` + - `from the live counters in ${Date.now() - startedAt}ms — terminal, not a wait. If the ` + - `observed counts above look wrong for your data, run brain.repairIndex() to recount ` + - `from canonical.` - ) - } catch (error) { - prodLog.warn( - `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) — the tear will ` + - `narrate again at the next open, which is the honest outcome; the store still serves ` + - `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.` - ) - } - } - /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. @@ -16877,19 +16490,8 @@ export class Brainy implements BrainyInterface { if (legacyEntityPaths.length === 0) { // Already flat (root entities, no head-branch entities) → stamp the marker // so future opens short-circuit. A genuinely empty/fresh dir gets no marker. - // "Are there any entities?" is answered by ONE directory read, not by a - // recursive listing of every file in the tree: this runs on the open path - // of every store that does not yet carry the marker (a restore, a store - // built by an older release), and on a large store that listing walks the - // whole canonical tree to learn a boolean. - const oneLevel = ( - probe as unknown as { listRawPrefixes?: (prefix: string) => Promise } - ).listRawPrefixes - const hasRootEntities = - typeof oneLevel === 'function' - ? (await oneLevel.call(probe, 'entities')).length > 0 - : (await probe.listRawObjects('entities')).length > 0 - if (hasRootEntities) { + const rootEntities = await probe.listRawObjects('entities') + if (rootEntities.length > 0) { await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, @@ -17388,34 +16990,16 @@ export class Brainy implements BrainyInterface { if (assessment.reasons.length > 0 && assessment.report != null) { const generation = assessment.report.generation - // Dedupe by CONTENT, not by the provider's generation counter — see - // _lastNarratedHealth. The generation is still REPORTED (an operator - // wants to know which generation produced the verdict); it just no - // longer decides whether the line is worth saying. - const line = - `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + - assessment.reasons.join('; ') - const key = `${assessment.report.provider}\u0000${assessment.reasons.join('; ')}` - if (this._lastNarratedHealth.get(provider) !== key) { - this._lastNarratedHealth.set(provider, key) - prodLog.warn(line) + if (this._lastNarratedHealthGeneration.get(provider) !== generation) { + this._lastNarratedHealthGeneration.set(provider, generation) + prodLog.warn( + `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + + assessment.reasons.join('; ') + ) } } if (assessment.readiness === 'not-ready') { - // A provider REBUILDING ITSELF gets a refusal that says so, with its - // own progress: open deliberately did not wait for it, this door is - // temporarily closed, and it opens by itself. Distinct from a broken - // index, which needs an operator. - const rebuilding = assessProviderRebuild(provider) - if (rebuilding) { - throw new ErrorClass( - `${name} index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + - `Reads of this family refuse rather than serve an empty result. The brain is open ` + - `and every other family is serving; this door opens by itself when the provider ` + - `reports serving — no action is needed.` - ) - } throw new ErrorClass( `${name} index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` + @@ -17738,37 +17322,9 @@ export class Brainy implements BrainyInterface { // by awaitMigrationLock meanwhile (nothing serves from a half-built index). // Gated per-index, so a non-migrating sibling still rebuilds when it needs // to; a migrating provider is skipped even under epoch-drift or size()===0. - // SELF-REBUILD DEFERENCE (the sibling of the migration lock, and the - // reason a production open took 641 seconds): a provider that reports - // `rebuildInProgress()` is ALREADY rebuilding its own index. Brainy must - // neither start a second rebuild nor WAIT for the provider's — init() - // returns, every other family serves, and that family's own doors refuse - // by name (carrying this progress) until the provider reports serving. - // A provider without the hook behaves exactly as before. - const metadataRebuilding = assessProviderRebuild(this.metadataIndex) - const vectorRebuilding = assessProviderRebuild(this.index) - const graphRebuilding = assessProviderRebuild(this.graphIndex) - for (const [leg, progress] of [ - ['metadata', metadataRebuilding], - ['vector', vectorRebuilding], - ['graph', graphRebuilding] - ] as const) { - if (progress) { - prodLog.narrate( - `[Brainy] open(): the ${leg} provider is ${describeRebuildProgress(progress)} — ` + - `open does NOT wait for it. The brain opens now, every other family serves, and ` + - `${leg} reads refuse by name until the provider reports itself serving.` - ) - } - } - - const metadataMigrating = - this.providerIsMigrating(this.metadataIndex) || metadataRebuilding !== null - const vectorMigrating = this.providerIsMigrating(this.index) || vectorRebuilding !== null - const graphMigrating = this.providerIsMigrating(this.graphIndex) || graphRebuilding !== null - // The epoch stamp certifies EVERY derived index, so it must not advance - // while any family is still being built — by a migration lock or by the - // provider itself. + const metadataMigrating = this.providerIsMigrating(this.metadataIndex) + const vectorMigrating = this.providerIsMigrating(this.index) + const graphMigrating = this.providerIsMigrating(this.graphIndex) const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its @@ -17980,15 +17536,6 @@ export class Brainy implements BrainyInterface { // when the metadata provider holds the migration lock: a 0 count there // reflects its in-place rebuild in progress, not a missed rebuild, so // forcing a second rebuild would collide with the provider's own. - // THREE states, not two. `metadataMigrating` above is true for a - // provider holding the migration lock AND for one that reports it is - // rebuilding itself — a provider whose rebuild() returns once the - // rebuild is OWNED AND RUNNING (online, its doors refusing by name) - // legitimately reports 0 entries here, and calling that CRITICAL would - // print a false alarm and kick a redundant second rebuild on every - // first contact. The check's real class — a rebuild that ran to - // completion and produced nothing — is untouched: a provider reporting - // 0 entries with NO rebuild in progress still trips it. if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + @@ -18547,93 +18094,17 @@ export class Brainy implements BrainyInterface { * invariant-driven pass (it was already rebuilt unconditionally — a second, * report-driven pass over the same family would be redundant at best). * - * NARRATION IS PART OF THE CONTRACT. A repair on a production store ran for - * more than thirty minutes at a full core with NOT ONE log line between its - * start and its end while the doors kept serving; the operator could tell it - * was alive only from `top`. Every phase now announces itself before it - * works, a heartbeat names the phase still running every five seconds, and - * each phase reports its own wall — carried in the receipt as - * `durationMs` per family, so nobody has to infer progress from CPU. - * * @param options.rebuild - Family name(s) to unconditionally rebuild, or `'all'` for all three (`'metadata' | 'graph' | 'vector'`). - * @returns The full per-family receipt (see {@link RepairReport}); also narrated as it goes. + * @returns The full per-family receipt (see {@link RepairReport}); also narrated via `prodLog.warn`. */ async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() - // A repair recounts, prunes and rebuilds outside the commit paths; the - // dirty witness is set so a caller's flush after a repair does its normal - // work rather than finding the brain "clean". - this._dirtySinceLastFlush = true const startedAt = Date.now() const families: RepairFamilyReport[] = [] - - // THE REPAIR HEARTBEAT — the same law the open obeys: no stretch of work - // may be silent for more than REPAIR_HEARTBEAT_MS. Unref'd (it never holds - // a process open) and cleared in the `finally` below. - const REPAIR_HEARTBEAT_MS = 5_000 - let currentPhase = 'starting' - let currentPhaseCause = 'preparing the repair' - let phaseStartedAt = Date.now() - const heartbeat = setInterval(() => { - prodLog.narrate( - `[Brainy] repairIndex: still in "${currentPhase}" after ` + - `${Math.round((Date.now() - phaseStartedAt) / 1000)}s ` + - `(${Math.round((Date.now() - startedAt) / 1000)}s into the repair) — ${currentPhaseCause}` - ) - }, REPAIR_HEARTBEAT_MS) - if (typeof heartbeat.unref === 'function') heartbeat.unref() - - /** Announce a phase before it does any work, and start its clock. */ - const beginPhase = (name: string, cause: string): void => { - currentPhase = name - currentPhaseCause = cause - phaseStartedAt = Date.now() - prodLog.narrate(`[Brainy] repairIndex: "${name}" started — ${cause}`) + const record = (family: string, entry: Omit): void => { + families.push({ family, ...entry }) } - /** - * Close the current phase: stamp its wall into the receipt row and say - * what it did. Every family row carries its own `durationMs`. - */ - const record = (family: string, entry: Omit): void => { - const durationMs = Date.now() - phaseStartedAt - families.push({ family, ...entry, durationMs }) - prodLog.narrate( - `[Brainy] repairIndex: "${family}" finished in ${durationMs}ms — ` + - (entry.checked - ? `${entry.healed} heal(s)${entry.rebuilt ? ', rebuilt' : ''}` + - (entry.detail ? ` (${entry.detail})` : '') - : `skipped (${entry.skipped ?? entry.reason ?? 'no reason given'})`) - ) - phaseStartedAt = Date.now() - } - - try { - return await this.runRepairIndexPhases(options, families, record, beginPhase, startedAt) - } finally { - clearInterval(heartbeat) - } - } - - /** - * @description The phases of {@link repairIndex}, separated so its heartbeat - * can live in a `finally` around them. Not a public door — see `repairIndex` - * for the contract. - * @param options - As `repairIndex`. - * @param families - The receipt rows being accumulated. - * @param record - Closes a phase: stamps its wall and narrates its outcome. - * @param beginPhase - Announces a phase before it works. - * @param startedAt - When the repair began, for the closing line. - * @returns The full receipt. - */ - private async runRepairIndexPhases( - options: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' } | undefined, - families: RepairFamilyReport[], - record: (family: string, entry: Omit) => void, - beginPhase: (name: string, cause: string) => void, - startedAt: number - ): Promise { - // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete // defect: a delete that removed the metadata (content) leg but left the // vector leg + the entity directory (a "ghost"), or left an empty directory @@ -18647,10 +18118,6 @@ export class Brainy implements BrainyInterface { rebuildSubtypeCounts?: () => Promise } if (typeof pruner.pruneOrphanedEntities === 'function') { - beginPhase( - 'orphaned-containers', - 'walking every canonical id directory for ghost/scar containers left by a partial delete' - ) const orphans = await pruner.pruneOrphanedEntities() const pruned = orphans.nouns.length + orphans.verbs.length record('orphaned-containers', { @@ -18661,7 +18128,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (pruned > 0) { - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + `partial delete.` @@ -18677,10 +18144,6 @@ export class Brainy implements BrainyInterface { // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup // (scalar totals + per-type maps + type-statistics arrays) from one // canonical walk and persists them. - beginPhase( - 'count-rollups', - 'ONE canonical walk recomputing every counter rollup — scalar totals, per-type maps, type statistics' - ) await pruner.rebuildTypeCounts?.() await pruner.rebuildSubtypeCounts?.() record('count-rollups', { @@ -18703,10 +18166,6 @@ export class Brainy implements BrainyInterface { // concurrent writers. Canonical metadata.path is the truth; only VFS // containment edges are touched. Loud per repair. if (this._vfsInitialized && this._vfs) { - beginPhase( - 'vfs-containment', - 'reconciling VFS containment edges against canonical metadata.path' - ) const containment = await this._vfs.repairContainment() record('vfs-containment', { checked: true, @@ -18716,7 +18175,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (containment.removed + containment.restored > 0) { - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` ) @@ -18727,25 +18186,17 @@ export class Brainy implements BrainyInterface { record('vfs-containment', { checked: false, healed: 0, skipped: 'VFS not initialized' }) } - beginPhase( - 'metadata-corruption', - 'detect-and-repair pass over the metadata index' - ) await this.metadataIndex.detectAndRepairCorruption() record('metadata-corruption', { checked: true, healed: 0, detail: 'detect-and-repair pass ran (see its own narration for repairs)' }) // Lift a failed-rollback write-quarantine: force a full rebuild so the // derived indexes are provably reconciled with canonical, then clear the // flag so writes resume. if (this.storeInconsistency) { - beginPhase( - 'write-quarantine', - 'full derived-index rebuild to lift the quarantine set by a failed transaction rollback' - ) await this.rebuildIndexesIfNeeded(true) const cleared = this.storeInconsistency record('write-quarantine', { checked: true, healed: 1, detail: `lifted (${cleared.records.length} record(s) reconciled)` }) this.storeInconsistency = null - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` + `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` + `Writes are re-enabled.` @@ -18776,9 +18227,9 @@ export class Brainy implements BrainyInterface { record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no rebuild() contract' }) continue } - beginPhase( - `provider:${familyName}`, - `explicit rebuild requested — rebuilding '${familyName}' unconditionally, no invariant consulted` + prodLog.warn( + `[Brainy] repairIndex(): explicit rebuild requested for '${familyName}' — ` + + `rebuilding unconditionally (no invariant consulted).` ) // The metadata family routes through the online build-beside // orchestrator (B3 D3) instead of the provider's own rebuild() — @@ -18795,7 +18246,7 @@ export class Brainy implements BrainyInterface { rebuilt: true, reason: 'explicit rebuild requested' }) - prodLog.narrate(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) + prodLog.warn(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) continue } @@ -18804,16 +18255,11 @@ export class Brainy implements BrainyInterface { rebuild?: () => Promise } | null if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') { - beginPhase(`provider:${familyName}`, 'checking the provider contract') record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract' }) continue } - beginPhase( - `provider:${familyName}`, - `reading the '${familyName}' provider's own invariant report, then healing only what it asks for` - ) let report: ProviderInvariantReport try { report = await p.validateInvariants() @@ -18830,7 +18276,7 @@ export class Brainy implements BrainyInterface { checked: true, healed: 1, detail: `rebuilt from canonical (failing: ${report.invariants.filter((i) => !i.holds).map((i) => i.name).join(', ')})` }) - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + `requiring a rebuild — reconciling its derived state from canonical.` ) @@ -18854,7 +18300,7 @@ export class Brainy implements BrainyInterface { const failingRepairs = report.invariants .filter((i) => !i.holds && i.heal === 'repair') .map((i) => i.name) - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex(): provider '${report.provider}' asks for an incremental ` + `repair (${failingRepairs.join(', ')}) — running its own repair().` ) @@ -18893,7 +18339,6 @@ export class Brainy implements BrainyInterface { // rebuild failure are now reconciled — clear the queryable degraded state // and re-arm the read-path warning. if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { - beginPhase('degraded-read-state', 'clearing degraded ids and re-arming the read-path warning') this._indexDegradedIds.clear() this._indexRebuildFailed = null this._degradedReadWarned = false @@ -18902,13 +18347,11 @@ export class Brainy implements BrainyInterface { const healedTotal = families.reduce((n, f) => n + f.healed, 0) const report: RepairReport = { families, healedTotal, durationMs: Date.now() - startedAt } - prodLog.narrate( + prodLog.warn( `[Brainy] repairIndex complete in ${report.durationMs}ms — ` + `${families.filter((f) => f.checked).length}/${families.length} families checked, ` + `${healedTotal} heal(s): ` + - families - .map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}@${f.durationMs ?? 0}ms`) - .join(', ') + families.map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}`).join(', ') ) return report } @@ -19434,105 +18877,12 @@ export class Brainy implements BrainyInterface { } /** - * @description Close and clean up: flush every buffered component, stamp - * the durability markers, release resources, then give up the writer lock. + * Close and cleanup * - * TWO PARTS, AND THE SECOND IS UNCONDITIONAL. Everything that persists data - * runs in {@link closeDurableSteps}; the terminal releases — the flush-request - * watcher, the WRITER LOCK, the VFS timers, and the terminal `closed` flag — - * run whether those steps succeeded or not, in a `finally`. A close that - * threw halfway used to strand the writer lock on disk with this process's - * (soon dead) pid in it, so the next boot of every affected store announced - * `Overwriting stale writer lock … appears dead` after an orderly exit and - * an operator had to decide whether their database had crashed. A closed - * brain holds no lock — there is no failure for which the opposite is the - * safer answer. - * - * The original failure is never swallowed: it is narrated with what it costs - * the next open, then rethrown to the caller. - * @returns Nothing. - * @throws The first failure from the durable close steps, after the - * terminal releases have run. + * Now flushes HNSW dirty nodes before closing + * This ensures deferred persistence mode data is saved */ async close(): Promise { - let closeFailure: unknown = null - try { - await this.closeDurableSteps() - } catch (error) { - closeFailure = error - } - - // ---- TERMINAL RELEASES: always, even after a failure above ---- - - // Stop the cross-process flush-request watcher (no-op if never started). - try { - if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { - this.storage.stopFlushRequestWatcher() - } - } catch (error) { - console.warn('[Brainy] close: stopping the flush-request watcher failed:', error) - } - - // Release the writer lock. Runs after the metadata buffer drain in - // closeDurableSteps() — otherwise a pending write could land after a - // successor writer claimed the lock — and runs even if that drain threw: - // holding a lock from a process that is about to exit locks the store's - // next boot out of a clean verdict. - try { - if (this.storage && typeof this.storage.releaseWriterLock === 'function') { - await this.storage.releaseWriterLock() - } - } catch (error) { - console.warn('[Brainy] close: releasing the writer lock failed:', error) - } - - // Shut down the VFS: stops its background maintenance interval and the - // PathResolver's — both are ref'd timers that would keep the process - // alive after the last brain closes (consumer-reported hang). - try { - if (this._vfs) { - await this._vfs.close() - } - } catch (error) { - console.warn('[Brainy] close: VFS shutdown failed:', error) - } - - this.initialized = false - // close() is terminal: block lazy re-initialization on any subsequent - // operation (ensureInitialized() throws once this is set). Set even when - // the durable steps failed — a half-closed brain must not keep serving. - this.closed = true - - // Drop this instance from the global registry, and when it was the last - // one, deregister the global shutdown hooks — their ref'd signal handles - // would otherwise keep the process alive after every brain is closed. - const instanceIndex = Brainy.instances.indexOf(this) - if (instanceIndex !== -1) { - Brainy.instances.splice(instanceIndex, 1) - } - Brainy.deregisterShutdownHooksIfIdle() - - if (closeFailure !== null) { - console.error( - `[Brainy] close FAILED partway: ` + - `${closeFailure instanceof Error ? closeFailure.message : String(closeFailure)}\n` + - ` This brain is closed and holds no writer lock, but the clean-shutdown ` + - `marker may not have been written — the next open will run crash recovery ` + - `(a generation-log fold) and report its wall.` - ) - throw closeFailure - } - } - - /** - * @description The durable half of {@link close}: flush every component, - * persist the generation counter and its markers, close the components, - * deactivate plugins, drain the metadata write buffer. Separated from - * `close()` so the terminal releases there can run in a `finally` — see that - * method's contract. - * @returns Nothing. - */ - private async closeDurableSteps(): Promise { // Persistence cadence teardown: no background flush may fire after close // begins (close() runs its own final flush). if (this._persistIdleTimer) { @@ -19683,6 +19033,38 @@ export class Brainy implements BrainyInterface { } } + // Stop the cross-process flush-request watcher (no-op if never started). + if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { + this.storage.stopFlushRequestWatcher() + } + + // Release the writer lock (no-op for readers and for backends that don't + // hold a lock). Must run after the metadata buffer drain — otherwise a + // pending write could land after a successor writer claimed the lock. + if (this.storage && typeof this.storage.releaseWriterLock === 'function') { + await this.storage.releaseWriterLock() + } + + // Shut down the VFS: stops its background maintenance interval and the + // PathResolver's — both are ref'd timers that would keep the process + // alive after the last brain closes (consumer-reported hang). + if (this._vfs) { + await this._vfs.close() + } + + this.initialized = false + // close() is terminal: block lazy re-initialization on any subsequent + // operation (ensureInitialized() throws once this is set). + this.closed = true + + // Drop this instance from the global registry, and when it was the last + // one, deregister the global shutdown hooks — their ref'd signal handles + // would otherwise keep the process alive after every brain is closed. + const instanceIndex = Brainy.instances.indexOf(this) + if (instanceIndex !== -1) { + Brainy.instances.splice(instanceIndex, 1) + } + Brainy.deregisterShutdownHooksIfIdle() } } diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts index 2f01e935..98342884 100644 --- a/src/db/familyStamp.ts +++ b/src/db/familyStamp.ts @@ -12,11 +12,9 @@ * the verified surface is a small set of rollup invariants (entity/ * relationship counts) plus `sourceGeneration`. * - * `sourceGeneration` is the COMMITTED generation of the source-of-truth log - * this projection reflects — never the allocated counter, which names a - * generation that may never commit (see {@link StampVerdict.torn}) — so - * open-time coherence becomes a COMPARISON (stamp vs committed head), not a - * walk: + * `sourceGeneration` is the generation of the source-of-truth log this + * projection reflects — open-time coherence becomes a COMPARISON (stamp vs + * log head), not a walk: * * - equal + invariants hold → coherent, serve. * - behind → the projection missed the tail (crash between commit and stamp); @@ -26,9 +24,6 @@ * - invariants FAIL at equal generation → genuine incoherence: loud, and the * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * canonical walk) heals it. - * - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log - * tail's. TERMINAL, never a wait — the generation the stamp names does not - * exist to arrive. * * Stamps are JSON on purpose — every incident gets debugged by reading a * stamp in a terminal. @@ -75,12 +70,6 @@ export type StampVerdict = | { state: 'coherent' } | { state: 'absent' } // legacy store — first stamp writes at the next flush | { state: 'behind'; stampSource: number; head: number } - /** - * TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the - * store's committed watermark can no longer show. TERMINAL — there is no - * generation to wait for, so the open demotes (or refuses) and never spins. - */ - | { state: 'torn'; stampSource: number; head: number } | { state: 'incoherent'; failures: string[] } | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence @@ -129,15 +118,12 @@ export function verifyFamilyStamp( ): StampVerdict { if (stamp === null) return { state: 'absent' } if (stamp.sourceGeneration > head) { - // A stamp AHEAD of committed truth witnesses a generation the store can no - // longer show: the stamp's fsync survived a crash that the log tail did - // not. This is the TORN GENERATION-LOG TAIL — its own class, never folded - // in with `incoherent` (a count that drifted at a generation both sides - // agree on), because the two have opposite cures: incoherence is recounted, - // a tear is DEMOTED. It is also terminal by construction — there is no - // generation the open can wait for, because the one the stamp names is - // gone. - return { state: 'torn', stampSource: stamp.sourceGeneration, head } + // A stamp AHEAD of the log claims state that never committed — the + // projection was stamped against truth that a crash rolled back. + return { + state: 'incoherent', + failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] + } } if (stamp.sourceGeneration < head) { return { state: 'behind', stampSource: stamp.sourceGeneration, head } diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 91451281..0c14b60c 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,60 +147,6 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } - /** - * @description True when `meta` declares more generations than it holds - * frames — a segment sealed by a writer that folded across a hole. The - * manifest records `frames` at fold time, so this is an O(1) comparison - * against the declared span and needs no I/O. - */ - private isSparse(meta: SegmentMeta): boolean { - return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames - } - - /** - * @description The generations this tier ACTUALLY holds, as coalesced - * ascending intervals — not what the segments declare. - * - * Dense segments (every one a current writer produces) contribute their - * declared range with no I/O. A SPARSE segment — one sealed before the - * density law was enforced, whose declared range spans generations it has - * no frame for — has its real generation list read from its sidecar and - * contributed instead, with the discrepancy narrated once. - * - * This is what keeps a store that already carries the damage from wedging. - * `open()` seeds `committedRanges` from these intervals, so a hole is never - * re-admitted as a committed generation, and the auto-compaction pass that - * used to fail on every run with "packed history is damaged" simply never - * asks for the missing frame. - * - * @returns Ascending, non-overlapping `[first, last]` intervals. - */ - async actualRanges(): Promise> { - const out: Array<[number, number]> = [] - for (const meta of this.manifest.segments) { - if (!this.isSparse(meta)) { - out.push([meta.firstGeneration, meta.lastGeneration]) - continue - } - const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames - prodLog.warn( - `[GenerationSegments] sealed segment ${meta.file} declares generations ` + - `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + - `frame(s) — ${missing} generation(s) in that span were never folded into it. ` + - `Serving the frames it actually holds; the declared span is not treated as ` + - `committed history. (Written by a pre-density-law writer that folded across a ` + - `gap; the segment itself is intact and no record is lost.)` - ) - const idx = await this.sidecarFor(meta) - for (const [gen] of idx.generations) { - const last = out[out.length - 1] - if (last !== undefined && gen === last[1] + 1) last[1] = gen - else out.push([gen, gen]) - } - } - return out - } - /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -218,38 +164,6 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } - // THE DENSITY LAW, MADE MECHANICAL. - // - // A sealed segment declares a CONTIGUOUS range [firstGeneration, - // lastGeneration] and every reader treats that range as containment: - // `coveringSegment` is an interval test, `hasGeneration` returns true for - // anything inside it, and `open()` seeds committedRanges from it. So a - // segment folded from a SPARSE input silently claims generations it does - // not hold, and the first read of one of those holes throws - // "inside sealed segment ... but has no frame — packed history is damaged". - // - // That is exactly how the damage was produced. `repackHistory` skipped - // generations mid-batch — ones absent from committedRanges, ones still in - // the pending buffer, ones whose tx.json would not read — and handed the - // survivors here, where the range was computed from the first and last of - // them. Worse, the mis-declared range was then merged back into - // committedRanges at the next open, which is what turned a quiet hole into - // a repeating auto-compaction failure on every subsequent run. - // - // Callers now split at discontinuities; this refusal is what keeps any - // future caller from reintroducing the class. A refusal here loses - // nothing — the generations stay in the live tier, readable, and the next - // pass folds them correctly. - for (let i = 1; i < gens.length; i++) { - if (gens[i].generation !== gens[i - 1].generation + 1) { - throw new Error( - `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` + - `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + - `generation(s). A sealed segment declares a dense range, so folding a sparse ` + - `batch would claim generations it does not hold. Split the batch at the gap.` - ) - } - } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -450,37 +364,12 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // Inside the covering range but with no frame. Two very different causes, - // and conflating them is what made this class wedge every maintenance pass - // on the affected stores. - // - // (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than - // the span it declares. That segment was sealed by a writer that - // folded across a hole (the class this file's density law now bars). - // The segment is INTACT and nothing is lost; it simply never held this - // generation. Answering "not packed" is the honest answer, and it lets - // the caller's two-tier read decide what a genuinely absent generation - // means, instead of every compaction pass dying on a repeating throw. - // `actualRanges()` keeps such holes out of committedRanges at open, so - // in a healed store nobody asks this question in the first place. - // - // (2) A DENSE SEGMENT missing a frame it says it has — the manifest and - // the sidecar disagree about a segment that claims to be complete. - // That IS damage, and it stays loud. - if (this.isSparse(meta)) { - prodLog.warn( - `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + - `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + - `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + - `-generation span — it was sealed across a gap and never held this generation. ` + - `Reporting it as unpacked rather than as damage; no record is lost.` - ) - return null - } + // In the covering range but not present: the packed tier is dense by + // construction (fold packs every generation it is handed, including + // record-less ones) — absence inside a sealed range is damage. throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + - `span — the manifest and the sidecar disagree; packed history is damaged` + `range but has no frame — packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index da21dc61..81bed338 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,35 +96,6 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' -/** - * @description Split an ascending list of fold candidates into maximal - * CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. - * - * A sealed segment declares one dense range `[firstGeneration, - * lastGeneration]`, and every reader treats that range as containment. So a - * batch with a hole in it must never become one segment: it would claim a - * generation it does not hold, and the first read of that hole reports the - * packed history as damaged. One run, one segment — the ranges then describe - * exactly what the segments contain. - * - * @param gens - Fold candidates, strictly ascending by generation. - * @returns One array per contiguous run, in ascending order. Empty in, empty out. - */ -export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { - const runs: FoldGeneration[][] = [] - let run: FoldGeneration[] = [] - for (const g of gens) { - const prev = run[run.length - 1] - if (prev !== undefined && g.generation !== prev.generation + 1) { - runs.push(run) - run = [] - } - run.push(g) - } - if (run.length > 0) runs.push(run) - return runs -} - /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -566,29 +537,12 @@ export class GenerationStore { this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) - // Discover existing generation record directories — BY DIRECTORY NAME. - // This used to call listRawObjects(), which recurses the whole - // `_generations/` tree and returns every file in every generation, to - // extract a set of integers the top-level directory names already spell. - // MEASURED on a real store with an 11 GB generation history: the phase - // this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with - // no fold to blame — this walk is what it was doing. An adapter without - // the one-level door falls back to the recursive listing, unchanged. + // Discover existing generation record directories. + const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) const seenGens = new Set() - const oneLevel = ( - this.storage as { listRawPrefixes?: (prefix: string) => Promise } - ).listRawPrefixes - if (typeof oneLevel === 'function') { - for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) { - const gen = Number(name) - if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen) - } - } else { - const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) - for (const p of recordPaths) { - const gen = parseGenerationFromPath(p) - if (gen !== null) seenGens.add(gen) - } + for (const p of recordPaths) { + const gen = parseGenerationFromPath(p) + if (gen !== null) seenGens.add(gen) } let rolledBack = 0 @@ -698,7 +652,6 @@ export class GenerationStore { : 'WHOLE-LOG fold' : 'above-manifest replay' let replayed = 0 - const foldStartedAt = Date.now() const replayFact = async (fact: CommitFact): Promise => { for (const op of fact.ops) { let image: { metadata: unknown | null; vector: unknown | null } @@ -744,10 +697,9 @@ export class GenerationStore { } replayed++ if (replayed % 1000 === 0) { - prodLog.narrate( + prodLog.warn( `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + - `in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` + - `do not restart, the fold is finite` + `(at generation ${fact.generation}); do not restart, the fold is finite` ) } if (fact.generation > this.committed) { @@ -762,7 +714,7 @@ export class GenerationStore { } } if (uncleanOpen) { - prodLog.narrate( + prodLog.warn( `[GenerationStore] log-authority recovery: ${foldKind} beginning ` + `(unclean shutdown detected) — streaming replay, bounded memory, ` + `progress every 1000 facts. Do not restart the process; a restart ` + @@ -785,10 +737,9 @@ export class GenerationStore { } await this.storage.writeRawObject(MANIFEST_PATH, manifest) await this.storage.syncRawObjects([MANIFEST_PATH]) - prodLog.narrate( + prodLog.warn( `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` + - `${this.committed}) — an acked write is never lost` + `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost` ) } // A recovery fold re-applied (and the barrier below re-syncs) every @@ -813,15 +764,9 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law - // writer can declare a span wider than the frames it holds; seeding - // committedRanges from the declared span re-admits those holes as - // committed generations, and every later maintenance pass then asks for a - // frame that was never written. `actualRanges()` reads the real - // generation list from the sidecar for exactly those segments (and does - // no I/O for the dense ones, which is all of them on a healthy store). - const packedRanges = (await this.segments.actualRanges()) - .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) + const packedRanges = this.segments + .segments() + .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets — both ascending; @@ -3156,26 +3101,13 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three - // filters above punch holes in it: a generation missing from - // committedRanges never appears, one still in the pending buffer is - // skipped, and one whose tx.json will not read is skipped. A sealed - // segment declares a DENSE range, so folding across such a hole makes - // the segment claim a generation it does not hold; the next open - // merges that mis-declared range into committedRanges, and every - // subsequent auto-compaction pass then asks for the missing frame and - // fails with "packed history is damaged". Fold each contiguous RUN as - // its own segment instead — same bytes, honest ranges. - for (const run of contiguousRuns(foldInput)) { - if (deadline !== undefined && Date.now() >= deadline) break - await segments.fold(run) - segmentsCreated++ - // Segment + manifest durable → the live copies retire. - for (const g of run) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) - } - folded += run.length + await segments.fold(foldInput) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of foldInput) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) } + folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/src/db/types.ts b/src/db/types.ts index 56bdef11..363de086 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -450,21 +450,6 @@ export interface GenerationStorage { deleteRawObject(path: string): Promise /** List raw object paths under a prefix (normalized, `.gz`-stripped). */ listRawObjects(prefix: string): Promise - /** - * OPTIONAL: the IMMEDIATE child directory names under a prefix — one level, - * no recursion, no file paths. - * - * Why it exists: discovering which generations are on disk needs only the - * top-level directory NAMES under `_generations/`, but the only door for it - * was `listRawObjects`, which recurses the whole tree and returns every file - * in every generation. On a store with a long history that is a full walk of - * the entire generation log, paid on EVERY open, to learn a set of integers - * the directory names already spell out. - * - * An adapter without this door keeps working — the caller falls back to the - * recursive listing. - */ - listRawPrefixes?(prefix: string): Promise /** Remove every object under a prefix (and the directory itself on disk). */ removeRawPrefix(prefix: string): Promise /** Durability barrier: fsync the given object paths (no-op in memory). */ diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index ebd3b90c..d002164e 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1052,17 +1052,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { */ private startAutoFlush(): void { this.flushTimer = setInterval(async () => { - // NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an - // index nobody has written to since the last flush has nothing to - // write, and calling into the trees (and their logging) on a cadence - // over a quiet store is exactly the idle cost this law exists to - // remove. - if ( - !this.lsmTreeVerbsBySource.hasPendingWrites() && - !this.lsmTreeVerbsByTarget.hasPendingWrites() - ) { - return - } await this.flush() }, this.config.flushInterval) // Background maintenance must never keep the host process alive — diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index b4f6052f..e19ec145 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -687,17 +687,6 @@ export class LSMTree { } } - /** - * @description Whether this tree holds anything a flush would write — - * the MemTable is non-empty. Synchronous and O(1), so a background cadence - * can ask before it does anything at all: the engine does no periodic work - * without a cause. - * @returns true when a flush would write; false when it would be a no-op. - */ - hasPendingWrites(): boolean { - return !this.memTable.isEmpty() - } - async close(): Promise { this.stopCompactionTimer() diff --git a/src/index.ts b/src/index.ts index edc21809..973136a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -184,7 +184,6 @@ export { // Export version utilities export { getBrainyVersion } from './utils/version.js' -export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js' // Export plugin system export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 5ec1d88e..4f2a43b0 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -14,8 +14,7 @@ import { StorageBatchConfig, SYSTEM_DIR, STATISTICS_KEY, - WriterLockInfo, - WriterCloseRecord + WriterLockInfo } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' @@ -100,30 +99,7 @@ export class FileSystemStorage extends BaseStorage { // timer rewrites the lock every 10s so stale-lock detection can tell a dead // writer from a slow one. The constant name matches the file path used. private static readonly WRITER_LOCK_FILE = '_writer.lock' - /** - * The clean-close record at `locks/_writer.close` (see - * {@link WriterCloseRecord}). Written when the lock is released, consumed by - * the next claim, so an open can distinguish "the previous writer left" from - * "the previous writer died" without inferring either from a pid. - */ - private static readonly WRITER_CLOSE_FILE = '_writer.close' - /** - * How often the lock file's `lastHeartbeat` is rewritten. - * - * THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness - * is decided by PID LIVENESS alone (see isWriterLockStale) and the fence - * compares pid + hostname — no decision anywhere reads this timestamp. It - * exists so an operator inspecting a lock file, or reading the - * BRAINY_WRITER_LOCKED error, can judge liveness themselves. - * - * At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1 - * writes/s across a production process holding 21 idle brains, for a - * human-readable timestamp nothing computes with. At 60s an operator still - * sees a heartbeat inside the minute, at a sixth of the cost. With the - * clean-close record now recording orderly releases explicitly, the - * heartbeat carries even less weight than it did. - */ - private static readonly WRITER_HEARTBEAT_MS = 60_000 + private static readonly WRITER_HEARTBEAT_MS = 10_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo @@ -136,13 +112,6 @@ export class FileSystemStorage extends BaseStorage { */ private writerHeartbeatInFlight?: Promise - /** - * The in-flight background count-ledger derivation, if one was needed at - * open. See {@link scheduleCountLedgerDerivation} — awaited only by - * {@link whenCountLedgerSettled}, never by a read. - */ - private countLedgerDerivation?: Promise - // Flush-request RPC state. The writer polls `locks/_flush_requests/` for // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after // flushing. Inspectors call `requestFlushOverFilesystem` to drop a request @@ -151,16 +120,9 @@ export class FileSystemStorage extends BaseStorage { private static readonly FLUSH_REQUEST_DIR = '_flush_requests' private static readonly FLUSH_RESPONSE_DIR = '_flush_responses' private static readonly FLUSH_WATCH_INTERVAL_MS = 500 - /** - * The safety sweep behind the fs.watch: catches events an exotic filesystem - * dropped, and runs the stale-request GC. See startFlushRequestWatcher. - */ - private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000 private static readonly FLUSH_POLL_INTERVAL_MS = 100 private static readonly FLUSH_REQUEST_TTL_MS = 60_000 private flushWatcherInterval?: NodeJS.Timeout - /** The inotify-backed watch on the request directory, when the FS supports one. */ - private flushWatcher?: import('node:fs').FSWatcher private flushWatcherInFlight = false private flushWatcherOnRequest?: () => Promise @@ -709,30 +671,6 @@ export class FileSystemStorage extends BaseStorage { return pruned } - /** - * @description The IMMEDIATE child directory names under a prefix — ONE - * `readdir`, no recursion, no file paths. See the seam's JSDoc - * (`src/db/types.ts`) for what this replaced: discovering the generations on - * disk walked the entire generation log on every open, reading out every - * file in every generation, to learn the set of integers the top-level - * directory names already spell. - * @param prefix - Storage-root-relative directory prefix. - * @returns The child directory names (not paths); empty when the prefix does - * not exist. - */ - public override async listRawPrefixes(prefix: string): Promise { - await this.ensureInitialized() - const fullPath = path.join(this.rootDir, prefix) - try { - const entries = await fs.promises.readdir(fullPath, { withFileTypes: true }) - return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory()) - .map((e: { name: string }) => e.name) - } catch (error: any) { - if (error?.code === 'ENOENT') return [] - throw error - } - } - /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing @@ -1943,41 +1881,18 @@ export class FileSystemStorage extends BaseStorage { } } - // THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see - // WriterCloseRecord). A lock file whose release was RECORDED is - // bookkeeping left by an orderly shutdown, not evidence of anything — - // and that is true whether the previous holder was another process or - // an earlier instance in THIS one. A production restart reported - // "Re-acquiring writer lock ... this is a bug" immediately after a clean - // close, sending an operator hunting for a leak that did not exist. - const closeRecord = existing ? await this.readWriterCloseRecord() : null - const releasedCleanly = - existing !== null && - closeRecord !== null && - this.closeRecordVouchesFor(closeRecord, existing) - if (existing) { // Same-process re-open: a second Brainy instance in this Node process // (e.g. test "simulate server restart" patterns, or a consumer that // explicitly re-instantiates without closing first). This isn't the // dangerous cross-process case the lock exists to prevent — the two // instances share a memory space and can't silently diverge from each - // other beyond what their callers already see. Warn and take over — - // unless the record proves the previous instance already let go, in - // which case there is nothing to warn about. + // other beyond what their callers already see. Warn and take over. if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { - if (releasedCleanly) { - console.warn( - `[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` + - `instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` + - `${closeRecord!.closedAt} but could not remove the file. Nothing to recover.` - ) - } else { - console.warn( - `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + - `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` - ) - } + console.warn( + `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` + ) const info: WriterLockInfo = { pid: myPid, hostname, @@ -1987,18 +1902,11 @@ export class FileSystemStorage extends BaseStorage { rootDir: this.rootDir } await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) - await this.clearWriterCloseRecord() this.installWriterLock(info) return info } - // A cleanly-released lock is stale by RECORD, not by inference. Only - // when no record vouches for this lock do we fall back to pid - // liveness, and then we say THAT honestly too: an unrecorded lock - // means the writer did not complete its close, so the store was not - // closed cleanly and this open pays recovery. - const stale = - releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing))) + const stale = !options?.force && (await this.isWriterLockStale(existing)) if (!options?.force && !stale) { // Consumer-facing error contract: callers detect this case via // err.code and read the holder's details from err.lockInfo. @@ -2009,16 +1917,8 @@ export class FileSystemStorage extends BaseStorage { options?.force ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + `(was held by PID ${existing.pid} on ${existing.hostname}).` - : releasedCleanly - ? `[brainy] Clearing the leftover writer lock for ${this.rootDir} — ` + - `PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` + - `${closeRecord!.closedAt} but could not remove the file. ` + - `Nothing to recover.` - : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + - `(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` + - `clean-close record — that writer did not finish closing, so this ` + - `store was not closed cleanly; open will run crash recovery and ` + - `report its wall).` + : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + + `(PID ${existing.pid} on ${existing.hostname} appears dead).` ) // Takeover: verify the file still holds the lock we judged (a live // successor may have claimed meanwhile), then remove it and fall @@ -2072,12 +1972,6 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.unlink(claimTmp).catch(() => {}) } - // CONSUME the previous writer's clean-close record. It described the - // lock generation that just ended; leaving it in place would let it - // vouch for OUR lock if this process later dies without closing — - // turning a real crash into a "closed cleanly" verdict. One unlink. - await this.clearWriterCloseRecord() - this.installWriterLock(info) return info } @@ -2201,27 +2095,13 @@ export class FileSystemStorage extends BaseStorage { return } const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) - const released = this.writerLockInfo try { // Only delete if we still own it — avoid clobbering a successor that // claimed the lock via force-override. const current = await this.readWriterLock() - const ours = - current === null || - (current.pid === released.pid && current.hostname === released.hostname) - if (current && ours) { + if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) { await fs.promises.unlink(lockFile) } - // THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this - // instance gives up a lock nobody else has taken — the unlink above - // having succeeded OR the file already being gone. The next open reads - // it instead of guessing from pid liveness: a recorded release is an - // orderly shutdown, an absent record is a writer that never finished - // closing. Not written when a successor holds the lock: our release is - // then a no-op and a record would slander their live lock. - if (ours) { - await this.writeWriterCloseRecord(released) - } } catch (err: any) { if (err.code !== 'ENOENT') { console.warn('[brainy] Failed to release writer lock file:', err) @@ -2231,97 +2111,6 @@ export class FileSystemStorage extends BaseStorage { } } - /** - * @description Read the clean-close record at `locks/_writer.close`, or - * `null` when it is absent or unparseable. A torn record is treated as - * absent — the conservative direction, since an unreadable record can - * vouch for nothing. - * @returns The record, or null. - */ - public async readWriterCloseRecord(): Promise { - await this.ensureInitialized() - const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) - try { - const raw = await fs.promises.readFile(recordFile, 'utf-8') - const parsed = JSON.parse(raw) as WriterCloseRecord - if ( - typeof parsed?.pid !== 'number' || - typeof parsed?.hostname !== 'string' || - typeof parsed?.startedAt !== 'string' || - typeof parsed?.closedAt !== 'string' - ) { - return null - } - return parsed - } catch (err: any) { - if (err.code === 'ENOENT') return null - return null - } - } - - /** - * @description Whether a clean-close record describes the very lock - * generation `lock` represents. The match is pid + hostname + `startedAt`: - * `startedAt` is the lock generation's identity, so a record can never - * vouch for a LATER lock taken by the same pid on the same host (the - * same-process re-open path mints a fresh `startedAt`). - * @param record - The clean-close record read from disk. - * @param lock - The lock file's contents. - */ - private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean { - return ( - record.pid === lock.pid && - record.hostname === lock.hostname && - record.startedAt === lock.startedAt - ) - } - - /** - * @description Write the clean-close record for a lock this instance just - * released. Atomic (temp + rename) so a concurrent opener never reads half - * a record. A failure here costs the next open nothing but the honest - * fallback (pid liveness), so it warns rather than failing the close. - * @param released - The lock info this instance held. - */ - private async writeWriterCloseRecord(released: WriterLockInfo): Promise { - const record: WriterCloseRecord = { - pid: released.pid, - hostname: released.hostname, - startedAt: released.startedAt, - closedAt: new Date().toISOString(), - version: released.version - } - const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) - try { - await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2)) - } catch (err) { - // ENOENT = the lock directory is gone, i.e. the whole store was removed - // under us. There is no next open to inform. - if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return - console.warn( - `[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` + - `the next open will fall back to pid liveness and may report this orderly ` + - `shutdown as a crash:`, - err - ) - } - } - - /** - * @description Remove the clean-close record. Called by every successful - * lock claim so a record never outlives the lock generation it describes. - */ - private async clearWriterCloseRecord(): Promise { - const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) - try { - await fs.promises.unlink(recordFile) - } catch (err: any) { - if (err.code !== 'ENOENT') { - console.warn('[brainy] Failed to clear the writer clean-close record:', err) - } - } - } - public override async readWriterLock(): Promise { await this.ensureInitialized() const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) @@ -2408,115 +2197,36 @@ export class FileSystemStorage extends BaseStorage { /** * Start watching for cross-process flush requests. Called by Brainy.init() - * in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers - * the supplied callback (`brain.flush()`), after which an `.ack` is written - * to `locks/_flush_responses/` with the same request ID. Stale `.req` files - * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep. - * - * THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request - * directory every 500 ms, per brain, for the entire life of every writer — - * armed on every non-reader brain whether or not any inspector process - * existed. MEASURED on a production process holding 21 brains: 42 directory - * reads per second on a completely idle service, plus a stale-request GC - * pass on every one of them. The engine does no periodic work without a - * cause, and a request that has not been made is not a cause. - * - * `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is - * seen SOONER than the old poll saw it. Two honest concessions ride with it: - * - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because - * `fs.watch` can miss events on network and fuse filesystems and because - * the stale-request GC needs some tick of its own. At 30s that is 0.7 - * reads/s across 21 brains where the poll cost 42. - * - a filesystem that cannot watch at all falls back to the ORIGINAL - * 500 ms poll, narrated once, because correctness outranks idle cost: - * an inspector whose request is never seen waits forever. + * in writer mode. Polls `locks/_flush_requests/` every + * FLUSH_WATCH_INTERVAL_MS — each new `.req` file triggers the supplied + * callback (`brain.flush()`), after which an `.ack` is written to + * `locks/_flush_responses/` with the same request ID. Stale `.req` files + * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick. */ public override startFlushRequestWatcher(onRequest: () => Promise): void { - // Already watching — or already ARMING. The arm is asynchronous (the - // request directory is created before it can be watched), so neither the - // watcher nor the interval exists yet during that window; the callback is - // the flag that covers it. Without this a second call in the window would - // leave two watchers and two sweeps running for the life of the store. - if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return + if (this.flushWatcherInterval) return // already watching this.flushWatcherOnRequest = onRequest const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) - const sweep = (): void => { - if (this.flushWatcherInFlight) return // skip overlapping sweep + // Ensure both dirs exist up front so the first .req drop doesn't race with mkdir. + this.ensureDirectoryExists(reqDir).catch(() => {}) + this.ensureDirectoryExists(ackDir).catch(() => {}) + + this.flushWatcherInterval = setInterval(() => { + if (this.flushWatcherInFlight) return // skip overlapping tick this.flushWatcherInFlight = true this.processFlushRequests(reqDir, ackDir).finally(() => { this.flushWatcherInFlight = false }) - } - - // Ensure both dirs exist up front so the first .req drop doesn't race with - // mkdir — and so there is a directory to watch. - void this.ensureDirectoryExists(reqDir) - .then(() => this.ensureDirectoryExists(ackDir)) - .then(() => { - if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile - try { - const watcher = fs.watch(reqDir, () => sweep()) - this.flushWatcher = watcher - watcher.on('error', (err: Error) => { - // A watch that dies mid-life must not leave the door deaf. - console.warn( - `[brainy] Flush-request watch failed (${err.message}) — falling back to polling.` - ) - this.flushWatcher?.close() - this.flushWatcher = undefined - // The SAFETY sweep must go first. It is already armed at 30s, and - // startFlushRequestPolling() declines to arm over an existing - // interval — so leaving it would quietly leave this store answering - // flush requests on a 30s cadence instead of the 500ms one the door - // promises. A degrade nobody asked for is still a degrade. - if (this.flushWatcherInterval) { - clearInterval(this.flushWatcherInterval) - this.flushWatcherInterval = undefined - } - this.startFlushRequestPolling(sweep) - }) - if (typeof watcher.unref === 'function') watcher.unref() - // The safety sweep: missed events on exotic filesystems, and the - // stale-request GC. - this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS) - if (typeof this.flushWatcherInterval.unref === 'function') { - this.flushWatcherInterval.unref() - } - // One sweep now: a request may have been dropped before the watch armed. - sweep() - } catch (err) { - console.warn( - `[brainy] Flush-request directory cannot be watched on this filesystem ` + - `(${(err as Error).message}) — polling every ` + - `${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.` - ) - this.startFlushRequestPolling(sweep) - } - }) - .catch(() => { - // The request directory could not be created; nothing to watch. A - // cross-process flush request cannot be made either, so there is - // nothing to miss. - }) - } - - /** The original 500 ms poll — the fallback when a directory cannot be watched. */ - private startFlushRequestPolling(sweep: () => void): void { - if (this.flushWatcherInterval) return - this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) + }, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) if (typeof this.flushWatcherInterval.unref === 'function') { this.flushWatcherInterval.unref() } } public override stopFlushRequestWatcher(): void { - if (this.flushWatcher) { - this.flushWatcher.close() - this.flushWatcher = undefined - } if (this.flushWatcherInterval) { clearInterval(this.flushWatcherInterval) this.flushWatcherInterval = undefined @@ -2913,29 +2623,25 @@ export class FileSystemStorage extends BaseStorage { this.allCountsDerivedBy = undefined this.allCountsSuspect = true needsPersist = true - prodLog.narrate( + prodLog.warn( '[FileSystemStorage] canonical count ledger was derived under the legacy ' + - 'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' + - 'container inflates it. Marked suspect, and an honest recount is scheduled to ' + - 'run in the background after this open; until it lands, do not subtract ' + - 'against these ALL scalars.' + 'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' + + 'exact denominators' ) - // A suspect ledger used to stay wrong for the life of the store, - // waiting for an operator to run repairIndex. A downstream index - // heal took its "remaining" figure from these inflated - // denominators and reported work that did not exist. The ledger - // now HEALS ITSELF — in the background, because a denominator is - // a derived scalar and no read is ever served from it. - this.scheduleCountLedgerDerivation('legacy container-rule ledger') } } else { - // No ALL scalars at all. There is nothing to serve in the meantime — - // a zero would read as an empty store — so the scalars stay unknown - // and SUSPECT until the background derivation lands. The open does - // not wait for it: an id-tree walk is O(ids) and this file has been - // the whole reason a 24k-id store opened in silence. - this.allCountsSuspect = true - this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger') + const nouns = await this.scanCanonicalEntities('nouns') + const verbs = await this.scanCanonicalEntities('verbs') + this.totalNounCountAll = nouns.count + this.totalVerbCountAll = verbs.count + this.allCountsSuspect = false + this.allCountsDerivedBy = 'identity-record' + console.warn( + `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + + `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + + `every tier) and persisted; no further scan.` + ) + needsPersist = true } // The vectored-noun scalar (shipped after the ALL scalars above — a @@ -2948,12 +2654,14 @@ export class FileSystemStorage extends BaseStorage { if (typeof counts.totalVectoredNounCount === 'number') { this.totalVectoredNounCount = counts.totalVectoredNounCount } else { - // O(nouns) CONTENT reads — the most expensive derivation of the - // three, and the one most likely to have been the silent minutes at - // the front of a large store's open. Background, suspect until it - // lands, same as the ALL scalars. - this.allCountsSuspect = true - this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger') + const vectored = await this.scanVectoredNounCount() + this.totalVectoredNounCount = vectored + console.warn( + `[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` + + `derived once by reading every noun's vectors.json (${vectored} vectored) and ` + + `persisted; no further scan.` + ) + needsPersist = true } if (needsPersist) { await this.persistCounts() @@ -2982,22 +2690,6 @@ export class FileSystemStorage extends BaseStorage { * Initialize counts by scanning disk (only done once) */ private async initializeCountsFromDisk(): Promise { - const startedAt = Date.now() - // THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating: - // it derives `totalNounCount` / `totalVerbCount`, the scalars - // `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would - // make a populated store answer "0 entities" until the walk landed — a - // wrong answer, not a slow one, and the serving law grades a failure by - // whether an answer could be wrong. The ALL-visibility denominators, which - // no read is served from, DO run in the background (see - // scheduleCountLedgerDerivation). What this walk owes the operator instead - // is narration: it announces itself, and reports its wall. - prodLog.narrate( - `[FileSystemStorage] no usable counts.json — deriving the entity counters from ` + - `the canonical id tree now. This is O(ids) listings plus one vectors.json read ` + - `per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` + - `served from it. It runs once; the result is persisted.` - ) try { // Count the CANONICAL 8.0 layout (`entities////…`) — // the tree saveNoun/getNouns actually read and write. The previous scan @@ -3045,11 +2737,6 @@ export class FileSystemStorage extends BaseStorage { } await this.persistCounts() - prodLog.narrate( - `[FileSystemStorage] counter derivation from the canonical id tree finished in ` + - `${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` + - `${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.` - ) } catch (error) { console.error('Error initializing counts from disk:', error) } @@ -3071,118 +2758,6 @@ export class FileSystemStorage extends BaseStorage { * directories (absolute paths) — nouns feed the type-distribution estimate * above. An absent tree (fresh store) counts zero. */ - /** - * @description Derive the ALL-visibility count ledger honestly — one entity - * per IDENTITY RECORD, never per id directory — IN THE BACKGROUND, once, - * and persist the result stamped `identity-record`. - * - * Why background: these scalars are DENOMINATORS. No read is served from - * them, so deriving them cannot be allowed to hold an open hostage — a - * store with 24,898 ids spent minutes of a production restart inside walks - * exactly like these, in silence, before serving anything. Why at all: a - * ledger derived under the old container rule stayed wrong for the life of - * the store, and a downstream index heal subtracted against it and reported - * remaining work that did not exist (measured on a real store: 14,231 - * derived against 14,056 identity records — precisely the store's 25 noun - * scar directories; verbs 72,729 against 72,679, its 50 verb scars). - * - * Idempotent: a second call while one is in flight joins the first. - * @param reason - What made the ledger untrustworthy, quoted in narration. - * @returns Nothing; observe completion with {@link whenCountLedgerSettled}. - */ - private scheduleCountLedgerDerivation(reason: string): void { - if (this.countLedgerDerivation) return - this.countLedgerDerivation = (async () => { - const startedAt = Date.now() - prodLog.narrate( - `[FileSystemStorage] count-ledger derivation started in the background ` + - `(${reason}) — counting identity records, not id directories; the open does ` + - `not wait for it and no read is served from these scalars.` - ) - try { - const beforeNouns = this.totalNounCountAll - const beforeVerbs = this.totalVerbCountAll - const beforeVectored = this.totalVectoredNounCount - // A walk that RACED A WRITE cannot prove its number: a row that landed - // mid-walk may or may not have been in the shard the walk had already - // passed. Rather than persist a figure that might be off by one and - // stamp it "exact", the walk is repeated once on a quiet store, and if - // the store is never quiet the ledger stays SUSPECT and says so. One - // retry, never a spin. - let attempt = 0 - let derived: { nouns: number; verbs: number; vectored: number } | null = null - while (attempt < 2 && derived === null) { - attempt++ - const activityBefore = this.ledgerActivityStamp() - const nouns = await this.scanCanonicalEntities('nouns') - const verbs = await this.scanCanonicalEntities('verbs') - const vectored = await this.scanVectoredNounCount() - if (this.ledgerActivityStamp() === activityBefore) { - derived = { nouns: nouns.count, verbs: verbs.count, vectored } - } - } - if (derived === null) { - this.allCountsSuspect = true - prodLog.narrate( - `[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` + - `after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` + - `every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` + - `against; brain.repairIndex() derives them under a recount barrier.` - ) - return - } - this.totalNounCountAll = derived.nouns - this.totalVerbCountAll = derived.verbs - this.totalVectoredNounCount = derived.vectored - this.allCountsDerivedBy = 'identity-record' - this.allCountsSuspect = false - await this.persistCounts() - prodLog.narrate( - `[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` + - `${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` + - (beforeNouns !== derived.nouns || - beforeVerbs !== derived.verbs || - beforeVectored !== derived.vectored - ? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` + - `difference is ghost and scar containers the old rule counted as entities)` - : ' (unchanged)') + - ` — persisted, stamped identity-record, no longer suspect.` - ) - } catch (error) { - // The ledger stays suspect and the next open retries. Loud: a - // denominator nobody can derive is a fact an operator must have. - this.allCountsSuspect = true - prodLog.error( - `[FileSystemStorage] count-ledger derivation FAILED after ` + - `${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` + - `and must not be subtracted against; the next open retries:`, - error - ) - } - })() - } - - /** - * @description A cheap witness that the ledger changed while a walk was - * running. Every landed write moves one of these live counters, so an - * unchanged stamp across a walk means no write landed during it. - * @returns A value that differs whenever the live ALL scalars have moved. - */ - private ledgerActivityStamp(): string { - return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}` - } - - /** - * @description Resolve once any background count-ledger derivation has - * settled (succeeded or failed). Resolves immediately when none was needed. - * Exists so tests and operators can observe the ledger's honest value rather - * than race it; nothing in the read path waits on this. - * @returns A promise that settles with the derivation. - */ - public async whenCountLedgerSettled(): Promise { - await this.countLedgerDerivation - } - private async scanCanonicalEntities( kind: 'nouns' | 'verbs' ): Promise<{ count: number; sampleDirs: string[] }> { @@ -3341,15 +2916,10 @@ export class FileSystemStorage extends BaseStorage { lastUpdated: new Date().toISOString() } - // ATOMIC (temp + rename), never a plain writeFile. A direct write - // truncates the file first, so every persist opened a window — measured - // at roughly 750ms after a flush or close on a real store — in which a - // concurrent reader saw counts.json EMPTY. An empty file is unparseable, - // and an unparseable ledger sends the next open down the full-rescan - // path: the cheapest file in the store was costing the most expensive - // recovery. The rename is atomic, so a reader sees the old ledger or the - // new one, never neither. - await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2)) + await fs.promises.writeFile( + this.countsFilePath, + JSON.stringify(counts, null, 2) + ) } catch (error) { console.error('Error persisting counts:', error) } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index d8bcb780..5510f93b 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -125,36 +125,6 @@ export interface WriterLockInfo { rootDir?: string // Convenience for log lines / error messages } -/** - * THE CLEAN-CLOSE RECORD. Written by `releaseWriterLock()` at the instant it - * gives up the writer lock, naming the lock identity it released. The next - * `acquireWriterLock()` reads it and can then say — from a RECORD, not from a - * guess — whether the previous writer left on purpose. - * - * Why a record and not PID liveness: "the recorded PID is no longer alive" is - * true of every orderly restart AND of every crash, so the two were reported - * identically ("appears dead") and neither could be trusted. Worse, the same - * inference fails the other way when the operating system RECYCLES the pid — - * a live unrelated process makes a long-dead writer's lock look held, and the - * store refuses to open naming a pid that was never Brainy. A record settles - * both: matched → the previous writer closed cleanly, nothing to recover; - * absent → say so, and name what recovery the open will now run. - * - * Lifecycle: written at release, consumed (deleted) by the next successful - * lock claim — a record must never outlive the lock generation it describes, - * or it would vouch for a later crash. - */ -export interface WriterCloseRecord { - pid: number - hostname: string - /** `startedAt` of the lock this close released — the identity match key. */ - startedAt: string - /** ISO timestamp at which the lock was released. */ - closedAt: string - /** Brainy version that performed the close. */ - version: string -} - /** * FNV-1a hash returning a 2-char hex bucket (00-ff). * Distributes system keys across 256 sub-prefixes to avoid @@ -1437,29 +1407,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.listObjectsUnderPath(prefix) } - /** - * @description The IMMEDIATE child directory names under a prefix — one - * level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a - * separate door exists. This default derives them from the recursive - * listing, so it is never WRONG, only never faster; the filesystem adapter - * overrides it with a single directory read. - * @param prefix - Storage-root-relative directory prefix. - * @returns The child directory names (not paths), in listing order. - */ - public async listRawPrefixes(prefix: string): Promise { - await this.ensureInitialized() - const paths = await this.listObjectsUnderPath(prefix) - const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/` - const names = new Set() - for (const p of paths) { - const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null - if (rest === null) continue - const slash = rest.search(/[/\\]/) - if (slash > 0) names.add(rest.slice(0, slash)) - } - return [...names] - } - /** * Remove every object under a storage-root-relative prefix. The filesystem * adapter overrides this with a recursive directory removal; this default diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index a0d55c1e..d9934c3b 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1217,13 +1217,6 @@ export interface RepairFamilyReport { skipped?: string /** Why the outcome is what it is when neither `detail` nor `skipped` says it. */ reason?: string - /** - * The phase's own wall, in milliseconds. A repair on a production store ran - * for over thirty minutes without a single line of output; an operator had - * to read `top` to know it was alive. A receipt that cannot say WHERE the - * time went is not a receipt — every row carries its own. - */ - durationMs?: number } /** The full receipt returned by repairIndex(). */ diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts index f1b52e3b..498f2003 100644 --- a/src/utils/indexReadiness.ts +++ b/src/utils/indexReadiness.ts @@ -153,83 +153,3 @@ export function assessProviderHealth(provider: unknown): ProviderHealthAssessmen reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] } } - -/** - * @description A provider's self-report that it is REBUILDING ITS OWN index - * right now. Returned by the optional `rebuildInProgress()` hook. - * - * The distinction this exists to make: a provider reporting `serving: false` - * because it is BROKEN and a provider reporting `serving: false` because it is - * BUSY BUILDING ITSELF look identical through `healthReport()` alone, and - * brainy treated both the same way — it called `rebuild()` and waited for it, - * on the foreground of `init()`. A production store whose metadata provider - * had to rebuild paid 641 SECONDS of that wait before `init()` returned, with - * every other family idle behind it. - * - * A provider that reports progress here owns its own rebuild: brainy neither - * starts one nor waits for it, `init()` returns, the other families serve, and - * THAT family's doors refuse by name — carrying this progress — until the - * provider reports itself serving. - * - * Every field but `phase` is optional and every field is a MEASUREMENT: a - * provider reports only what it actually tracks, never an estimate dressed as - * a fact. - */ -export interface ProviderRebuildProgress { - /** The provider's own name for what it is doing. Quoted verbatim in refusals. */ - phase: string - /** Units completed so far, if the provider counts them. */ - done?: number - /** Units expected in total, if the provider knows it. */ - total?: number - /** Epoch millis when this rebuild started, if the provider tracks it. */ - startedAt?: number -} - -/** A provider that can report a rebuild it is running itself. */ -interface MaybeRebuildingProvider { - rebuildInProgress?: () => ProviderRebuildProgress | null -} - -/** - * @description Ask a provider whether it is rebuilding itself right now. - * Synchronous, O(1), feature-detected: a provider without the hook reports - * nothing and is treated exactly as before. - * @param provider - Any index provider, or `null`/`undefined`. - * @returns The provider's progress, or `null` when it is not rebuilding (or - * does not implement the hook). - */ -export function assessProviderRebuild(provider: unknown): ProviderRebuildProgress | null { - const p = provider as MaybeRebuildingProvider | null | undefined - if (p == null || typeof p.rebuildInProgress !== 'function') return null - try { - const progress = p.rebuildInProgress() - if (!progress || typeof progress.phase !== 'string' || progress.phase.length === 0) { - return null - } - return progress - } catch { - // A throwing hook says nothing trustworthy about a rebuild; fall through to - // the ordinary health verdict rather than inventing one. - return null - } -} - -/** - * @description Render a rebuild progress report as one operator-facing clause, - * for a refusal message. Includes only what the provider actually measured. - * @param progress - The provider's report. - * @returns A clause such as `rebuilding ("metadata shadow build", 4,096/14,056, 12s elapsed)`. - */ -export function describeRebuildProgress(progress: ProviderRebuildProgress): string { - const parts: string[] = [`"${progress.phase}"`] - if (typeof progress.done === 'number' && typeof progress.total === 'number') { - parts.push(`${progress.done.toLocaleString()}/${progress.total.toLocaleString()}`) - } else if (typeof progress.done === 'number') { - parts.push(`${progress.done.toLocaleString()} done`) - } - if (typeof progress.startedAt === 'number') { - parts.push(`${Math.round((Date.now() - progress.startedAt) / 1000)}s elapsed`) - } - return `rebuilding (${parts.join(', ')})` -} diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 0d6b6594..5154d4fd 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -266,26 +266,6 @@ export const prodLog = { console.error(message, ...args) }, - /** - * THE NARRATION CHANNEL — always visible, exactly like `error`. - * - * `warn`/`info`/`log` below are clamped to ERROR in any environment that - * looks like production (see isProductionEnvironment), which is the right - * default for chatter and the wrong one for the two things an operator is - * entitled to hear from a database no matter what: WHY IT IS SLOW and WHAT - * IT IS DOING ABOUT IT. A production service opening a 16 GB store spent - * three minutes emitting nothing at all — the phase timings that would have - * named the slow phase were written to `warn` and thrown away by the log - * level. Progress and cost narration goes here; it is never a per-record - * line, always a phase, a wall, or a bounded-cadence heartbeat. - * - * `silent: true` still silences it — that is the consumer's explicit - * request, not a cost default. - */ - narrate: (message?: any, ...args: any[]) => { - console.warn(message, ...args) - }, - // These are suppressed in production unless BRAINY_LOG_LEVEL is set warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args), info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args), diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 3e0e3d17..3cc56b2e 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2241,74 +2241,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { break } - // ===== ARRAY SET OPERATORS ===== - // An element-indexed array field makes all three exact on the - // index path. They were previously ABSENT from this switch, so - // `fieldResults` kept its initial `[]` and the whole find() - // returned an empty page — a documented, matcher-implemented - // operator answering silently wrong. Served here instead. - - // hasAll: [a, b] — the field's array contains EVERY operand: - // the intersection of each element's posting set. - case 'hasAll': { - if (!Array.isArray(operand)) { - fieldResults = [] - break - } - if (operand.length === 0) { - // Vacuously true of every row that HAS the field. - const anyBitmap = (this.columnStore && this.columnStore.hasField(field)) - ? await this.columnStore.rangeQuery(field) - : await this.getExistsBitmapLegacy(field) - fieldResults = this.idMapper.intsIterableToUuids(anyBitmap) - break - } - let intersection: Set | null = null - for (const item of operand) { - const ids = new Set(await this.getIds(field, item)) - if (intersection === null) { - intersection = ids - } else { - for (const id of [...intersection]) { - if (!ids.has(id)) intersection.delete(id) - } - } - if (intersection.size === 0) break - } - fieldResults = intersection ? [...intersection] : [] - break - } - - // noneOf: [a, b] — the field's value is NONE of the operands: - // the complement of their union. - case 'noneOf': { - if (!Array.isArray(operand)) { - fieldResults = [] - break - } - const excludeInts: number[] = [] - for (const value of operand) { - for (const uuid of await this.getIds(field, value)) { - const intId = this.idMapper.getInt(uuid) - if (intId !== undefined) excludeInts.push(intId) - } - } - fieldResults = this.complementIds(excludeInts) - break - } - - // excludes: value — the field's array does NOT contain the value: - // the complement of `contains`. - case 'excludes': { - const excludeInts: number[] = [] - for (const uuid of await this.getIds(field, operand)) { - const intId = this.idMapper.getInt(uuid) - if (intId !== undefined) excludeInts.push(intId) - } - fieldResults = this.complementIds(excludeInts) - break - } - // ===== MISSING OPERATOR ===== // missing: boolean - equivalent to exists: !boolean case 'missing': { @@ -2325,27 +2257,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { } break } - - // ===== EVERYTHING ELSE: REFUSED BY NAME, NEVER ANSWERED EMPTY ==== - // An equality/range posting index cannot evaluate a substring, a - // pattern or an array length without reading every row, and this - // path exists precisely to avoid that. It used to fall out of the - // switch with `fieldResults` still `[]`, so `find({ where: { name: - // { startsWith: 'a' } } })` returned an empty page and looked like - // an answer. An accepted operator either works or refuses — the - // matcher's own support for these operators governs in-memory - // filtering, never an index-backed find(). - default: - throw new BrainyError( - `Filter operator "${op}" on field "${rawField}" cannot be served by the ` + - `metadata index: an equality/range posting index cannot evaluate substrings, ` + - `patterns or array lengths without reading every row. It is REFUSED rather ` + - `than answered with an empty page. Filter on an indexable operator ` + - `(equals/eq, notEquals/ne, oneOf/in, noneOf, greaterThan/gt, ` + - `greaterThanOrEqual/gte, lessThan/lt, lessThanOrEqual/lte, between, contains, ` + - `excludes, hasAll, exists, missing) and narrow the rest in your own code.`, - 'INVALID_QUERY' - ) } // Intersect this operator's matches with the running set (AND semantics // for multiple operators on the same field). diff --git a/src/utils/version.ts b/src/utils/version.ts index 327f923c..f302eae0 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -83,27 +83,3 @@ export function getAugmentationVersion(service: string): { augmentation: string; version: getBrainyVersion() } } - -/** - * The API-contract version this build implements — a single integer that two - * engines can compare without probing prototypes. - * - * A MINOR release is ADDITIVE: doors and error codes may be added, never - * removed or narrowed, and the contract integer does not move. A MAJOR release - * is what a REQUIRED door's removal or a behavioural narrowing costs, and it - * bumps this integer. A consumer pinning `brainyContract` in a peer range is - * therefore pinning "what I may call", not "which build I run". - * - * Declared in package.json as `"brainyContract"` so a manifest, a tool, or a - * sibling package can read it without importing the engine, and returned here - * so a running process can state its own. - */ -export const BRAINY_CONTRACT_VERSION = 1 as const - -/** - * @description The API-contract version this build implements. - * @returns The contract integer — see {@link BRAINY_CONTRACT_VERSION}. - */ -export function contractVersion(): number { - return BRAINY_CONTRACT_VERSION -} diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 46c6a12d..59a16be4 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -6,7 +6,6 @@ */ import { Readable, Writable } from 'stream' -import { prodLog } from '../utils/logger.js' import crypto from 'crypto' import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' @@ -67,20 +66,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { private config: Required> & { rootEntityId?: string } private rootEntityId?: string private initialized = false - /** - * The one-time old-root sweep, in flight. See {@link sweepOldRootsIfNeeded}. - */ - private rootSweep?: Promise - /** - * Where the completed old-root sweep is recorded. Engine plumbing under - * `_system/`, like every other marker there — never enumerated as data. - */ - private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json' - /** - * Below this wall, a sweep that removed nothing says nothing — see - * {@link sweepOldRootsIfNeeded}. - */ - private static readonly ROOT_SWEEP_NARRATE_MS = 1_000 private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer features available via augmentation (brain.use('knowledge')) @@ -158,17 +143,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Create or find root entity this.rootEntityId = await this.initializeRoot() - // Clean up old UUID-based roots — ONCE PER STORE, BEHIND THE DOORS. - // This is a migration sweep for roots created before the fixed root id - // existed. It ran on EVERY open, forever: a filtered find over the whole - // store hunting for duplicates that a store has either always had or - // never will. MEASURED on a 14,056-noun / 72,679-verb store: the phase it - // dominates cost 43-53 SECONDS of every open, warm reopens included. - // Now: a durable marker records that the sweep has run, and a store - // carrying it never sweeps again; a store without one sweeps in the - // BACKGROUND (the sweep only removes duplicate roots — nothing serves - // from them — and it was always declared non-critical). - this.rootSweep = this.sweepOldRootsIfNeeded() + // Clean up old UUID-based roots (one-time migration) + await this.cleanupOldRoots() // Initialize projection registry with auto-discovery of built-in projections this.projectionRegistry = new ProjectionRegistry() @@ -418,100 +394,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { * * This is a one-time migration helper that can be removed in future versions. */ - /** - * @description Run the old-root sweep at most once per store, in the - * background, and record that it ran. See the call site in {@link init} for - * the measurement that made this necessary. - * @returns A promise that settles when the sweep has finished (or was - * skipped); nothing in the read path awaits it. - */ - private async sweepOldRootsIfNeeded(): Promise { - const store = this.rawObjectStore() - if (store === null) { - // A storage adapter with no raw-object door cannot carry the marker. - // Sweep every open, as before — correctness over cost. - await this.cleanupOldRoots() - return - } - try { - const marker = await store.readRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH) - if (marker !== null && marker !== undefined) return - } catch { - // Unreadable marker: sweep, and rewrite it below. - } - // NARRATION HAS A THRESHOLD, like every other line this engine emits on the - // always-visible channel. On a fresh or small store this sweep finds - // nothing and costs a millisecond, and announcing it — twice — on a - // channel a production log level deliberately CANNOT silence would train - // operators to ignore the one channel that exists to be impossible to - // ignore. It speaks when it has something to say: duplicates removed, or a - // wall long enough that somebody watching a slow first open deserves to - // know what is running. Otherwise it does its work and stays quiet. - const startedAt = Date.now() - const duplicatesRemoved = await this.cleanupOldRoots() - const elapsedMs = Date.now() - startedAt - try { - await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, { - sweptAt: new Date().toISOString(), - durationMs: elapsedMs - }) - if (duplicatesRemoved > 0 || elapsedMs >= VirtualFileSystem.ROOT_SWEEP_NARRATE_MS) { - prodLog.narrate( - `[VFS] one-time old-root sweep complete in ${elapsedMs}ms` + - (duplicatesRemoved > 0 - ? `, ${duplicatesRemoved} pre-fixed-id root(s) removed` - : '') + - ' and recorded — no future open pays for it.' - ) - } - } catch (error) { - // Unrecorded sweep = the next open sweeps again. Conservative, and said - // out loud rather than quietly repeated forever. - prodLog.narrate( - `[VFS] old-root sweep finished in ${Date.now() - startedAt}ms but could NOT be ` + - `recorded (${(error as Error).message}) — the next open will sweep again.` - ) - } - } - - /** - * @description Settle once the background old-root sweep has finished. - * Resolves immediately when the store already carried the marker. Exists so - * tests and operators can observe the sweep instead of racing it; no read - * path waits on it. - * @returns A promise that settles with the sweep. - */ - public async whenRootSweepSettled(): Promise { - await this.rootSweep - } - - /** - * @description The brain's storage adapter, narrowed to the raw-object door - * this migration marker needs. Boundary: `Brainy.storage` is private, and - * this is the same reach-in the engine uses elsewhere for exactly this kind - * of engine-internal artifact. Returns null when the adapter has no - * raw-object door. - */ - private rawObjectStore(): { - readRawObject: (key: string) => Promise - writeRawObject: (key: string, value: unknown) => Promise - } | null { - const storage = (this.brain as unknown as { storage?: Record }).storage - if ( - storage && - typeof storage.readRawObject === 'function' && - typeof storage.writeRawObject === 'function' - ) { - return storage as unknown as { - readRawObject: (key: string) => Promise - writeRawObject: (key: string, value: unknown) => Promise - } - } - return null - } - - private async cleanupOldRoots(): Promise { - let removed = 0 + private async cleanupOldRoots(): Promise { try { // Find any old VFS roots with UUID-based IDs (not our fixed ID) const oldRoots = await this.brain.find({ @@ -533,7 +416,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { for (const duplicate of duplicates) { try { await this.brain.remove(duplicate.id) - removed++ console.log(`VFS: Deleted old root ${duplicate.id.substring(0, 8)}`) } catch (error) { console.warn(`VFS: Failed to delete old root ${duplicate.id}:`, error) @@ -546,7 +428,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Non-critical error - log and continue console.warn('VFS: Cleanup of old roots failed (non-critical):', error) } - return removed } /** diff --git a/tests/integration/count-ledger-identity-record.test.ts b/tests/integration/count-ledger-identity-record.test.ts deleted file mode 100644 index 1066213a..00000000 --- a/tests/integration/count-ledger-identity-record.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -/** - * @module tests/integration/count-ledger-identity-record - * @description THE COUNT LEDGER COUNTS RECORDS, NOT DIRECTORIES — and heals - * itself when it was derived the other way. - * - * Measured on a real store: the ALL-visibility ledger read 14,231 nouns - * against 14,056 identity records, and 72,729 verbs against 72,679 — exactly - * that store's 25 noun and 50 verb SCAR directories (empty `/` containers - * left by a pre-8.3.1 partial delete). Two copies of the SAME archive derived - * different numbers, because each had been persisted at a different moment - * under the old container rule. A downstream index heal subtracted against - * those denominators and reported remaining work that did not exist. - * - * The membership predicate is the IDENTITY RECORD (the metadata content leg). - * The scan already applies it; what is pinned here is that a ledger persisted - * under the OLD rule does not go on lying — it is corrected in the background, - * without blocking the open, and two copies of one archive agree. - */ - -import { describe, it, expect, afterEach } from 'vitest' -import { - mkdtempSync, - mkdirSync, - rmSync, - writeFileSync, - readFileSync, - cpSync, - existsSync -} from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { FileSystemStorage as FileSystemStorageClass } from '../../src/storage/adapters/fileSystemStorage.js' -import type { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' - -const NOUN_COUNT = 6 -const NOUN_SCARS = 3 -const VERB_SCARS = 2 -/** A REAL two-hex shard — the scan skips any directory that is not one. */ -const SCAR_SHARD = 'ab' - -function makeTempDir(): string { - return mkdtempSync(join(tmpdir(), 'brainy-count-ledger-')) -} - -/** The FileSystemStorage behind a brain. */ -function storageOf(brain: Brainy): FileSystemStorage { - return (brain as unknown as { storage: FileSystemStorage }).storage -} - -/** - * Add `count` empty `/` container directories under - * `entities///` — scars, exactly as a partial delete leaves them. - */ -function addScarContainers(dir: string, kind: 'nouns' | 'verbs', count: number): void { - for (let i = 0; i < count; i++) { - const id = `${SCAR_SHARD}5ca4000-0000-0000-0000-00000000000${i}` - mkdirSync(join(dir, 'entities', kind, SCAR_SHARD, id), { recursive: true }) - } -} - -/** Add one GHOST container: a `vectors.json` leg with no identity record. */ -function addGhostContainer(dir: string): void { - const id = `${SCAR_SHARD}9405700-0000-0000-0000-000000000000` - const idDir = join(dir, 'entities', 'nouns', SCAR_SHARD, id) - mkdirSync(idDir, { recursive: true }) - writeFileSync(join(idDir, 'vectors.json'), JSON.stringify({ id, vector: [0.1, 0.2] })) -} - -/** - * Rewrite counts.json into the LEGACY shape: ALL scalars inflated by the - * containers, and no `allCountsDerivedBy` stamp — exactly what a store carried - * when it was last written by a build that counted directories. - */ -function writeLegacyCountsLedger(dir: string, inflateNouns: number, inflateVerbs: number): void { - const file = join(dir, '_system', 'counts.json') - const counts = JSON.parse(readFileSync(file, 'utf-8')) - counts.totalNounCountAll = (counts.totalNounCountAll ?? 0) + inflateNouns - counts.totalVerbCountAll = (counts.totalVerbCountAll ?? 0) + inflateVerbs - delete counts.allCountsDerivedBy - delete counts.allCountsSuspect - writeFileSync(file, JSON.stringify(counts, null, 2)) -} - -/** - * Seed a store and return the HONEST ledger it holds when freshly written — - * the baseline the correction must return to. Read from the engine rather than - * hardcoded: an open creates its own rows (the VFS root), and a pin that - * asserts a literal would be pinning that incidental fact instead of the rule. - */ -async function seedStore(dir: string): Promise<{ nouns: number; verbs: number }> { - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const ids: string[] = [] - for (let i = 0; i < NOUN_COUNT; i++) { - ids.push(await brain.add({ data: `entity number ${i}`, type: NounType.Concept })) - } - await brain.relate({ from: ids[0], to: ids[1], type: 'relatedTo' } as never) - await brain.relate({ from: ids[1], to: ids[2], type: 'relatedTo' } as never) - await brain.flush() - const ledger = await storageOf(brain).getCanonicalCounts() - const baseline = { nouns: ledger.nouns.all, verbs: ledger.verbs.all } - await brain.close() - return baseline -} - -/** - * Make the ledger walk take `ms` so a test can observe the open completing - * WITHOUT it. Patches the prototype before any brain is constructed; returns - * the restore function. - */ -function slowTheLedgerWalk(ms: number): () => void { - const proto = ( - FileSystemStorageClass as unknown as { - prototype: Record Promise> - } - ).prototype - const real = proto.scanCanonicalEntities - proto.scanCanonicalEntities = async function slow(this: unknown, ...args: unknown[]) { - await new Promise((r) => setTimeout(r, ms)) - return real.apply(this, args) - } - return () => { proto.scanCanonicalEntities = real } -} - -describe('the canonical count ledger', () => { - const dirs: string[] = [] - - afterEach(() => { - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - }) - - function trackDir(): string { - const dir = makeTempDir() - dirs.push(dir) - return dir - } - - it('corrects a legacy container-rule ledger in the background, counting identity records', async () => { - const dir = trackDir() - const baseline = await seedStore(dir) - - // Scars and a ghost: containers with no identity record. - addScarContainers(dir, 'nouns', NOUN_SCARS) - addScarContainers(dir, 'verbs', VERB_SCARS) - addGhostContainer(dir) - // The ledger as the old rule left it: every container counted. - writeLegacyCountsLedger(dir, NOUN_SCARS + 1, VERB_SCARS) - - const restore = slowTheLedgerWalk(1_500) - let brain: Brainy - try { - const openStarted = Date.now() - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const openMs = Date.now() - openStarted - const storage = storageOf(brain) - - // THE OPEN DID NOT WAIT. Two walks of 1.5s each would have added 3s. - expect(openMs).toBeLessThan(2_500) - // And while it runs, the scalars say so instead of being subtracted against. - const atOpen = await storage.getCanonicalCounts() - expect(atOpen.suspect).toBe(true) - expect(atOpen.nouns.all).toBe(baseline.nouns + NOUN_SCARS + 1) - - await storage.whenCountLedgerSettled() - } finally { - restore() - } - const storage = storageOf(brain!) - - const healed = await storage.getCanonicalCounts() - expect(healed.nouns.all).toBe(baseline.nouns) - expect(healed.verbs.all).toBe(baseline.verbs) - expect(healed.suspect).toBe(false) - - // And it is PERSISTED with the honest stamp — the correction survives a - // reopen instead of being re-derived (or re-lost) every time. - await brain!.close() - const persisted = JSON.parse(readFileSync(join(dir, '_system', 'counts.json'), 'utf-8')) - expect(persisted.totalNounCountAll).toBe(baseline.nouns) - expect(persisted.totalVerbCountAll).toBe(baseline.verbs) - expect(persisted.allCountsDerivedBy).toBe('identity-record') - - const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await reopened.init() - const afterReopen = await storageOf(reopened).getCanonicalCounts() - expect(afterReopen.nouns.all).toBe(baseline.nouns) - expect(afterReopen.suspect).toBe(false) - await reopened.close() - }, 180_000) - - it('derives the same number from two copies of one archive', async () => { - const source = trackDir() - const baseline = await seedStore(source) - addScarContainers(source, 'nouns', NOUN_SCARS) - addGhostContainer(source) - - // Two copies of the SAME bytes, each carrying a DIFFERENT legacy ledger — - // the situation that made one archive report 14,231 and its twin 14,081. - const copyA = trackDir() - const copyB = trackDir() - cpSync(source, copyA, { recursive: true }) - cpSync(source, copyB, { recursive: true }) - writeLegacyCountsLedger(copyA, NOUN_SCARS + 1, 0) - writeLegacyCountsLedger(copyB, 1, 0) - - const derived: number[] = [] - for (const dir of [copyA, copyB]) { - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const storage = storageOf(brain) - await storage.whenCountLedgerSettled() - derived.push((await storage.getCanonicalCounts()).nouns.all) - await brain.close() - } - expect(derived[0]).toBe(derived[1]) - expect(derived[0]).toBe(baseline.nouns) - }, 180_000) - - it('writes counts.json atomically — no reader ever sees it empty', async () => { - const dir = trackDir() - await seedStore(dir) - const file = join(dir, '_system', 'counts.json') - expect(existsSync(file)).toBe(true) - - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - const storage = storageOf(brain) - - // Watch the ledger across many persists. A truncating write leaves a - // window in which the file parses as nothing; a temp+rename never does. - let sawUnparseable = 0 - const watcher = setInterval(() => { - try { - JSON.parse(readFileSync(file, 'utf-8')) - } catch { - sawUnparseable++ - } - }, 1) - for (let i = 0; i < 40; i++) { - await (storage as unknown as { persistCounts: () => Promise }).persistCounts() - } - clearInterval(watcher) - await brain.close() - expect(sawUnparseable).toBe(0) - }, 180_000) -}) diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts index 23cc0a15..deefc5e6 100644 --- a/tests/integration/entity-tree-stamp.test.ts +++ b/tests/integration/entity-tree-stamp.test.ts @@ -57,11 +57,7 @@ describe('entity-tree family stamp', () => { const invariants = (stamp.members as any).invariants expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the - // counter labelled the stamp with a generation a write in flight had merely - // claimed, so every crash inside a write window produced a spurious verdict - // at the next open (see the torn-tail pins below). - expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration()) + expect(stamp.sourceGeneration).toBe(brain.generation()) expect(stamp.generation).toBeGreaterThanOrEqual(1) }) @@ -116,96 +112,6 @@ describe('entity-tree family stamp', () => { expect(stillIncoherent).toEqual([]) }) - /** - * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's - * committed watermark — the durable shape a torn generation-log tail leaves - * behind (the stamp's fsync outlived the tail's). Fabricated rather than - * crash-produced so the pin is deterministic; the seeded-SIGKILL lane - * (`scripts/crash-consistency.mjs` in the engine repo) produces the same - * shape from a real abrupt termination. - */ - const fabricateTear = (ahead: number): FamilyStamp => { - const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`) - const zlib = require('node:zlib') - const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp - const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead } - fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn))) - return torn - } - - it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => { - for (let i = 0; i < 3; i++) - await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } }) - await brain.close() - const torn = fabricateTear(5) - - const warn = vi.spyOn(prodLog, 'warn') - const startedAt = Date.now() - brain = await open() - const openMs = Date.now() - startedAt - - const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) - expect(tearLines.length).toBe(1) - const said = String(tearLines[0][0]) - // Narrated PRECISELY: both generations, the file, and the named cure. - expect(said).toContain(`source generation ${torn.sourceGeneration}`) - expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`) - expect(said).toContain(ENTITY_TREE_STAMP_PATH) - expect(said).toContain('DEMOTED') - expect(said).toMatch(/repairIndex\(\)/) - // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear - // cannot turn an open into the 8-minute spin this class was reported as. - expect(openMs).toBeLessThan(30_000) - - // The store SERVES — a tear in a stamp never locks an owner out of the - // canonical tree the stamp merely describes. - expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3) - - // The demotion CONVERGED: the stamp now names committed truth, and the - // next open is quiet. A verdict that re-narrates every open is a wait - // wearing a different hat. - const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp - expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration()) - await brain.close() - const warn2 = vi.spyOn(prodLog, 'warn') - brain = await open() - expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([]) - }) - - it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => { - await brain.add({ data: 'ro', type: 'document', metadata: {} }) - await brain.close() - const torn = fabricateTear(3) - - const warn = vi.spyOn(prodLog, 'warn') - const reader: any = await Brainy.openReadOnly({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) - expect(tearLines.length).toBe(1) - const said = String(tearLines[0][0]) - expect(said).toContain('READ-ONLY') - expect(said).toContain('UNVERIFIED') - expect(said).toMatch(/repairIndex\(\)/) - await reader.close() - - // A reader never rewrites the store: read the bytes back off disk (not - // through a writer open, which would demote them) — the torn stamp is - // exactly as it was found. - const onDisk = JSON.parse( - require('node:zlib') - .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) - .toString('utf-8') - ) as FamilyStamp - expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration) - expect(onDisk.generation).toBe(torn.generation) - - brain = await open() - }) - it('the one verifier handles both member modes', () => { const rollup: FamilyStamp = { family: 'x', @@ -221,13 +127,7 @@ describe('entity-tree family stamp', () => { stampSource: 5, head: 9 }) - // AHEAD is its own class — a torn generation-log tail, never folded in - // with `incoherent`: the two have opposite cures (recount vs demote). - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({ - state: 'torn', - stampSource: 5, - head: 3 - }) + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) const enumerated: FamilyStamp = { diff --git a/tests/integration/filter-operator-conformance.test.ts b/tests/integration/filter-operator-conformance.test.ts deleted file mode 100644 index 628017e7..00000000 --- a/tests/integration/filter-operator-conformance.test.ts +++ /dev/null @@ -1,151 +0,0 @@ -/** - * @module tests/integration/filter-operator-conformance - * @description THE OPERATOR SET, AND WHAT EACH TOKEN DOES ON THE INDEX PATH. - * - * The contract-1 manifest splits this engine's `where` operators three ways — - * served, served-beyond-baseline, refused-by-name — and two engines must agree - * token for token. This lane is the machine-checkable side of that agreement: - * it asserts the EXACT accepted set (so a manifest can be diffed against a run - * rather than against prose), and it pins each of the three classes. - * - * The defect it closes: the metadata index's operator switch had no default - * case, so an operator it does not implement — `hasAll`, `noneOf`, `excludes`, - * `startsWith`, `endsWith`, `matches`, `length` — left the field's match set at - * its initial `[]` and `find()` returned an empty page. A documented operator, - * implemented in the in-memory matcher, answering silently wrong. Three of the - * seven are now SERVED on the index path; the other four are REFUSED BY NAME, - * because an equality/range posting index cannot evaluate a substring, a - * pattern or an array length without reading every row. - */ - -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync, readFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { contractVersion, BRAINY_CONTRACT_VERSION } from '../../src/utils/version.js' - -/** The accepted `where` value-operator tokens, as a sorted list. */ -const ACCEPTED_OPERATORS = [ - 'between', 'contains', 'endsWith', 'eq', 'equals', 'excludes', 'exists', - 'greaterThan', 'greaterThanOrEqual', 'gt', 'gte', 'hasAll', 'in', 'length', - 'lessThan', 'lessThanOrEqual', 'lt', 'lte', 'matches', 'missing', 'ne', - 'noneOf', 'notEquals', 'oneOf', 'startsWith' -] as const - -/** Served on the index path with exact posting-set semantics. */ -const SERVED_ON_INDEX = [ - 'between', 'contains', 'eq', 'equals', 'exists', 'greaterThan', - 'greaterThanOrEqual', 'gt', 'gte', 'in', 'lessThan', 'lessThanOrEqual', - 'lt', 'lte', 'missing', 'ne', 'notEquals', 'oneOf', - 'excludes', 'hasAll', 'noneOf' -] as const - -/** Accepted by name, refused by the index path — never answered empty. */ -const REFUSED_BY_INDEX = ['endsWith', 'length', 'matches', 'startsWith'] as const - -describe('filter operator conformance', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - }) - - async function seeded(): Promise { - const dir = mkdtempSync(join(tmpdir(), 'brainy-operators-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - await brain.add({ - data: 'a document about ferrets', - type: NounType.Document, - metadata: { tags: ['ferret', 'small', 'furry'], team: 'alpha' } - }) - await brain.add({ - data: 'a document about whales', - type: NounType.Document, - metadata: { tags: ['whale', 'large'], team: 'beta' } - }) - await brain.flush() - return brain - } - - it('the accepted operator set is exactly these 25 tokens', async () => { - const brain = await seeded() - // The engine names its own valid set in the refusal it raises for an - // unknown token — the honest place to read it from. - let message = '' - try { - await brain.find({ where: { team: { notIn: ['alpha'] } } } as never) - } catch (err) { - message = (err as Error).message - } - expect(message).toMatch(/Unknown filter operator "notIn"/) - const listed = (message.match(/Valid operators: ([^.]+)\./)?.[1] ?? '') - .split(',') - .map((t) => t.trim()) - .filter(Boolean) - .sort() - expect(listed).toEqual([...ACCEPTED_OPERATORS].sort()) - expect(listed.length).toBe(25) - // Four tokens a sibling manifest listed as served aliases are NOT in this - // engine's set and never have been — they raise INVALID_QUERY. - for (const absent of ['is', 'isNot', 'greaterEqual', 'lessEqual']) { - expect(listed).not.toContain(absent) - await expect( - brain.find({ where: { team: { [absent]: 'alpha' } } } as never) - ).rejects.toThrow(/Unknown filter operator/) - } - }, 120_000) - - it('serves hasAll, noneOf and excludes on the index path — never an empty page', async () => { - const brain = await seeded() - - const hasAll = await brain.find({ where: { tags: { hasAll: ['ferret', 'furry'] } } } as never) - expect(hasAll.length).toBe(1) - expect((hasAll[0] as { metadata?: Record }).metadata?.team).toBe('alpha') - - const noneOf = await brain.find({ where: { team: { noneOf: ['alpha'] } } } as never) - expect(noneOf.length).toBe(1) - expect((noneOf[0] as { metadata?: Record }).metadata?.team).toBe('beta') - - const excludes = await brain.find({ where: { tags: { excludes: 'whale' } } } as never) - expect(excludes.length).toBe(1) - expect((excludes[0] as { metadata?: Record }).metadata?.team).toBe('alpha') - - // hasAll with an operand nothing carries is EMPTY because it is empty — - // the honest zero, reached by evaluating the operator. - const none = await brain.find({ where: { tags: { hasAll: ['ferret', 'whale'] } } } as never) - expect(none.length).toBe(0) - }, 120_000) - - it('refuses the four index-unserveable operators BY NAME', async () => { - const brain = await seeded() - for (const op of REFUSED_BY_INDEX) { - const operand = op === 'length' ? 3 : 'a' - await expect( - brain.find({ where: { team: { [op]: operand } } } as never), - `${op} must refuse, never answer an empty page` - ).rejects.toThrow(new RegExp(`Filter operator "${op}".*cannot be served by the metadata index`, 's')) - } - }, 120_000) - - it('declares its contract version in code and in package.json', async () => { - expect(contractVersion()).toBe(1) - expect(BRAINY_CONTRACT_VERSION).toBe(1) - const pkg = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8')) - expect(pkg.brainyContract).toBe(contractVersion()) - }) - - it('the three classes partition the accepted set', () => { - expect([...SERVED_ON_INDEX, ...REFUSED_BY_INDEX].sort()).toEqual([...ACCEPTED_OPERATORS].sort()) - }) -}) diff --git a/tests/integration/flush-watcher-event-driven.test.ts b/tests/integration/flush-watcher-event-driven.test.ts deleted file mode 100644 index 4b2e80c4..00000000 --- a/tests/integration/flush-watcher-event-driven.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * @module tests/integration/flush-watcher-event-driven - * @description THE FLUSH-REQUEST WATCH IS EVENT-DRIVEN. - * - * It used to `readdir` the request directory every 500 ms, per brain, for the - * life of every writer — armed on every non-reader brain whether or not any - * inspector process existed. MEASURED on a production process holding 21 - * brains: 42 directory reads per second on a completely idle service, plus a - * stale-request GC pass on every one of them. - * - * The law: a request that has not been made is not a cause. The arrival itself - * wakes the watcher, so the request is seen SOONER than the poll saw it, and a - * slow safety sweep covers filesystems that drop watch events and the GC. - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' -import * as nodeFs from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -describe('the flush-request watcher', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - vi.restoreAllMocks() - }) - - async function openWriter(): Promise<{ brain: Brainy; dir: string }> { - const dir = mkdtempSync(join(tmpdir(), 'brainy-flush-watch-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - await brain.add({ data: 'a row', type: NounType.Concept }) - await brain.flush() - return { brain, dir } - } - - it('does not poll the request directory on an idle writer', async () => { - const { dir } = await openWriter() - const reqDir = join(dir, 'locks', '_flush_requests') - - // Count real reads of the request directory over a window far longer than - // the old 500ms poll (which would have made ~16 of them). - const realReaddir = nodeFs.promises.readdir - let requestDirReads = 0 - const spy = vi - .spyOn(nodeFs.promises, 'readdir') - .mockImplementation((async (p: unknown, ...rest: unknown[]) => { - if (String(p) === reqDir) requestDirReads++ - return (realReaddir as unknown as (...a: unknown[]) => Promise)(p, ...rest) - }) as typeof nodeFs.promises.readdir) - - await new Promise((r) => setTimeout(r, 8_000)) - spy.mockRestore() - - // The old poll: 500ms → ~16 reads. The safety sweep is 30s → 0 in this window. - expect(requestDirReads).toBeLessThanOrEqual(1) - }, 120_000) - - it('answers a request that arrives, without waiting for the sweep', async () => { - const { brain, dir } = await openWriter() - const reqDir = join(dir, 'locks', '_flush_requests') - const ackDir = join(dir, 'locks', '_flush_responses') - mkdirSync(reqDir, { recursive: true }) - - // Drop a request exactly as an out-of-process inspector does. - const id = 'test-request-0001' - writeFileSync(join(reqDir, `${id}.req`), JSON.stringify({ at: Date.now() })) - - // The ack must land far sooner than the 30s safety sweep. - const deadline = Date.now() + 10_000 - let acked = false - while (Date.now() < deadline) { - try { - const entries = await nodeFs.promises.readdir(ackDir) - if (entries.some((e) => e.startsWith(id))) { acked = true; break } - } catch { /* dir not created yet */ } - await new Promise((r) => setTimeout(r, 100)) - } - expect(acked, 'the watcher must answer an arriving request').toBe(true) - void brain - }, 120_000) -}) diff --git a/tests/integration/health-gate.test.ts b/tests/integration/health-gate.test.ts index f2952116..1c9a642d 100644 --- a/tests/integration/health-gate.test.ts +++ b/tests/integration/health-gate.test.ts @@ -188,7 +188,7 @@ describe('health gate (b) — unledgered is unknown: never blocks a serving prov describe('health gate (c) — degraded-but-serving narrates once per generation', () => { // PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the // degraded report lives on the family the read actually consults. - it('a heal:"repair" failure serves; narrates once per DISTINCT VERDICT, not once per generation bump', async () => { + it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => { const brain = new Brainy(createTestConfig({ silent: true })) await brain.init() brains.push(brain) @@ -197,13 +197,12 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' const internals = internalsOf(brain) let generation = 1 - let detail = 'counter drift' internals.metadataIndex.healthReport = () => healthReport({ provider: 'vector', serving: true, healthy: false, - invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail })], + invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail: 'counter drift' })], generation }) @@ -213,22 +212,11 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(1) // same verdict both times — one narration + expect(countNarrations()).toBe(1) // same generation both times — one narration - // THE DEDUPE KEY IS THE VERDICT, NOT THE COUNTER. A provider's `generation` - // bumps on every ledger mutation and every rebuild boundary, so keying the - // narration on it re-printed an UNCHANGED health line on every read that - // consulted a busy provider — and, in the other direction, let a provider - // that never bumped suppress a line whose reasons had genuinely changed. - // An unchanged verdict is silent however the counter moves: generation = 2 await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(1) // generation bumped, verdict identical — still silent - - // ...and a CHANGED verdict is always heard, bump or no bump: - detail = 'counter drift widened to 12 rows' - await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(2) // the reasons changed — a new narration + expect(countNarrations()).toBe(2) // generation bumped — a second narration delete internals.metadataIndex.healthReport }) diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index bb07268d..2bcee038 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,7 +16,6 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' -import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -58,107 +57,6 @@ describe('history repacking — the two-tier lifecycle', () => { } }) - /** - * THE HOLE, END TO END — the shape a real store carries. - * - * A forensic fixture was measured with generation directories 1..2503 - * present except for exactly one: 1416. Its fact-log segment already showed - * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration - * 1940 (531 generations) while recording only 530 facts. - * - * Before the fix, repacking such a store folded ACROSS that hole: the batch - * skipped 1416 (no readable delta) and the sealed segment declared a range - * spanning it anyway. The next open merged that declared range back into - * committedRanges, re-admitting 1416 as committed history, and every - * subsequent auto-compaction pass then asked the packed tier for a frame - * that was never written — producing, on EVERY run, the non-fatal narration - * - * Auto-compaction of generational history failed (non-fatal): generation - * N is inside sealed segment seg-....bgs's declared range but has no frame - * — packed history is damaged - * - * This pin removes a generation directory to make the same hole, then - * requires repack + reopen + compaction to complete cleanly. - */ - it('a missing generation directory does not poison the packed tier', async () => { - const dir = tempDir() - // `retention: 'all'` throughout: close() otherwise auto-compacts the - // history away, and this pin needs the cold generations still on disk so - // there is something to punch a hole in. The live window stays at its - // production default for the build phase, so nothing folds yet. - const archival = async (): Promise => { - const b = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - embeddingFunction: stub, - retention: 'all' - }) - await b.init() - return b - } - const brain = await archival() - - const id = await brain.add({ - data: 'holed-entity', - type: NounType.Document, - metadata: { v: 0 } - }) - // One flush per update: single-op writes coalesce inside a flush window, - // so a history deep enough to have a middle needs the windows separated. - for (let v = 1; v <= 12; v++) { - await brain.update({ id, metadata: { v } }) - await brain.flush() - } - await brain.close() - - // Punch the hole: delete ONE generation directory in the middle of the - // cold range, exactly as the real store presents it. - const genRoot = path.join(dir, '_generations') - const numeric = fs - .readdirSync(genRoot, { withFileTypes: true }) - .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) - .map((e) => Number(e.name)) - .sort((a, b) => a - b) - expect(numeric.length).toBeGreaterThan(6) - const victim = numeric[Math.floor(numeric.length / 2)] - fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) - - // Now shrink the live window and reopen. close() repacks automatically - // (brainy.ts phase 0b), so this is the production sequence exactly: a - // store with a hole in its history gets folded by ordinary housekeeping, - // with nobody asking for it. - ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 - const reopened = await archival() - const result = await reopened.repackHistory() - expect(result.foldedGenerations).toBeGreaterThan(0) - - const segDir = path.join(dir, SEGMENTS_PREFIX) - const manifestPath = ['manifest.json', 'manifest.json.gz'] - .map((f) => path.join(segDir, f)) - .find((p) => fs.existsSync(p))! - const raw = manifestPath.endsWith('.gz') - ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') - : fs.readFileSync(manifestPath, 'utf8') - const manifest = JSON.parse(raw) as { - segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> - } - - // THE LAW: every sealed segment declares exactly as many generations as it - // holds frames, and none of them spans the victim. - for (const s of manifest.segments) { - expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) - expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) - } - - await reopened.close() - - // And the pass that used to fail on every run now completes: reopen (which - // re-seeds committedRanges from the packed tier) then compact history. - const third = await openBrain(dir) - await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() - await third.close() - }) - it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/integration/hybrid-search-vfs.test.ts b/tests/integration/hybrid-search-vfs.test.ts index 219c4c83..c881fa97 100644 --- a/tests/integration/hybrid-search-vfs.test.ts +++ b/tests/integration/hybrid-search-vfs.test.ts @@ -21,16 +21,10 @@ describe('Hybrid Search with VFS', () => { testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`) fs.mkdirSync(testDir, { recursive: true }) - // `storage.path`, NOT the pre-8.0 `options.basePath` alias. That alias was - // removed at the 8.0 major and configures nothing, so this suite silently - // opened the DEFAULT store instead of its own temp directory — sharing one - // on-disk brain with every other run on the machine, accumulating tens of - // thousands of rows, and eventually failing on that shared store's graph - // adjacency rather than on anything it was written to test. brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', - path: testDir + options: { basePath: testDir } } }) await brain.init() diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts deleted file mode 100644 index b5c386cf..00000000 --- a/tests/integration/idle-costs-nothing.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -/** - * @module tests/integration/idle-costs-nothing - * @description AN IDLE BRAIN DOES NO WORK. - * - * A flush used to re-persist state identical to what was already on disk — - * the provider flushes, the watermark stamps, the generation counter, the - * entity-tree stamp, roughly 28 writes — because `flush()` never asked whether - * anything had changed. - * - * The field observation that started this: a production process holding 21 - * brains printed "All indexes flushed to disk in 216–601ms" per brain every - * ~35 seconds and idled at 1.26 cores, with no writes for ten minutes. This - * engine's cadence is WRITE-DRIVEN, so that observation is NOT explained by - * the cadence and is not claimed to be fixed here — what is fixed is that such - * a call now costs nothing. Who was calling flush() remains open. - * - * The laws pinned here: - * (a) the persistence cadence arms only on a write — a brain nobody writes - * to flushes zero times, however long it is left open; - * (b) a flush on a clean brain is O(1): no provider is called, nothing is - * written, and nothing is printed; - * (c) one write earns exactly one flush's worth of work, and no more. - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -/** Wait for any in-flight background flush, then let the idle timer settle. */ -async function drainCadence(brain: Brainy): Promise { - const inner = brain as unknown as { _persistBackgroundFlight: Promise | null } - await new Promise((r) => setTimeout(r, 3_000)) - await (inner._persistBackgroundFlight ?? Promise.resolve()) - await new Promise((r) => setTimeout(r, 500)) -} - -/** How long an idle brain is watched. Longer than the 30s flush interval. */ -const IDLE_WATCH_MS = 90_000 - -describe('an idle brain costs nothing', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - vi.restoreAllMocks() - }) - - async function openBrain(): Promise { - const dir = mkdtempSync(join(tmpdir(), 'brainy-idle-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - return brain - } - - it('flushes zero times over 90 idle seconds, and prints nothing', async () => { - const brain = await openBrain() - // One write and one flush to reach a clean, settled state — then nothing. - await brain.add({ data: 'the only write this test performs', type: NounType.Concept }) - await brain.flush() - - const logged: string[] = [] - const origLog = console.log - console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log - - // Watch the providers directly: a flush that runs calls all of them. - const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage - const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex - const graphIndex = (brain as unknown as { graphIndex: { flush: () => Promise } }).graphIndex - const countsSpy = vi.spyOn(storage, 'flushCounts') - const metadataSpy = vi.spyOn(metadataIndex, 'flush') - const graphSpy = vi.spyOn(graphIndex, 'flush') - - try { - await new Promise((r) => setTimeout(r, IDLE_WATCH_MS)) - } finally { - console.log = origLog - } - - // (a) + (b): nothing ran, nothing was said. - expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) - expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([]) - expect(countsSpy).not.toHaveBeenCalled() - expect(metadataSpy).not.toHaveBeenCalled() - expect(graphSpy).not.toHaveBeenCalled() - }, 180_000) - - it('an explicit flush over a clean brain calls no provider and prints nothing', async () => { - const brain = await openBrain() - await brain.add({ data: 'one write', type: NounType.Concept }) - await brain.flush() // this one does the work - - const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage - const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex - const countsSpy = vi.spyOn(storage, 'flushCounts') - const metadataSpy = vi.spyOn(metadataIndex, 'flush') - const logged: string[] = [] - const origLog = console.log - console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log - try { - await brain.flush() // ...and this one has nothing to do - await brain.flush() - await brain.flush() - } finally { - console.log = origLog - } - - expect(countsSpy).not.toHaveBeenCalled() - expect(metadataSpy).not.toHaveBeenCalled() - expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) - }, 120_000) - - it('one write earns exactly one flush', async () => { - const brain = await openBrain() - await brain.add({ data: 'first', type: NounType.Concept }) - await brain.flush() - // Settle: the first write also kicked a BACKGROUND flush, which is not - // awaited by design. Drain it before counting, or its provider calls land - // inside this test's window and are attributed to the write below. - await drainCadence(brain) - - // Count the flushes that actually RAN. (Provider spies cannot answer this: - // the storage adapter's own count ledger is write-through, so a write calls - // flushCounts() on its own account, with no flush involved.) - const logged: string[] = [] - const origLog = console.log - console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log - const ran = () => logged.filter((l) => /All indexes flushed to disk/.test(l)).length - try { - await brain.add({ data: 'second — this is the cause', type: NounType.Concept }) - await brain.flush() - expect(ran()).toBe(1) - - // No further cause, no further work. - await brain.flush() - await brain.flush() - expect(ran()).toBe(1) - } finally { - console.log = origLog - } - }, 120_000) -}) diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts index 7d09e893..cb19af4a 100644 --- a/tests/integration/ledger-derivation-identity.test.ts +++ b/tests/integration/ledger-derivation-identity.test.ts @@ -11,22 +11,14 @@ * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per * metadata content leg (`metadata.json` or `.json.gz`), the same test * `pruneOrphanedEntities()` uses, so the two agree by construction. - * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AND THE OPEN NEVER WALKS — a - * counts.json that carries the ALL scalars but no - * `allCountsDerivedBy: 'identity-record'` stamp predates this fix; - * loading it marks `suspect = true` from a single field read alone and - * warns exactly once naming the cause. The open itself never pays a - * directory walk. - * (2b) AND IT HEALS ITSELF. The ledger used to stay wrong for the life of the - * store, waiting for an operator to run `repairIndex()` — and a - * downstream index heal subtracted against the inflated denominator and - * reported work that did not exist. An honest derivation now runs in the - * BACKGROUND after the open (never blocking it, observable via - * `whenCountLedgerSettled()`), and refuses to stamp a number it derived - * while writes were landing. - * (3) THE SANCTIONED RECOUNT ALSO CLEARS IT — `repairIndex()` prunes the - * orphaned containers, recounts from the canonical metadata.json walk, - * and re-stamps — the ALL scalar is exact and the containers are gone. + * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that + * carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'` + * stamp predates this fix; loading it marks `suspect = true` from a + * single field read alone, never a directory walk, and warns exactly + * once naming the cause. + * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned + * containers, recounts from the canonical metadata.json walk, and + * re-stamps — suspect clears and the ALL scalar is exact again. * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store * with no counts.json stamps as it writes, so a brand-new store never * carries the legacy signature. @@ -123,43 +115,29 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p delete raw.allCountsDerivedBy fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) - const narrateSpy = vi.spyOn(prodLog, 'narrate') - // The derivation walks live on FileSystemStorage's prototype. Slow them - // deliberately: the OPEN must not wait for them, and on a two-row store a - // real walk finishes too fast to tell "not awaited" from "instant". - const proto = FileSystemStorage.prototype as any - const realScanEntities = proto.scanCanonicalEntities - let scanEntitiesCalls = 0 - proto.scanCanonicalEntities = async function slow(this: any, ...args: any[]) { - scanEntitiesCalls++ - await new Promise((r) => setTimeout(r, 1_200)) - return realScanEntities.apply(this, args) - } - try { - const openStarted = Date.now() - brain = await open() - const openMs = Date.now() - openStarted + const warnSpy = vi.spyOn(prodLog, 'warn') + // The two derivation walks live on FileSystemStorage's prototype — + // spying here (rather than on fs.promises.readdir globally) isolates + // THIS code path's behavior from unrelated walks elsewhere in the open + // sequence (a separate, pre-existing engine's own O(store) cost — not + // this fix's concern, and not something this pin should be sensitive + // to). Neither derivation method may run: the stamp check is a field + // read on the already-parsed counts.json, nothing more. + const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities') + const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount') - // THE OPEN DID NOT WALK: two slowed walks would have added 2.4s to it. - expect(openMs).toBeLessThan(2_000) + brain = await open() - // The stamp check itself is an O(1) field read, and it names the cause. - const atOpen = await brain.storage.getCanonicalCounts() - expect(atOpen.suspect).toBe(true) - const stampWarnings = narrateSpy.mock.calls.filter( - ([msg]: any[]) => String(msg).includes('legacy') && String(msg).includes('container rule') - ) - expect(stampWarnings.length).toBe(1) // exactly one, loud + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(true) - // ...and the honest derivation is already running behind the open. - await brain.storage.whenCountLedgerSettled() - expect(scanEntitiesCalls).toBeGreaterThan(0) - const healed = await brain.storage.getCanonicalCounts() - expect(healed.suspect).toBe(false) - expect(healed.nouns.all).toBe(raw.totalNounCountAll) - } finally { - proto.scanCanonicalEntities = realScanEntities - } + const stampWarnings = warnSpy.mock.calls.filter( + ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule') + ) + expect(stampWarnings.length).toBe(1) // exactly one, loud + + expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk + expect(scanVectoredSpy).not.toHaveBeenCalled() await brain.close() }) @@ -185,14 +163,7 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) brain = await open() - // Named suspect at load, then healed in the background WITHOUT the - // operator asking — the inflated container count is corrected to the - // identity-record population, though the orphaned containers themselves - // are still on disk (only repairIndex() removes those). - await brain.storage.whenCountLedgerSettled() - let healed = await brain.storage.getCanonicalCounts() - expect(healed.suspect).toBe(false) - expect(healed.nouns.all).toBe(realTotal) + expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load await brain.repairIndex() diff --git a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts deleted file mode 100644 index a46ad6a5..00000000 --- a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -/** - * @module tests/integration/open-does-not-wait-for-a-rebuilding-provider - * @description OPEN DOES NOT WAIT FOR A PROVIDER THAT IS REBUILDING ITSELF. - * - * Measured on a production store: a metadata provider that had to rebuild made - * `init()` pay the ENTIRE rebuild on the foreground — 641 seconds — with every - * other family idle behind it, because a provider reporting `serving: false` - * because it is BUSY BUILDING and one reporting `serving: false` because it is - * BROKEN were indistinguishable, and both were answered the same way: call - * `rebuild()`, and wait. - * - * The law: a provider that reports `rebuildInProgress()` owns its own rebuild. - * `init()` returns; every other family serves; THAT family's doors refuse by - * name, carrying the provider's own progress; and the doors open by themselves - * when the provider reports serving. Nothing is ever served empty. - */ - -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import type { ProviderRebuildProgress } from '../../src/utils/indexReadiness.js' - -/** How long the stub provider claims to be rebuilding. */ -const REBUILD_MS = 6_000 - -describe('a provider rebuilding itself never blocks open', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - }) - - it('init() returns in milliseconds, the family refuses by name, then answers', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-provider-')) - dirs.push(dir) - - // Seed a store so the open has something to (not) rebuild. - const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await seed.init() - await seed.add({ data: 'a row with a plain field', type: NounType.Concept, metadata: { kind: 'report' } }) - await seed.flush() - await seed.close() - - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - - // Dress the metadata index as a provider that is rebuilding ITSELF: not - // serving, and honest about why. `init()` wires the real index first, so - // the hooks are installed on the instance as soon as it exists — the gate - // reads them by feature detection, exactly as it would a native provider's. - const rebuildStartedAt = Date.now() - const stillRebuilding = () => Date.now() - rebuildStartedAt < REBUILD_MS - let rebuildCalls = 0 - - const inner = brain as unknown as { - metadataIndex: Record - setupIndex?: unknown - } - // Install on the prototype-free instance right after construction by - // patching the property the moment init() assigns it. - const install = (target: Record) => { - const realRebuild = target.rebuild as () => Promise - target.rebuildInProgress = (): ProviderRebuildProgress | null => - stillRebuilding() - ? { phase: 'metadata shadow build', done: 4_096, total: 14_056, startedAt: rebuildStartedAt } - : null - target.healthReport = () => ({ - provider: 'metadata', - healthy: !stillRebuilding(), - serving: !stillRebuilding(), - generation: 1, - invariants: [], - unledgered: [] - }) - target.rebuild = async () => { - rebuildCalls++ - return realRebuild.call(target) - } - } - - // init() constructs the metadata index; patch as soon as it exists, before - // the gate consults it. A microtask hop after the index is assigned is - // enough because the gate runs later in the same init. - const initPromise = (async () => { - const originalEnsure = (brain as unknown as { setupIndex?: () => unknown }).setupIndex - void originalEnsure - return brain.init() - })() - // Patch on the first tick the index exists. - const patcher = setInterval(() => { - if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { - install(inner.metadataIndex) - } - }, 1) - const startedAt = Date.now() - try { - await initPromise - } finally { - clearInterval(patcher) - } - const openMs = Date.now() - startedAt - - // If the patch did not land before the gate ran, this test proves nothing — - // say so loudly rather than passing vacuously. - expect( - typeof inner.metadataIndex.rebuildInProgress, - 'the stub provider was never installed — the test is vacuous' - ).toBe('function') - - // 1. The open did not wait out the rebuild. - expect(openMs).toBeLessThan(REBUILD_MS) - // 2. And brainy did not start a rebuild of its own on top of the provider's. - expect(rebuildCalls).toBe(0) - - // 3. The family's door refuses BY NAME, carrying the provider's progress. - let refusal: Error | null = null - try { - await brain.find({ where: { kind: 'report' } } as never) - } catch (err) { - refusal = err as Error - } - expect(refusal, 'a not-serving metadata family must refuse, never serve empty').not.toBeNull() - expect(refusal!.message).toMatch(/metadata shadow build/i) - expect(refusal!.message).toMatch(/4,096\/14,056/) - expect(refusal!.message).toMatch(/no action is needed/i) - - // 4. Other families keep serving — the brain is open. - const all = await brain.getNouns?.({ pagination: { limit: 1 } } as never) - expect(all ?? true).toBeTruthy() - - // 5. When the provider reports itself serving, the door opens by itself. - await new Promise((r) => setTimeout(r, REBUILD_MS)) - ;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false - await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined() - }, 180_000) - - it('a rebuilding provider reporting 0 entries is not a CRITICAL, and gets no second rebuild', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-critical-')) - dirs.push(dir) - const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await seed.init() - await seed.add({ data: 'a stored entity', type: NounType.Concept }) - await seed.flush() - await seed.close() - - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - - let rebuildCalls = 0 - const errors: string[] = [] - const origError = console.error - console.error = ((...a: unknown[]) => { errors.push(a.map(String).join(' ')) }) as typeof console.error - - const inner = brain as unknown as { metadataIndex: Record } - const patcher = setInterval(() => { - if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { - const target = inner.metadataIndex - target.rebuildInProgress = () => ({ phase: 'online metadata rebuild', startedAt: Date.now() }) - target.healthReport = () => ({ - provider: 'metadata', healthy: false, serving: false, - generation: 1, invariants: [], unledgered: [] - }) - // The shape the native engine now has: the index reports NOTHING while - // its rebuild runs online behind refusing doors. - target.getStats = async () => ({ totalEntries: 0 }) - target.rebuild = async () => { rebuildCalls++ } - } - }, 1) - try { - await brain.init() - } finally { - clearInterval(patcher) - console.error = origError - } - - expect( - typeof inner.metadataIndex.rebuildInProgress, - 'the stub provider was never installed — the test is vacuous' - ).toBe('function') - expect(errors.filter((l) => /CRITICAL: Metadata index has 0 entries/.test(l))).toEqual([]) - expect(rebuildCalls).toBe(0) - }, 180_000) -}) diff --git a/tests/integration/open-narration.test.ts b/tests/integration/open-narration.test.ts deleted file mode 100644 index 95aba9f1..00000000 --- a/tests/integration/open-narration.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * @module tests/integration/open-narration - * @description THE OPEN IS NEVER SILENT. - * - * A production service opened a 16 GB store and logged nothing at all for - * three minutes before its first line of work. Two defects made that possible - * and both are pinned here: - * - * 1. The phase breakdown was written to `prodLog.warn`, which every - * environment that looks like production clamps away. The narration - * channel (`prodLog.narrate`) is always visible, like `error`. - * 2. Nothing spoke DURING a phase — only after the whole open finished, if - * at all. A heartbeat now names the phase currently running and its - * elapsed wall while the open is still happening. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' -import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' - -function makeTempDir(): string { - return mkdtempSync(join(tmpdir(), 'brainy-open-narration-')) -} - -/** Capture console.warn lines emitted while `fn` runs. */ -async function captureWarn(fn: () => Promise): Promise<{ result: T; lines: string[] }> { - const lines: string[] = [] - const orig = console.warn - console.warn = ((...args: unknown[]) => { - lines.push(args.map((a) => String(a)).join(' ')) - }) as typeof console.warn - try { - return { result: await fn(), lines } - } finally { - console.warn = orig - } -} - -describe('open narration', () => { - let dir: string - let brain: Brainy | null = null - - beforeEach(() => { dir = makeTempDir() }) - - afterEach(async () => { - if (brain) { - try { await brain.close() } catch { /* already closed */ } - brain = null - } - try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } - }) - - it('narrate() survives the production log clamp that silences warn()', async () => { - // Exactly what isProductionEnvironment() does to the logger: level ERROR. - configureLogger({ level: LogLevel.ERROR }) - try { - const { lines } = await captureWarn(async () => { - prodLog.warn('[Brainy] this line is chatter and may be clamped') - prodLog.narrate('[Brainy] this line is why the database is slow') - }) - expect(lines.some((l) => /why the database is slow/.test(l))).toBe(true) - expect(lines.some((l) => /chatter/.test(l))).toBe(false) - } finally { - configureLogger({ level: LogLevel.INFO }) - } - }) - - it('names a slow phase as it ends, and heartbeats while it is still running', async () => { - // Seed a store, then reopen it with a deliberately slow storage init so - // the first phase crosses both the heartbeat and the narrate thresholds. - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - await brain.add({ data: 'seed entity', type: NounType.Concept }) - await brain.flush() - await brain.close() - brain = null - - const realInit = FileSystemStorage.prototype.init - FileSystemStorage.prototype.init = async function slowInit(this: FileSystemStorage) { - await new Promise((r) => setTimeout(r, 6_500)) - return realInit.call(this) - } - // Clamped to ERROR for the whole open: the narration must survive it. - configureLogger({ level: LogLevel.ERROR }) - try { - const { result, lines } = await captureWarn(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - return next - }) - brain = result - - // The heartbeat spoke DURING the phase, naming the phase and its cause. - const heartbeats = lines.filter((l) => /open: still in phase 1\/5 "storage-init"/.test(l)) - expect(heartbeats.length).toBeGreaterThanOrEqual(1) - expect(heartbeats[0]).toMatch(/loading its count ledger/) - - // And the phase named its own wall as it ended. - const ended = lines.filter((l) => /open: phase 1\/5 "storage-init" finished in \d+ms/.test(l)) - expect(ended.length).toBe(1) - - // The whole-open breakdown is on the same always-visible channel. - expect(lines.some((l) => /slow open: \d+ms total \(.*storage-init=/.test(l))).toBe(true) - } finally { - FileSystemStorage.prototype.init = realInit - configureLogger({ level: LogLevel.INFO }) - } - }, 120_000) -}) diff --git a/tests/integration/repair-narration.test.ts b/tests/integration/repair-narration.test.ts deleted file mode 100644 index 1fbe15e4..00000000 --- a/tests/integration/repair-narration.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -/** - * @module tests/integration/repair-narration - * @description A REPAIR NARRATES ITSELF, AND ITS RECEIPT SAYS WHERE THE TIME - * WENT. - * - * On a production store (14,647 nouns / 73,070 verbs) a `repairIndex()` ran - * for more than thirty minutes at roughly a full core with ZERO log lines - * between its start and its end, while the read doors kept serving. The - * operator could tell it was alive only from `top`, and could not tell which - * of its single-threaded walks it was inside. The law pinned here: - * - * - every phase announces itself BEFORE it works, naming what it is about - * to walk; - * - a heartbeat names the phase still running, at a bounded cadence, for as - * long as it runs; - * - every phase reports its own wall, and that wall is carried in the typed - * receipt (`RepairFamilyReport.durationMs`) — not only in a log line. - * - * All of it on the narration channel, which production's log clamp cannot - * silence (see tests/integration/open-narration.test.ts). - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' -import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' - -describe('repairIndex narration', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - configureLogger({ level: LogLevel.INFO }) - }) - - async function seededBrain(): Promise { - const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-narration-')) - dirs.push(dir) - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - for (let i = 0; i < 5; i++) { - await brain.add({ data: `repair subject ${i}`, type: NounType.Concept }) - } - await brain.flush() - return brain - } - - it('announces every phase, reports its wall, and carries that wall in the receipt', async () => { - const brain = await seededBrain() - const narrateSpy = vi.spyOn(prodLog, 'narrate') - - const report = await brain.repairIndex() - - const lines = narrateSpy.mock.calls.map(([m]) => String(m)) - - // Every family that ran has BOTH a start line and a finish line naming it. - for (const family of report.families) { - const started = lines.filter((l) => l.includes(`"${family.family}" started —`)) - const finished = lines.filter((l) => - new RegExp(`"${family.family}" finished in \\d+ms`).test(l) - ) - expect(finished.length, `no finish line for ${family.family}`).toBeGreaterThanOrEqual(1) - // A skipped family may be recorded without a start line only if it never - // began; every family that began must have announced itself. - if (family.checked) { - expect(started.length, `no start line for ${family.family}`).toBeGreaterThanOrEqual(1) - } - // THE RECEIPT CARRIES THE WALL — not only the log. - expect(typeof family.durationMs, `${family.family} has no durationMs`).toBe('number') - expect(family.durationMs).toBeGreaterThanOrEqual(0) - } - - // The closing line accounts for the whole repair, per family. - const closing = lines.filter((l) => /repairIndex complete in \d+ms/.test(l)) - expect(closing.length).toBe(1) - expect(closing[0]).toMatch(/@\d+ms/) - }, 180_000) - - it('heartbeats while a single phase is still walking', async () => { - const brain = await seededBrain() - - // Make one phase long enough to cross the heartbeat cadence, exactly as a - // multi-minute canonical walk does on a real store. - const proto = FileSystemStorage.prototype as unknown as Record< - string, - (...args: unknown[]) => Promise - > - const realPrune = proto.pruneOrphanedEntities - proto.pruneOrphanedEntities = async function slow(this: unknown, ...args: unknown[]) { - await new Promise((r) => setTimeout(r, 6_500)) - return realPrune.apply(this, args) - } - // Clamped as production clamps it: the narration must survive. - configureLogger({ level: LogLevel.ERROR }) - const narrateSpy = vi.spyOn(prodLog, 'narrate') - try { - await brain.repairIndex() - } finally { - proto.pruneOrphanedEntities = realPrune - } - - const beats = narrateSpy.mock.calls - .map(([m]) => String(m)) - .filter((l) => /repairIndex: still in "orphaned-containers" after \d+s/.test(l)) - expect(beats.length).toBeGreaterThanOrEqual(1) - expect(beats[0]).toMatch(/ghost\/scar containers/) - }, 180_000) -}) diff --git a/tests/integration/vfs-root-sweep-once.test.ts b/tests/integration/vfs-root-sweep-once.test.ts deleted file mode 100644 index cac59b70..00000000 --- a/tests/integration/vfs-root-sweep-once.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -/** - * @module tests/integration/vfs-root-sweep-once - * @description THE OLD-ROOT SWEEP RUNS ONCE PER STORE, NOT ONCE PER OPEN. - * - * The VFS bootstrap ran a filtered `find()` over the whole store on EVERY - * open, hunting for root directories created before the fixed root id existed - * — duplicates a store has either always had or never will. MEASURED on a - * 14,056-noun / 72,679-verb store: the phase it dominates cost 43–53 SECONDS - * of every open, warm reopens included. - * - * The law: a migration sweep is caused by the store's state, not by the clock - * or the open count. It runs behind the doors, records that it ran, and a - * store carrying that record never sweeps again. - */ - -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync, existsSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' -import { prodLog } from '../../src/utils/logger.js' - -describe('the VFS old-root sweep', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - afterEach(async () => { - for (const b of brains.splice(0)) { - try { await b.close() } catch { /* already closed */ } - } - for (const d of dirs.splice(0)) { - try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } - } - vi.restoreAllMocks() - }) - - async function open(dir: string): Promise { - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - brains.push(brain) - await brain.init() - return brain - } - - it('sweeps on the first open, records it, and never sweeps again', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-')) - dirs.push(dir) - - const sweepSpy = vi.spyOn( - VirtualFileSystem.prototype as unknown as { cleanupOldRoots: () => Promise }, - 'cleanupOldRoots' - ) - - const first = await open(dir) - await (first.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - expect(sweepSpy).toHaveBeenCalledTimes(1) - // The record is durable engine plumbing under _system/, like every other marker. - expect( - existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || - existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) - ).toBe(true) - - await first.add({ data: 'a row so the store is not trivially empty', type: NounType.Concept }) - await first.flush() - await first.close() - brains.splice(brains.indexOf(first), 1) - - sweepSpy.mockClear() - const second = await open(dir) - await (second.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - expect(sweepSpy).not.toHaveBeenCalled() - - await second.close() - brains.splice(brains.indexOf(second), 1) - - // ...and a third open, to prove it is the record and not a one-off. - sweepSpy.mockClear() - const third = await open(dir) - await (third.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - expect(sweepSpy).not.toHaveBeenCalled() - }, 180_000) - - it('a sweep that removes nothing on a fresh store says nothing', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-quiet-')) - dirs.push(dir) - - // The always-visible channel cannot be silenced by a log level, so a line - // on it has to earn its place. A fresh store's sweep finds no duplicate - // roots and costs a millisecond — it must do its work, record its marker, - // and stay quiet, or it trains operators to ignore the one channel that - // exists to be impossible to ignore. - const narrated: string[] = [] - const spy = vi.spyOn(prodLog, 'narrate').mockImplementation(((...args: unknown[]) => { - narrated.push(args.map((a) => String(a)).join(' ')) - }) as typeof prodLog.narrate) - - const brain = await open(dir) - await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - spy.mockRestore() - - expect(narrated.filter((l) => /old-root sweep/i.test(l))).toEqual([]) - // ...and it still did the work: the marker is recorded, so no future open sweeps. - expect( - existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || - existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) - ).toBe(true) - }, 180_000) - - it('the open does not wait for the sweep', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-')) - dirs.push(dir) - - const proto = VirtualFileSystem.prototype as unknown as Record< - string, - (...args: unknown[]) => Promise - > - const real = proto.cleanupOldRoots - proto.cleanupOldRoots = async function slow(this: unknown, ...args: unknown[]) { - await new Promise((r) => setTimeout(r, 4_000)) - return real.apply(this, args) - } - try { - const startedAt = Date.now() - const brain = await open(dir) - const openMs = Date.now() - startedAt - expect(openMs).toBeLessThan(3_000) - await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() - } finally { - proto.cleanupOldRoots = real - } - }, 180_000) -}) diff --git a/tests/integration/writer-lock-clean-close.test.ts b/tests/integration/writer-lock-clean-close.test.ts deleted file mode 100644 index 7d9c59d6..00000000 --- a/tests/integration/writer-lock-clean-close.test.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** - * @module tests/integration/writer-lock-clean-close - * @description THE CLEAN-CLOSE CONTRACT for the writer lock. - * - * A production restart made this lane necessary: a service stopped with exit - * code 0, having awaited `close()` on every pooled brain, and its next boot - * announced `[brainy] Overwriting stale writer lock … appears dead` for every - * store it owned. "The pid is gone" is equally true of an orderly restart and - * of a crash, so the message could not tell an operator which one they had. - * - * The contract pinned here: - * 1. A completed close leaves NO lock file and DOES leave a clean-close - * record; the next open says nothing about staleness. - * 2. The next lock claim CONSUMES that record — it may never outlive the - * lock generation it describes, or a later crash would read as clean. - * 3. A close whose durable steps FAIL still releases the lock (and still - * rethrows the failure). - * 4. A killed process (SIGKILL, no close at all) leaves the lock behind with - * NO record, and the next open says exactly that — crash, recovery ahead. - * 5. A host application with its own SIGTERM handler is never force-exited - * out from under its own shutdown by Brainy's handler. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' -import { spawn } from 'node:child_process' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' - -const REPO_ROOT = process.cwd() -const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') - -function makeTempDir(): string { - return mkdtempSync(join(tmpdir(), 'brainy-clean-close-')) -} - -/** - * Write a child script to disk and start it under tsx. A file (not `tsx -e`) - * because the eval form compiles to CommonJS, which has no top-level await. - * The script imports Brainy by ABSOLUTE path, so its own dependency - * resolution still happens from inside the repository. - */ -function startChild(dir: string, body: string): ReturnType { - const scriptPath = join(dir, 'child-process.mts') - writeFileSync(scriptPath, body) - // `detached` puts the child in its own process GROUP: tsx runs the script in - // a grandchild process, and only a group-wide signal reaches the process - // that actually holds the writer lock. - return spawn(TSX, [scriptPath], { - cwd: REPO_ROOT, - stdio: ['ignore', 'pipe', 'pipe'], - detached: true - }) -} - -/** Capture every console.warn/error line emitted while `fn` runs. */ -async function captureConsole(fn: () => Promise): Promise<{ result: T; lines: string[] }> { - const lines: string[] = [] - const origWarn = console.warn - const origError = console.error - const sink = (...args: unknown[]) => { - lines.push(args.map((a) => String(a)).join(' ')) - } - console.warn = sink as typeof console.warn - console.error = sink as typeof console.error - try { - const result = await fn() - return { result, lines } - } finally { - console.warn = origWarn - console.error = origError - } -} - -/** - * Run a child process that opens `dir`, writes one row, prints `READY`, and - * then waits forever. Resolves with the child once READY is seen. - */ -function spawnHoldingChild(dir: string): Promise<{ - child: ReturnType - output: () => string -}> { - const script = ` - import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) - await brain.init() - await brain.add({ data: 'row from the child', type: 'concept' }) - await brain.flush() - console.log('READY') - setInterval(() => {}, 1000) - ` - const child = startChild(dir, script) - let out = '' - child.stdout.on('data', (d) => { out += String(d) }) - child.stderr.on('data', (d) => { out += String(d) }) - return new Promise((resolvePromise, rejectPromise) => { - const timer = setTimeout(() => rejectPromise(new Error(`child never became READY:\n${out}`)), 120_000) - child.stdout.on('data', () => { - if (out.includes('READY')) { - clearTimeout(timer) - resolvePromise({ child, output: () => out }) - } - }) - child.on('exit', (code) => { - clearTimeout(timer) - if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`)) - }) - }) -} - -describe('writer lock — the clean-close contract', () => { - let dir: string - let brain: Brainy | null = null - - beforeEach(() => { dir = makeTempDir() }) - - afterEach(async () => { - if (brain) { - try { await brain.close() } catch { /* may already be closed */ } - brain = null - } - try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } - }) - - const lockPath = () => join(dir, 'locks', '_writer.lock') - const recordPath = () => join(dir, 'locks', '_writer.close') - - it('a completed close leaves no lock, leaves a record, and the reopen is silent about staleness', async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - expect(existsSync(lockPath())).toBe(true) - - await brain.add({ data: 'seed entity', type: NounType.Concept }) - await brain.flush() - await brain.close() - brain = null - - // 1. The lock is gone and the release is RECORDED. - expect(existsSync(lockPath())).toBe(false) - expect(existsSync(recordPath())).toBe(true) - const record = JSON.parse(readFileSync(recordPath(), 'utf-8')) - expect(record.pid).toBe(process.pid) - expect(typeof record.closedAt).toBe('string') - expect(typeof record.startedAt).toBe('string') - - // 2. The reopen says nothing about a stale lock. - const { result: reopened, lines } = await captureConsole(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - return next - }) - brain = reopened - expect(lines.filter((l) => /stale writer lock|appears dead/i.test(l))).toEqual([]) - - // 3. The claim CONSUMED the record — it must not outlive its lock generation. - expect(existsSync(recordPath())).toBe(false) - expect(existsSync(lockPath())).toBe(true) - }, 120_000) - - it('releases the writer lock even when a durable close step fails — and still rethrows', async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await brain.init() - await brain.add({ data: 'seed entity', type: NounType.Concept }) - await brain.flush() - expect(existsSync(lockPath())).toBe(true) - - // Inject a failure into a durable close step (the counts flush). - const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage - const boom = new Error('injected: counts flush failed during close') - storage.flushCounts = async () => { throw boom } - - await expect(brain.close()).rejects.toThrow(/injected: counts flush failed/) - brain = null - - // The lock is released regardless: a process on its way out holds nothing. - expect(existsSync(lockPath())).toBe(false) - - // And the next writer opens without a stale-lock verdict. - const { lines } = await captureConsole(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - await next.close() - }) - expect(lines.filter((l) => /appears dead/i.test(l))).toEqual([]) - }, 120_000) - - it('a SIGKILLed writer leaves the lock with no record, and the next open names the crash', async () => { - const { child } = await spawnHoldingChild(dir) - expect(existsSync(lockPath())).toBe(true) - expect(existsSync(recordPath())).toBe(false) - - // Group-wide: the lock holder is tsx's grandchild, not the spawned pid. - process.kill(-(child.pid as number), 'SIGKILL') - await new Promise((r) => child.on('exit', () => r())) - // The grandchild's death is asynchronous with the wrapper's exit event. - await new Promise((r) => setTimeout(r, 500)) - - // The lock survives the kill — a dead process releases nothing. - expect(existsSync(lockPath())).toBe(true) - expect(existsSync(recordPath())).toBe(false) - - const { lines } = await captureConsole(async () => { - const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) - await next.init() - await next.close() - }) - const verdict = lines.filter((l) => /Overwriting stale writer lock/i.test(l)) - expect(verdict.length).toBe(1) - // The verdict must name the ABSENT record and the recovery it implies — - // not merely that a pid is gone. - expect(verdict[0]).toMatch(/NO\s+clean-close record/i) - expect(verdict[0]).toMatch(/crash recovery/i) - }, 180_000) - - it("does not force-exit a host application that owns its own SIGTERM handler", async () => { - const script = ` - import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} - const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) - await brain.init() - await brain.add({ data: 'row from the host app', type: 'concept' }) - await brain.flush() - // The host application's OWN graceful shutdown, registered after Brainy's. - process.on('SIGTERM', async () => { - await new Promise((r) => setTimeout(r, 1500)) - console.log('APP-CLOSE-DONE') - process.exit(0) - }) - console.log('READY') - setInterval(() => {}, 1000) - ` - const child = startChild(dir, script) - let out = '' - child.stdout.on('data', (d) => { out += String(d) }) - child.stderr.on('data', (d) => { out += String(d) }) - await new Promise((r, reject) => { - const timer = setTimeout(() => reject(new Error(`child never became READY:\n${out}`)), 120_000) - child.stdout.on('data', () => { if (out.includes('READY')) { clearTimeout(timer); r() } }) - child.on('exit', () => { clearTimeout(timer); if (!out.includes('READY')) reject(new Error(`child died:\n${out}`)) }) - }) - - process.kill(-(child.pid as number), 'SIGTERM') - const code = await new Promise((r) => child.on('exit', (c) => r(c))) - expect(code).toBe(0) - // The host's own shutdown ran to completion — Brainy's handler did not - // exit the process out from under it. - expect(out).toContain('APP-CLOSE-DONE') - }, 180_000) -}) diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index f16e67b3..27ab85cb 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,119 +147,4 @@ describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) - - // ========================================================================== - // THE DENSITY LAW - // ========================================================================== - // - // A sealed segment declares a CONTIGUOUS range and every reader treats that - // range as containment. Folding a sparse batch therefore makes the segment - // claim generations it does not hold — and because `open()` merges declared - // ranges back into committedRanges, the hole is re-admitted as committed - // history and every later maintenance pass fails asking for a frame that was - // never written. That is the "generation N is inside sealed segment - // seg-....bgs's declared range but has no frame — packed history is damaged" - // narration seen on every run of the affected stores. - - it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => { - await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( - /not contiguous: 2 → 4 skips 1 generation/ - ) - // The refusal loses nothing: no segment was sealed, so the generations - // stay in the live tier and the next pass folds them correctly. - expect(store.segments()).toHaveLength(0) - expect(store.hasGeneration(1)).toBe(false) - }) - - it('a wider gap names how many generations it would have swallowed', async () => { - await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( - /not contiguous: 10 → 20 skips 9 generation\(s\)/ - ) - }) - - it('two contiguous runs folded separately declare honest ranges', async () => { - // What the caller now does instead of folding across the gap. - const a = await store.fold([gen(1), gen(2), gen(3)]) - const b = await store.fold([gen(7), gen(8)]) - expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) - expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) - // The gap is honestly outside the packed tier. - for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) - for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) - expect(await store.actualRanges()).toEqual([ - [1, 3], - [7, 8] - ]) - }) - - it('actualRanges() is exact and I/O-free for dense segments', async () => { - await store.fold([gen(1), gen(2)]) - await store.fold([gen(3), gen(4)]) - // Adjacent dense segments each contribute their declared range. - expect(await store.actualRanges()).toEqual([ - [1, 2], - [3, 4] - ]) - }) - - // ---- pre-existing damage: a store sealed by the old writer ---------------- - - /** - * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and - * sidecar for a contiguous run, then rewrite the manifest so the segment - * declares a wider range than the frames it holds. This reproduces on disk - * exactly what the affected stores carry, without needing the old code. - */ - const sealSparseSegment = async (): Promise => { - await store.fold([gen(1), gen(2), gen(3)]) - const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any - // Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become - // holes inside a sealed range. - manifest.segments[0].lastGeneration = 5 - await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) - } - - it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { - await sealSparseSegment() - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - - // The frames it really holds still serve, byte-faithfully. - expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) - expect(await reopened.readRecords(3)).toHaveLength(2) - - // The holes answer "not packed" instead of throwing. This is the fix for - // the wedge: the old reader threw here on EVERY maintenance pass. - expect(await reopened.readDelta(4)).toBeNull() - expect(await reopened.readRecords(5)).toBeNull() - }) - - it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { - await sealSparseSegment() - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - // Declared 1..5; actually holds 1..3. The store seeds committedRanges from - // THIS, so generations 4 and 5 never become committed history again. - expect(await reopened.actualRanges()).toEqual([[1, 3]]) - }) - - it('a DENSE segment missing a frame is still loud damage', async () => { - // The other side of the branch: when the manifest claims a complete span, - // a missing frame means the manifest and sidecar disagree — real damage, - // and it must not be quietly downgraded to "unpacked". - await store.fold([gen(1), gen(2), gen(3)]) - const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` - const raw = (await storage.readRawBytes(idxPath))! - const { decode, encode } = await import('@msgpack/msgpack') - const idx = decode(raw) as any - // Drop generation 2's entry while the manifest still declares 3 frames. - idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) - await storage.writeRawBytes(idxPath, encode(idx)) - - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - await expect(reopened.readDelta(2)).rejects.toThrow( - /manifest and the sidecar disagree; packed history is damaged/ - ) - }) })