diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index fec679a8..cdb2ab14 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -1,13 +1,7 @@ name: CI -# Branch pushes only — a release TAG deliberately does not re-run CI: the -# tagged commit's CI already ran on its branch push, and the runner is -# sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the -# tag's publish-source run and starve every release (observed on 8.10.3 and -# 9.0.0: the publish sat behind the tag's own redundant CI). on: push: - branches: ['**'] pull_request: jobs: diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-forge.yml similarity index 59% rename from .forgejo/workflows/publish-source.yml rename to .forgejo/workflows/publish-forge.yml index 8220bac9..fb7428bf 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-forge.yml @@ -1,12 +1,10 @@ -name: Publish (The Source) +name: Publish (forge) -# Datacenter-side publish to The Source (source.soulcraft.com — our -# self-hosted Forgejo; never call it "the forge", Forge is a different -# product), moved off the laptop: an 87MB tarball PUT over the laptop's WAN -# times out; The Source's own runner does it in seconds. +# Datacenter-side forge publish, moved off the laptop: an 87MB tarball PUT +# over the laptop's WAN times out; the forge's own runner does it in seconds. # scripts/release.sh tags + pushes, then polls this workflow's result (npm -# view against The Source's registry) before it ever touches the npmjs leg — -# see the "delegation contract" in scripts/release.sh's home-publish step. +# view against the forge registry) before it ever touches the npmjs leg — +# see the "delegation contract" in scripts/release.sh's forge-publish step. on: push: @@ -15,7 +13,7 @@ on: jobs: publish: - name: Publish to The Source registry + name: Publish to the forge registry runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -25,21 +23,20 @@ jobs: cache: npm - run: npm ci - run: npm run build - - name: Publish + readback-verify on The Source registry + - name: Publish + readback-verify on the forge registry env: - # The stored repo-settings secret keeps its historical name. FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }} run: | set -eo pipefail - SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" + FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" VERSION="$(node -p "require('./package.json').version")" - echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry..." + echo "Publishing @soulcraft/brainy@${VERSION} to the forge registry..." TMPRC="$(mktemp)" chmod 600 "$TMPRC" { - echo "@soulcraft:registry=${SOURCE_NPM_REG}" + echo "@soulcraft:registry=${FORGE_NPM_REG}" echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" } > "$TMPRC" @@ -59,12 +56,12 @@ jobs: rm -f "$TMPRC" if [ "$LANDED_VERSION" != "$VERSION" ]; then - echo "::error::Readback verify FAILED — The Source registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." + echo "::error::Readback verify FAILED — the forge registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." exit 1 fi if [ "$PUBLISH_OK" = true ]; then - echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry." + echo "Published and verified @soulcraft/brainy@${VERSION} on the forge registry." else - echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on the forge (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 5482bf3f..4cb9a405 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,38 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12) - -- fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) -- fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) -- fix(recovery): walks are healers — the typed/tolerant boundary redrawn where block-layer fault injection proved it belonged (0e3facf4) -- feat(log): log authority is the fleet default — adopt-at-open, oracle-gated; plus the power-cut throw-site cures and the loud torn-record contract (214c98b4) -- fix(durability): three block-layer power-loss findings from the first fault-injection box run — all cured, matrix 15/15 (67c606be) -- docs: RELEASES.md frames the release as 10.0.0 — honest major (log format v2 forward-only); comment wording cleanup (d1698fa5) -- fix(persistence): the idle flush trigger debounces under load — deferred to the floor, never dropped, never a flush-per-gap amplifier (a50726e6) -- feat(reprojection): the one doors-open machinery — budget-capped, yielding, foreground-preempted, atomic-swap; poison records quarantine typed (d1651f98) -- feat(embedding): deferred-embed markers become log records — the sidecar recovery path is deleted (b47787bb) -- feat(conformance): the golden-log fold oracle — encoder bytes and fold semantics pinned by content hash (c95bea88) -- feat(engine): the wiring wave — stamps ride every flush, provider generations, waitForIndexed, adopt-backfill, match-all serves (b53e6e89) -- feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data (b35d87a7) -- feat(log): v2 is the LIVE write format — envelope records with minted ints, genesis, sector seals; v1 readable forever (26c60251) -- docs: RELEASES.md — the unreleased write-path and lifecycle entry (consumer-facing draft; version set at cut) (73eb88d4) -- feat(temporal): as-of semantic recall joins the release contract — past vectors byte-exact, pinned (f7ca0d26) -- fix(log): acked writes survive power loss; rejected writes never silently commit — the kill-matrix goes 11/11 with zero .fails debt (13022c51) -- feat(plugin): every provider write surface carries the real committed generation (2d532684) -- feat(log): fact-log format v2 codec — record envelope, type registry, genesis, sector seals; fault-injection shim (34841074) -- feat(log): the guarded log-authority core — group-commit durable-at-ack, the per-brain switch, the verification oracle (65953097) -- docs: Path Registry rows DP6/DP8/MT5 flip to contracted+pinned — the deferred-embedding and atomic-update train landed with cited tests (9fda6d95) -- feat(embedding): MT5 — deferred embedding with durable markers; write acks never wait on a neural net (287384cf) -- fix(index): the flicker window dies — atomic in-place vector update; lazy open honors every provider's not-ready report; the Path Registry twin table (ebe06cdf) -- feat(persistence): the engine owns its flush cadence — callers never call flush() in hot paths again (3236a01b) -- fix(aggregation): the lifecycle cluster — flush stamps, behind-stamp catches up incrementally, the native rebuild finally gets invoked, deletes are never silently skipped (1dc861d2) -- perf(sort): ordered reads never do per-row storage round-trips — the 199-317s production scan class dies structurally (607b6b56) -- chore: the home registry is The Source, never 'the forge' — sweep the misnomer out of the release rail, workflows, and release notes (Forge is a different product; the stored CI secret keeps its historical name) (09352c2b) -- ci: tags stop triggering the CI matrix (redundant re-run of already-tested commits starved every release's publish run on the sequential runner) + release.sh forge poll window 20→50 min (c6c6ea6b) -- test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) - - ### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04) - docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) diff --git a/RELEASES.md b/RELEASES.md index df05a81e..ce7b5f99 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,76 +31,6 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v10.0.0 — 2026-08-10 (the write-path and lifecycle release) - -The theme: **writes ack fast and honestly, startup adopts instead of rebuilding, and -every query path serves, announces, or refuses — never silently degrades.** Ships as -one release together with the matching native accelerator version. - -**The storage-authority posture (the release's headline):** a NEW brain's default -is **durable-at-ack log authority** — the generation log is the source of truth, -every write acknowledgment is covered by a group-committed fsync, and crash -recovery is a replay of the log (an acked write survives power loss, proven by -fault-injection tests). An EXISTING brain adopts at its first open under 10.0.0, -gated by a verification oracle: the log is replayed and diffed against stored -truth record-by-record; curable gaps are backfilled; the brain flips only on a -green verdict and a brain that cannot verify stays on the previous posture and -says so loudly. The explicit opt-out is `logAuthority: 'defer'` in the config -(no automatic adoption; flip later with `adoptLogAuthority()`). - -**Why a major:** the generation log gains write format v2 — new segments carry typed, -versioned records with integrity seals. A 9.x build refuses a v2 segment with a clear -version-naming error (never a misread), which means **a brain written by 10.x cannot -be opened by 9.x**. Existing v1 history stays readable forever; upgrading requires no -migration and no data touch — the format moves forward only as you write. - -### New capabilities - -- **`deferEmbedding: true`** on `add()`/`update()`: the write acks at durability; the - embedding runs on a crash-safe background worker and the vector swaps in atomically. - The row is id/metadata-findable immediately; semantic recall converges when the embed - lands. Barriers and gauges: `awaitPendingEmbeds()`, `waitForIndexed('semantic')`, - `getIndexStatus().pendingEmbeds`. VFS file writes adopt this end to end — file-write - ack no longer waits on a neural net (measured ~50× faster serial writes on a - production-shaped corpus). -- **`waitForIndexed(path?, { generation?, timeoutMs? })`** — the one honest read - barrier for write-then-recall flows. Typed timeout error naming what was still - pending; never a silent partial wait. -- **Engine-owned persistence cadence** (`persistence.policy: 'auto'`, now the default): - the engine flushes on write-count/interval/idle triggers in the background, - single-flight. **Delete `flush()` calls from hot paths** — `flush()` remains as an - awaitable durability barrier. A hung flush can never block a write ack. -- **Time-travel recall contract**: `asOf(G).find()` serves vectors exactly as they - stood at G — a later update never leaks into an earlier pin; deleted rows mask; - beyond-head pins refuse typed. -- **Log-authority storage (opt-in, per brain)**: `verifyLogAuthority()` audits the - generation log against stored truth record-by-record and names every divergence; - `adoptLogAuthority()` flips a brain to log-authoritative storage only on a green - audit (self-healing curable divergences first), enabling durable-at-ack writes: - concurrent writers share one fsync and an acked write survives power loss, by - construction (crash-recovery replay is pinned by fault-injection tests). - -### Behaviour changes - -- **`find({ where: {} })` now serves match-all** (previously returned an empty result - silently — warm and cold). Same fix applies to count, streaming, and graph-scoped - seeding paths. -- **`removeMany({ where: {} })` now refuses with a typed error** — a match-all bulk - delete must be explicit, never inherited from an empty filter object. -- **Aggregations always answer**: state persists at every `flush()` (not only close), - an unclean exit reconciles incrementally instead of rescanning the store, and - deletes without a before-image flag a loud rescan instead of silently skipping. -- **Vector updates are atomic in place** — a row is never transiently absent from - search during an update (the "flicker" class is gone); type-only re-index of an - unchanged vector is a no-op. - -### Format note - -- The generation log gains **format v2** (typed, versioned records with integrity - seals). v1 segments remain readable forever; new segments write v2. Older brainy - builds refuse v2 segments with a clear version-naming error rather than misreading - them. Records reserve encryption fields for a future release — zero behaviour today. - ## v8.11.0 — 2026-07-27 (canonical enumeration mode for export — storage-walked, canon-complete) From a fleet data-migration program's requirement for whole-brain exports that are @@ -140,8 +70,8 @@ to the caller today. pre-existing meaning). **Migration-grade exports set `includeHidden: true`** — a complete-canon export must carry every visibility tier; consumer-facing exports leave it off. -- **Ops note (consumer-invisible): the release pipeline's home-registry publish (The - Source, source.soulcraft.com) now runs on CI**, triggered by the release tag, instead of PUTting the tarball from the laptop +- **Ops note (consumer-invisible): the release pipeline's forge-registry publish now runs + on CI**, triggered by the release tag, instead of PUTting the tarball from the laptop over WAN — no change to what gets published or how a consumer installs it. ## v9.0.0 — 2026-08-04 (the field-addressing law: your names and system.*, nothing in between) diff --git a/docs/path-registry.md b/docs/path-registry.md deleted file mode 100644 index 8a55004c..00000000 --- a/docs/path-registry.md +++ /dev/null @@ -1,86 +0,0 @@ -# The Path Registry — brainy's twin table - -The brainy half of the cross-engine Path Registry (the native accelerator -maintains the master list; IDs are shared and stable — `LC3`, `DP7`, … are -citable in commits, board rounds, release notes, and pins). Every row owes -five things: **service class** (INDEX-SERVED | BOUNDED-FALLBACK, announced | -TYPED REFUSAL), **latency budget** at 1k/10k/100k/1M (design bar: billions), -**lifecycle behavior**, **failure narration**, and a **test pin**. A path not -in this registry does not ship; an unregistered path is a red gate in the -scan audit. - -**The availability bar governing every row: user-visible downtime is -seconds, at restart only.** Migration, heal, compaction, embedding, and -retention run behind the doors — yielding, budget-capped, narrated. No path -may hold the doors while it does housekeeping. - -Status legend: ✅ contracted + pinned (test cited) · 🟡 partial (what holds -and what's missing, stated) · 🔴 owed (named, never silent). - -## LC — Lifecycle - -| ID | Brainy row | Status | -|----|-----------|--------| -| LC1 | Same-version reopen adopts everything: brain-format epoch match → zero rebuilds; aggregation state adopts by stamp; persisted indexes load. | ✅ `tests/unit/brainy/brain-format-handshake` + `migration-deference` (no-drift reopen never rebuilds) | -| LC2 | New empty brain: doors immediate. | ✅ exercised by every suite's setup | -| LC3 | Upgrade, same epoch: as LC1 — new code on unchanged formats owes nothing at open. | ✅ same pins as LC1 (epoch equality is the gate) | -| LC4 | Upgrade with epoch migration: TODAY brainy's epoch rebuild runs at open before doors. | 🔴 **owed — the sev's lockout row.** The doors-open-serving-old-structures design (yielding installments + atomic swap) lands measured-and-gated behind the service-class pair, per the lifecycle-sprint choreography. Acceptance case: the 9,184-row hours-lockout. | -| LC5 | Crash recovery: bounded, resumable, narrated. Aggregation leg ✅ (behind-stamp → incremental catch-up off the fact log + time-travel reconciliation, capped at 5,000 affected before an ANNOUNCED rescan). Vector/metadata legs ride epoch machinery (rebuild-from-canonical, narrated). | 🟡 aggregation pinned (`tests/integration/aggregation-lifecycle-catchup`); the rebuild legs are narrated but not yet installment-yielding (couples to LC4) | -| LC6 | Shutdown under load: close() drains the background flush flight, tears down cadence timers, runs ONE time-bounded compaction pass (~5s budget, resumable). | 🟡 pinned for flush/compaction (8.9.0 suites); SIGTERM drain budget not yet declared | -| LC7 | Rollback/downgrade: an N−1 build opening an N brain. | 🔴 owed — no declared read-compat window or typed refusal today (epoch mismatch triggers a rebuild, not a refusal; v2 nested-bag records read as a phantom user field on pre-law builds). Needs the declared-window contract. | -| LC8 | Relocatable brain directory: no absolute paths in artifacts; persist()/load() round-trips. | 🟡 persist/load pinned; byte-for-byte relocation depot cases are the pair gate's (shared corpora) | -| LC9 | Double-open: second writer gets a typed lock refusal (PID-liveness + heartbeat stale detection; `force` escape hatch logs loudly). | ✅ writer-lock suites (8.7.1) | - -## DP — Data plane - -| ID | Brainy row | Status | -|----|-----------|--------| -| DP1 | `get()` by id: direct storage read + hydrate. INDEX-SERVED (id-mapped). Milliseconds at every scale. | ✅ exercised everywhere; budget rides the pair speed table | -| DP2 | `find({query})`: embed + vector search. The embed dominates (native side owns the budget); JS HNSW serves the search leg. | 🟡 300ms-class p95 is the pair speed-table row; brainy-alone budget declared there | -| DP3 | Filtered/sorted list: column top-K when the field is columnized (INDEX-SERVED, zero canonical reads on the sorted page — value pairs come from ONE batched metadata-record pass); no-column fallback is BOUNDED-ANNOUNCED (one batch pass, announces once per field past 500 rows); unknown field → TYPED REFUSAL naming both candidate spellings. | ✅ `tests/unit/utils/metadataIndex-sort-callshape` (zero per-row reads, batch-only — latency-blind) + `metadataIndex-nested-orderby` (dotted keys serve-or-refuse) + `tests/integration/orderby-sort-bug` | -| DP4 | Aggregation/stats: ALWAYS answers. Write-time incremental; behind-stamp reconciles incrementally; genuine rebuilds go through the native parallel door or the paged JS walk; nothing ever latches off; before-image-less deletes flag a LOUD rescan, never a silent skip. | ✅ `tests/integration/aggregation-lifecycle-catchup` + `tests/unit/aggregation/aggregation-provider-rebuild` | -| DP5 | Graph traversal: `related()` paged via adjacency; whole-graph analytics carry declared cost. | 🟡 paged reads pinned; analytics cost-class declaration owed (rides VENUE-GRAPH-TRUST audit tool) | -| DP6 | Single write: ack at the canonical commit; visibility committed at ack (the atomic vector update kills the remove→add dark window); maintenance NEVER holds the ack (background flush cadence — THE ACK LAW pins: a hung flush cannot block a write, a hung EMBEDDER cannot block a write). | ✅ `tests/unit/brainy/persistence-policy` + `tests/unit/hnsw/update-item-atomic` + `tests/integration/deferred-embedding` | -| DP7 | Bulk ingest: sustained rate holds flat — per-write maintenance taxes must not grow with brain size (A4 removed caller-flush convoys; deferred embedding removes the per-write embed tax where opted). | 🟡 the decay-curve row is a pair speed-table RED GATE; brainy-alone sustained-rate run rides the same corpora | -| DP8 | Read under write pressure: no flicker window — a row that exists is never invisible to recall, even transiently (same-vector re-index is a no-op; changed-vector swaps in place, node never leaves the index; deferred updates serve the OLD vector until the atomic swap — stale-beats-absent). | ✅ brainy leg pinned (`tests/unit/hnsw/update-item-atomic` 9/9 + `deferred-embedding` stale-beats-absent); the symmetry property suite + runtime sentinels remain the B4 program | -| — | **As-of semantic recall** (time-travel vector search): `asOf(G).find()` serves the vectors AS THEY STOOD at G — byte-exact past vectors, tombstone masking, the deferred-embed cell honest on the vector leg, TYPED refusal beyond the head. Brainy-alone leg = ephemeral at-generation materialization (documented O(n log n at G) build, bounded); the at-scale leg rides the accelerated provider's as-of index. | ✅ `tests/integration/asof-semantic-recall` 4/4 (registry ID pending the master table's mint) | -| — | **The lazy-open gate honors EVERY provider's not-ready report** (a not-ready metadata provider can no longer latch the silent-empty state under `disableAutoRebuild`). | ✅ `tests/unit/brainy/lazy-notready-honor` | - -## MT — Maintenance (never in the door path) - -| ID | Brainy row | Status | -|----|-----------|--------| -| MT1 | Flush/checkpoint: ENGINE-OWNED cadence (write-count/interval/idle triggers, single-flight, background, loud on failure; callers never flush in hot paths; `flush()` stays as an awaitable barrier). | ✅ `tests/unit/brainy/persistence-policy` | -| MT2 | Compaction: never on flush (durability-only law, 8.9.0); close-time pass time-budgeted + resumable; explicit `compactHistory({timeBudgetMs})`. | ✅ 8.9.0 suites | -| MT3 | Index upkeep (mapper folds, delta promotion): native-side machinery; brainy's JS legs are small and synchronous-cheap. | 🟡 declared; yield audit rides the pair | -| MT4 | Heal/rebuild walks (`repairIndex`, backfill walks): paged; failure latches with cooldown; NOT yet yield-to-foreground installments. | 🔴 owed — the priority-isolation clause (couples to LC4; same choreography) | -| MT5 | Deferred embedding worker: ack at durability, durable pending markers (written BEFORE the commit — orphan-safe), crash-recovered at open via a bounded prefix listing, single-flight, 60s hang guard, `awaitPendingEmbeds()` barrier + `pendingEmbeds` gauge. VFS write paths adopt it end-to-end. | ✅ `tests/integration/deferred-embedding` 5/5 | -| MT6 | Retention/archival walks: retention `'all'` does nothing by design; bounded-retention reclaim is close-time/explicit only. | 🟡 8.9.0 behavior pinned; archival profile is the co-frozen D1+D3 unit | - -## FM — Failure modes - -| ID | Brainy row | Status | -|----|-----------|--------| -| FM1 | Disk full / IO error mid-op: transaction rollback + typed error; failed rollback → StoreInconsistentError quarantines writes until repairIndex(). | 🟡 rollback paths pinned; explicit disk-full depot case owed | -| FM2 | Memory pressure: query limits + reserved-memory config; unified cache eviction. | 🟡 declared budgets; cascade pin owed | -| FM3 | Torn/corrupt file on open: malformed brain-format marker → safe rebuild (never trusting a bad epoch); corrupt records surface loudly. | 🟡 marker pin ✅ (`brain-format-handshake`); broader quarantine is native-side | -| FM4 | Native module unavailable: plugin load failure is LOUD (version-coupling law throws on range mismatch — never silently version-drifted); JS engine serves with its own declared budgets, named as the active backend in op names. | ✅ `tests/unit/plugin-version-coupling` + op-name stamping | - -## FL — Fleet - -| ID | Brainy row | Status | -|----|-----------|--------| -| FL1 | Cold open on demand: LC1's adopt-everything open; warm() available for eager paths. | 🟡 open cost pinned at LC1; millisecond budget rides the speed table | -| FL2–FL4 | Boot storm / upgrade wave / isolation: fleet-layer policies over LC1/LC4 — engine leg = budgeted opens + LC4's behind-doors migration. | 🔴 owed with LC4 | -| FL5 | Brain as product object: create instant (LC2) · erase = `clear()` explicit + complete · export = portable-graph, canon-complete mode available. | ✅ clear-persistence + portable-graph + canonical-enumeration suites | - -## Status summary - -Contracted + pinned this train: **DP3, DP4, DP6, DP8(brainy leg), MT1, -MT5, LC5(aggregation), the lazy-open not-ready gate, LC1/LC3/LC9, FM4, -FL5** — each with the cited test. Owed, in production-risk order, all -coupled to the priority-isolation program the lifecycle sev opened: **LC4 -(doors-open migration), MT4 (yielding heals), LC7 (downgrade contract), -LC6 (SIGTERM budget), FL2–FL4, FM1/FM2 depot cases, B4 symmetry suite + -sentinels.** Rows move from owed to contracted only with a cited test — -none lands by prose. diff --git a/package-lock.json b/package-lock.json index 6193a630..af338ad8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.0.0", + "version": "9.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.0.0", + "version": "9.0.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 7b93cdd7..f4458a1d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.0.0", + "version": "9.0.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/release.sh b/scripts/release.sh index ce2d0882..b386e580 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -175,79 +175,76 @@ echo -e "${BLUE}7️⃣ Creating git tag v${NEW_VERSION}...${NC}" git tag -a "v${NEW_VERSION}" -m "Release v${NEW_VERSION}" echo -e "${GREEN}✅ Tag created${NC}\n" -# Step 9: Push to origin — The Source is the one home (ruled 2026-07-23; the +# Step 9: Push to origin — the forge is the one home (ruled 2026-07-23; the # old public GitHub repo is archived history, no longer part of any release). echo -e "${BLUE}8️⃣ Pushing to origin...${NC}" git push --follow-tags origin "$CURRENT_BRANCH" echo -e "${GREEN}✅ Pushed to origin${NC}\n" -# Step 10: The home publish (The Source, source.soulcraft.com) is CI's job -# now, not the laptop's — a tag push (just above) triggers -# .forgejo/workflows/publish-source.yml, which builds and publishes on The -# Source's own runner (datacenter-side: seconds, not the laptop's WAN timing -# out on an 87MB tarball PUT). The laptop holds no home-registry publish -# credential anymore; it only waits for CI's result before trusting the -# home/npmjs pair enough to publish the storefront leg. -SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" -SOURCE_POLL_INTERVAL_S=15 -SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml - # backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0); - # ci.yml no longer runs on tag pushes, but same-day branch pushes still queue ahead -echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}" -SOURCE_LANDED=false -for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do - LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") +# Step 10: Forge publish is CI's job now, not the laptop's — a tag push (just +# above) triggers .forgejo/workflows/publish-forge.yml, which builds and +# publishes on the forge's own runner (datacenter-side: seconds, not the +# laptop's WAN timing out on an 87MB tarball PUT). The laptop holds no forge +# publish credential anymore; it only waits for CI's result before trusting +# the forge/npmjs pair enough to publish the storefront leg. +FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" +FORGE_POLL_INTERVAL_S=15 +FORGE_POLL_MAX_ATTEMPTS=80 # 80 × 15s = 20 minutes — the runner is sequential; the publish run queues behind ci.yml jobs +echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to the forge registry (home)...${NC}" +FORGE_LANDED=false +for ((attempt = 1; attempt <= FORGE_POLL_MAX_ATTEMPTS; attempt++)); do + LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${FORGE_NPM_REG}" 2>/dev/null || echo "") if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then - SOURCE_LANDED=true + FORGE_LANDED=true break fi - echo -e "${YELLOW} … not yet on The Source (attempt ${attempt}/${SOURCE_POLL_MAX_ATTEMPTS}); retrying in ${SOURCE_POLL_INTERVAL_S}s${NC}" - sleep "$SOURCE_POLL_INTERVAL_S" + echo -e "${YELLOW} … not yet on the forge (attempt ${attempt}/${FORGE_POLL_MAX_ATTEMPTS}); retrying in ${FORGE_POLL_INTERVAL_S}s${NC}" + sleep "$FORGE_POLL_INTERVAL_S" done -if [ "$SOURCE_LANDED" = true ]; then - echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n" +if [ "$FORGE_LANDED" = true ]; then + echo -e "${GREEN}✅ CI published v${NEW_VERSION} to the forge${NC}\n" else - echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" + echo -e "${RED}❌ CI forge publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}" - echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}" + echo -e "${RED} forge registry after ${FORGE_POLL_MAX_ATTEMPTS} attempts, ${FORGE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}" exit 1 fi echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" # BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download -# the tarball The Source serves and publish that file, never a fresh local pack +# the tarball the forge serves and publish that file, never a fresh local pack # (a local rebuild can differ byte-wise, and the fleet verifies the pair by # shasum across registries). STOREFRONT_TMP="$(mktemp -d)" -(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null) -SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)" -echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}" -npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" +(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${FORGE_NPM_REG}" >/dev/null) +FORGE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)" +echo -e "${BLUE} forge artifact: $(sha256sum "$FORGE_TARBALL" | cut -d' ' -f1)${NC}" +npm publish "$FORGE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" rm -rf "$STOREFRONT_TMP" # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true # Verify the pair is byte-identical by registry-reported shasum — divergence here # means the storefront leg must be treated as failed, loudly. -SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable") +FORGE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${FORGE_NPM_REG}" 2>/dev/null || echo "forge-unavailable") NPMJS_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=https://registry.npmjs.org/" 2>/dev/null || echo "npmjs-unavailable") -if [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then +if [ "$FORGE_SHA" = "$NPMJS_SHA" ]; then echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n" else - echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} — investigate before announcing${NC}\n" + echo -e "${RED}❌ REGISTRY DIVERGENCE: forge shasum ${FORGE_SHA} != npmjs shasum ${NPMJS_SHA} — investigate before announcing${NC}\n" exit 1 fi -# Step 11: Release object on The Source (presentational — the tag, CHANGELOG, -# and RELEASES.md are the record; this just gives The Source's UI a release page). -echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" +# Step 11: Release object on the forge (presentational — the tag, CHANGELOG, +# and RELEASES.md are the record; this just gives the forge UI a release page). +echo -e "${BLUE}🔟 Creating forge release...${NC}" if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then - echo -e "${GREEN}✅ Release page created on The Source${NC}\n" + echo -e "${GREEN}✅ Forge release created${NC}\n" else - echo -e "${RED}⚠️ Release-page API call failed — tag + CHANGELOG remain the record; create the page via The Source's UI if wanted${NC}\n" + echo -e "${RED}⚠️ Forge release API call failed — tag + CHANGELOG remain the record; create the release page via the forge UI if wanted${NC}\n" fi else echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" @@ -270,4 +267,4 @@ echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" -echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" +echo -e "🏠 Forge: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index d3a1fd74..ca44ac8b 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -371,15 +371,6 @@ export class AggregationIndex { */ private pendingAdopt = new Set() - /** - * Aggregates adopted with a BEHIND stamp: name → the exact generation - * window `(from, to]` whose writes the adopted state has not seen. The - * owner (Brainy) drains this via {@link getPendingCatchUps} + - * {@link reconcileEntity} + {@link finishCatchUp} BEFORE serving queries — - * cost bounded by the window's affected entities, never store size. - */ - private pendingCatchUp = new Map() - /** * In-flight rescan targets. While a name has a staging map, ALL * contributions (the walk's and concurrent write hooks') land there instead @@ -446,47 +437,25 @@ export class AggregationIndex { } /** - * The adoption verdict for persisted state, against the store's committed - * watermark (SELF-ENGINE-LIFECYCLE-SPRINT ask (b) — behind-stamp is no - * longer a whole-store rescan): - * - * - `'adopt'` — stamp equals the watermark (clean), or the store has no - * watermark capability (hash-only adoption, the pre-stamp behavior). - * - `'catchup'` — stamp is BEHIND the watermark (an unclean exit after - * later writes, or a long-lived writer whose last flush predates recent - * writes). The state is exact AS OF its stamp, so it is adopted and the - * missing window `(stamp, committed]` is reconciled INCREMENTALLY per - * affected entity via time-travel reads — bounded by writes since the - * last flush, never by store size. The owner drains - * {@link getPendingCatchUps} before serving queries. - * - `'rescan'` — no stamp (pre-stamp state on a stamped store) or stamp - * AHEAD of the watermark (e.g. a fact-log truncation on a copied store - * pulled the watermark back): the state over-counts unverifiably; one - * exact rescan, said out loud. + * May this persisted state be ADOPTED? When the store exposes its committed + * watermark, the state's `sourceGeneration` must EQUAL it: behind means + * later writes are missing from the state (unclean shutdown); ahead means + * it counts writes that no longer exist (e.g. a fact-log truncation on a + * copied store pulled the watermark back). Either way: one exact rescan, + * said out loud — never a silent adopt. Stores without the capability (and + * pre-stamp state on them) fall back to hash-only adoption. */ - private stateAdoptionVerdict( - name: string, - stateData: unknown - ): 'adopt' | 'catchup' | 'rescan' { + private stateGenerationAdoptable(name: string, stateData: unknown): boolean { const committed = this.storage.committedGeneration?.() ?? null - if (committed === null) return 'adopt' + if (committed === null) return true const raw = (stateData as Record).sourceGeneration const stamped = typeof raw === 'number' ? raw : null - if (stamped === committed) return 'adopt' - if (stamped !== null && stamped < committed) { - this.pendingCatchUp.set(name, { from: stamped, to: committed }) - prodLog.info( - `[Aggregation] '${name}': persisted state is at generation ${stamped}, store is at ` + - `${committed} — adopting and reconciling the ${committed - stamped}-generation window ` + - `incrementally (no store rescan)` - ) - return 'catchup' - } + if (stamped === committed) return true prodLog.warn( `[Aggregation] '${name}': persisted state is at generation ${stamped ?? 'unstamped'} ` + `but the store's committed generation is ${committed} — rescanning instead of adopting` ) - return 'rescan' + return false } private async loadPersisted(): Promise { @@ -507,21 +476,20 @@ export class AggregationIndex { const appHash = this.definitionHashes.get(def.name) || '' if (appHash === savedHash && this.pendingAdopt.has(def.name)) { const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - const verdict = - stateData && stateData.groups - ? this.stateAdoptionVerdict(def.name, stateData) - : 'rescan' - if (verdict !== 'rescan') { + if ( + stateData && + stateData.groups && + this.stateGenerationAdoptable(def.name, stateData) + ) { const groupMap = new Map() - for (const group of stateData!.groups as AggregateGroupState[]) { + for (const group of stateData.groups as AggregateGroupState[]) { groupMap.set(serializeGroupKey(group.groupKey), group) } this.states.set(def.name, groupMap) this.pendingAdopt.delete(def.name) this.needsBackfill.delete(def.name) prodLog.info( - `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — ` + - (verdict === 'catchup' ? 'incremental catch-up pending' : 'no rescan') + `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — no rescan` ) } // No/invalid persisted state: stays in pendingAdopt and resolves @@ -536,23 +504,22 @@ export class AggregationIndex { const currentHash = hashDefinition(def) const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - const restoreVerdict = - stateData && stateData.groups && savedHash === currentHash - ? this.stateAdoptionVerdict(def.name, stateData) - : 'rescan' - if (restoreVerdict !== 'rescan') { - // Definition unchanged — load state (exact as of its stamp; a - // 'catchup' verdict reconciles the missing window incrementally). + if ( + stateData && + stateData.groups && + savedHash === currentHash && + this.stateGenerationAdoptable(def.name, stateData) + ) { + // Definition unchanged — load state const groupMap = new Map() - for (const group of stateData!.groups as AggregateGroupState[]) { + for (const group of stateData.groups as AggregateGroupState[]) { const serialized = serializeGroupKey(group.groupKey) groupMap.set(serialized, group) } this.states.set(def.name, groupMap) this.needsBackfill.delete(def.name) prodLog.info( - `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)` + - (restoreVerdict === 'catchup' ? ' — incremental catch-up pending' : '') + `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)` ) } else { // Definition changed or no saved state — start fresh and backfill from @@ -570,35 +537,15 @@ export class AggregationIndex { } } - // Restore native provider state from persistence — GATED by the same - // adoption verdict as caller-side state (the unconditional adopt was an - // asymmetry: a stale native blob restored over a moved store silently - // over/under-counted). 'adopt' restores; 'catchup' restores too (the - // incremental reconciliation drives the provider through - // incrementalUpdate over the exact missing window); 'rescan' SKIPS the - // blob — the flagged rebuild repopulates the provider from source. - // Legacy unstamped envelopes verdict as rescan, loudly, never silently. + // Restore native provider state from persistence if (this.nativeProvider?.restoreState) { const nativeState = await this.storage.getMetadata('__aggregation_native_state__') - const blob = - nativeState && typeof nativeState === 'string' - ? nativeState - : nativeState && typeof nativeState === 'object' && nativeState.data - ? (nativeState.data as string) - : null - if (blob !== null) { - const verdict = this.stateAdoptionVerdict( - '__native__', - nativeState && typeof nativeState === 'object' ? (nativeState as Record) : {} - ) - if (verdict === 'adopt' || verdict === 'catchup') { - this.nativeProvider.restoreState(blob) - } else { - prodLog.warn( - `[Aggregation] native provider state not adopted (verdict: ${verdict}) — ` + - `the flagged rescan repopulates the provider from source` - ) - } + if (nativeState && typeof nativeState === 'string') { + this.nativeProvider.restoreState(nativeState) + } else if (nativeState && typeof nativeState === 'object' && nativeState.data) { + // flush() persists `{ data: serializeState() }`, so `data` is the + // provider's serialized state string. + this.nativeProvider.restoreState(nativeState.data as string) } } } @@ -634,17 +581,12 @@ export class AggregationIndex { } } - // Persist native provider state — stamped. noteSourceGeneration lets the - // provider bake the committed watermark into its OWN envelope before - // serializing (so a native-side reopen can verify honesty without our - // wrapper); the wrapper carries the same stamp for OUR adoption verdict. + // Persist native provider state if (this.nativeProvider?.serializeState) { - const nativeGen = this.storage.committedGeneration?.() ?? null - if (nativeGen !== null) this.nativeProvider.noteSourceGeneration?.(nativeGen) const nativeState = this.nativeProvider.serializeState() await this.storage.saveMetadata( '__aggregation_native_state__', - nativeGen === null ? { data: nativeState } : { data: nativeState, sourceGeneration: nativeGen } + { data: nativeState } ) } @@ -805,119 +747,6 @@ export class AggregationIndex { this.dirty.add(name) } - // ============= Incremental Catch-Up (behind-stamp adoption) ============= - - /** The aggregates adopted behind the watermark, with their exact missing windows. */ - getPendingCatchUps(): Array<{ name: string; from: number; to: number }> { - return Array.from(this.pendingCatchUp, ([name, w]) => ({ name, ...w })) - } - - /** - * Reconcile ONE entity's contribution across a catch-up window using the - * same exact delta algebra the write-time hooks use: remove the - * contribution the adopted state counted (the entity AS OF the stamp), - * add the contribution it should count (AS OF the window's end). `null` - * on either side means the entity did not exist then. Composes exactly - * with live hooks because every application is a precise old/new pair — - * order between catch-up and post-window writes cannot drift the totals. - */ - reconcileEntity( - name: string, - id: string, - before: Record | null, - after: Record | null - ): void { - const def = this.definitions.get(name) - if (!def) return - if (before && after) { - if (isAggregateEntity(after)) return - const oldMatches = matchesSource(before, def.source) - const newMatches = matchesSource(after, def.source) - if (this.nativeProvider && (oldMatches || newMatches)) { - this.applyNativeResults( - name, - this.nativeProvider.incrementalUpdate(name, def, after, 'update', before) - ) - return - } - if (oldMatches) this.removeContribution(name, def, before) - if (newMatches) this.addContribution(name, def, after) - return - } - if (after) { - if (isAggregateEntity(after) || !matchesSource(after, def.source)) return - if (this.nativeProvider) { - this.applyNativeResults(name, this.nativeProvider.incrementalUpdate(name, def, after, 'add')) - } else { - this.addContribution(name, def, after) - } - return - } - if (before) { - if (isAggregateEntity(before) || !matchesSource(before, def.source)) return - if (this.nativeProvider) { - this.applyNativeResults(name, this.nativeProvider.incrementalUpdate(name, def, before, 'delete')) - } else { - this.removeContribution(name, def, before) - } - } - } - - /** Whether the native provider offers the parallel whole-rebuild path. */ - hasProviderRebuild(): boolean { - return typeof this.nativeProvider?.rebuildAggregate === 'function' - } - - /** The catch-up window for `name` is fully reconciled; state is current. */ - finishCatchUp(name: string): void { - this.pendingCatchUp.delete(name) - this.dirty.add(name) - } - - /** - * A catch-up could not complete (window unreadable, affected set over the - * bound, …): demote to an exact rescan, loudly — never serve un-reconciled. - */ - demoteCatchUpToBackfill(name: string, reason: string): void { - this.pendingCatchUp.delete(name) - this.needsBackfill.add(name) - prodLog.warn(`[Aggregation] '${name}': catch-up demoted to full rescan — ${reason}`) - } - - /** - * Rebuild an aggregate through the native provider's parallel path - * (SELF-ENGINE-LIFECYCLE-SPRINT ask (c) — `rebuildAggregate` existed on - * the provider contract but was never invoked; the JS walk fed - * per-entity FFI calls instead). Returns false when no provider rebuild - * exists — the caller streams the JS walk as before. - */ - rebuildWithProvider(name: string, entities: Array>): boolean { - const def = this.definitions.get(name) - if (!def || !this.nativeProvider?.rebuildAggregate) return false - const rebuilt = this.nativeProvider.rebuildAggregate( - def, - entities.filter(e => !isAggregateEntity(e) && matchesSource(e, def.source)) - ) - this.states.set(name, rebuilt) - this.backfillStaging.delete(name) - this.needsBackfill.delete(name) - this.dirty.add(name) - return true - } - - /** - * A write-path hook could not see the entity it needed (e.g. a delete - * whose before-image was unavailable): flag EVERY defined aggregate for - * an exact rescan, loudly — the counts must never silently drift - * (SELF-ENGINE-LIFECYCLE-SPRINT ask (d): the gated hook used to SKIP). - */ - flagAllForRescan(reason: string): void { - for (const name of this.definitions.keys()) this.needsBackfill.add(name) - prodLog.warn( - `[Aggregation] all ${this.definitions.size} aggregate(s) flagged for rescan — ${reason}` - ) - } - // ============= Write-Time Hooks ============= /** diff --git a/src/brainy.ts b/src/brainy.ts index a0f06931..600e4474 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -25,7 +25,7 @@ import { } from './storage/brainFormat.js' import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' -import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' +import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, cosineDistance, @@ -97,7 +97,6 @@ import { SaveVerbOperation, AddToGraphIndexOperation, RemoveFromVectorIndexOperation, - ReplaceInVectorIndexOperation, RemoveFromMetadataIndexOperation, RemoveFromGraphIndexOperation, UpdateNounMetadataOperation, @@ -161,8 +160,6 @@ import { AggregationIndex } from './aggregation/AggregationIndex.js' import { AggregateMaterializer } from './aggregation/materializer.js' import type { AggregateDefinition, AggregateQueryParams, AggregateResult } from './types/brainy.types.js' import type { MigrationProgress } from './types/brainy.types.js' -import type { IndexedProjectionPath, WaitForIndexedOptions } from './types/brainy.types.js' -import { WaitForIndexedTimeoutError } from './types/brainy.types.js' import { resolveJsHnswConfig, DEFAULT_RECALL } from './utils/recallPreset.js' import * as fs from 'node:fs' import * as os from 'node:os' @@ -178,7 +175,7 @@ import { type ImportResult } from './db/portableGraph.js' import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js' -import type { FactScanHandle, FactMarkerRecord } from './db/factLog.js' +import type { FactScanHandle } from './db/factLog.js' import { ENTITY_TREE_STAMP_PATH, readFamilyStamp, @@ -196,18 +193,6 @@ import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' import { assessIndexReadiness } from './utils/indexReadiness.js' -import { reconstructNounWrapper } from './db/factLog.js' -import { - readLogAuthority, - runLogCompletenessOracle, - flipToLogAuthority, - recordDigest, - nounEntityTruth, - LOG_AUTHORITY_PATH, - type LogAuthorityRecord, - type LogAuthorityStorage, - type OracleReport -} from './db/logAuthority.js' import { MemoryStorage } from './storage/adapters/memoryStorage.js' import type { CompactHistoryOptions, @@ -287,7 +272,6 @@ type ResolvedBrainyConfig = Required< | 'eagerEmbeddings' | 'migrationWaitTimeoutMs' | 'transactionBudgetFloorMs' - | 'persistence' > > & Pick< @@ -301,7 +285,6 @@ type ResolvedBrainyConfig = Required< | 'eagerEmbeddings' | 'migrationWaitTimeoutMs' | 'transactionBudgetFloorMs' - | 'persistence' > /** @@ -385,13 +368,6 @@ interface PlannedTransact { * rejected batch (CAS conflict, failed apply) emits nothing. */ changeEvents: PendingChangeEvent[] - /** - * V2 marker records riding the batch's ONE commit fact (e.g. the - * deferred-embedding pending markers) — same generation, same atomic - * append as the batch itself. A rejected batch appends no fact, so no - * marker outlives its write. - */ - markerRecords: FactMarkerRecord[] } /** @@ -543,24 +519,9 @@ export class Brainy implements BrainyInterface { * store has assigned the batch generation by then; for single-op writes it * reads the post-write watermark. The arrow body reads `generationStore` * lazily, so it is safe to define before `init()` assigns the store. - * Metadata/vector index writes use the bootstrap-honest twin - * {@link indexWriteGeneration} below. */ private readonly graphWriteGeneration = (): bigint => BigInt(this.generationStore.generation()) - /** - * The metadata/vector twin of {@link graphWriteGeneration}, honest about - * bootstrap: while generation stamping is inactive (init-time - * infrastructure writes, e.g. the VFS root, applied via - * `runWithoutGeneration`) there IS no commit generation — this resolves to - * `undefined` so a provider records "unstamped", never a fabricated 0. - * The graph thunk keeps its non-optional `bigint` contract (no graph - * writes occur during bootstrap). - */ - private readonly indexWriteGeneration = (): bigint | undefined => - this._generationStampingActive - ? BigInt(this.generationStore.generation()) - : undefined /** Lazily built host surface shared by every `Db` value of this brain. */ private _dbHost?: DbHost /** @@ -722,27 +683,6 @@ export class Brainy implements BrainyInterface { private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets private _aggregationIndex?: AggregationIndex // Incremental aggregation engine private _aggregationBackfillFlight: Promise | null = null // Single-flight backfill walk - private _aggregationCatchUpFlight: Promise | null = null // Single-flight behind-stamp catch-up - - // ENGINE-OWNED PERSISTENCE CADENCE (SELF-ENGINE-LIFECYCLE-SPRINT): - // write-count / interval / idle triggers → ONE background flush at a time. - // Write acks NEVER await it; a failed background flush is LOUD and re-armed. - private _persistDirtyWrites = 0 - private _persistLastFlushAt = Date.now() - private _persistIdleTimer: ReturnType | null = null - private _persistBackgroundFlight: Promise | null = null - - // DEFERRED EMBEDDING (MT5): pending markers are LOG RECORDS — an - // embed.pending record rides the deferred write's own commit fact and - // embed.landed rides the landing commit; this set is the in-memory - // fast-path index, rebuilt at open by folding the log's marker records. - // ONE background worker drains it. A crash can delay a vector, never - // lose one. - private _pendingEmbedIds = new Set() - private _embedWorkerFlight: Promise | null = null - - /** The stored log-authority switch, read once at open (default: tree). */ - private _logAuthority: LogAuthorityRecord = { authority: 'tree' } // A failed walk latches its error: retries within the cooldown rethrow it // instantly instead of re-walking, so a tight caller-side retry loop costs // one loud error per query, never a full store walk per query. @@ -1288,42 +1228,6 @@ export class Brainy implements BrainyInterface { this.graphIndex = graphIndex } - // Fact-log v2 mint seam: after-image records carry minted dense ints, - // and the ONE authority for those assignments is the metadata index's - // id mapper (append-only getOrAssign — a rebuilt mapper reproduces - // them exactly). The generation store cannot know the mapper, so the - // mint thunk is injected here, immediately after the index is ready; - // installing it is what flips the fact log's LIVE writes to the v2 - // segment format. A configuration whose mapper is unavailable throws - // at mint time — an int of 0 is never written. - this.generationStore.setIntMinter((kind, id) => { - const mapper = this.metadataIndex?.getIdMapper?.() - if (!mapper || typeof mapper.getOrAssign !== 'function') { - throw new Error( - `fact log v2: cannot mint the ${kind} int for ${id} — the metadata index's ` + - `id mapper is unavailable on this configuration; refusing to write an ` + - `after-image without a reproducible int` - ) - } - const minted = mapper.getOrAssign(id, undefined) - const asBigint = typeof minted === 'bigint' ? minted : BigInt(minted) - // THE RESERVED-ROOT EXEMPTION: the VFS root (the all-zeros UUID) is - // minted int 0 BY CONSTRUCTION at genesis on existing brains — the - // one legitimate zero in the id space. Zero for ANY other id is a - // corrupt mint and refuses. (Without this, every existing brain's - // adoption oracle false-flagged its own root and refused the flip.) - const isReservedRoot = - asBigint === 0n && id === '00000000-0000-0000-0000-000000000000' - if (asBigint < 0n || (asBigint === 0n && !isReservedRoot)) { - throw new Error( - `fact log v2: the id mapper minted ${asBigint} for ${kind} ${id} — ` + - `minted ints are positive (int 0 is reserved for the VFS root alone); ` + - `refusing to write` - ) - } - return asBigint - }) - // Eager cold-load (readiness contract). A provider that persists its // derived state exposes init?(): trigger the load NOW — AFTER // metadataIndex.init() above (the id-mapper is hydrated first, so a @@ -1381,20 +1285,7 @@ export class Brainy implements BrainyInterface { // gap for observability. for (const provider of this.versionedIndexProviders()) { const providerGen = provider.generation() - // Defensive finite-integer guard: committedGeneration() is validated - // at the store's open (torn artifacts discard, narrated) — but a - // RangeError here would kill the whole open, so the consumer guards - // too. A non-finite value narrates and skips the gap check (the - // provider's own replay contract still governs). - const committedRaw = this.generationStore.committedGeneration() - if (!Number.isSafeInteger(committedRaw) || committedRaw < 0) { - prodLog.warn( - `[Brainy] committed generation is non-integer (${String(committedRaw)}) at ` + - `init — torn-artifact survivor; skipping the provider replay-gap check` - ) - continue - } - const committed = BigInt(committedRaw) + const committed = BigInt(this.generationStore.committedGeneration()) if (providerGen < committed) { prodLog.info( `[Brainy] Versioned index provider is at generation ${providerGen} ` + @@ -1515,89 +1406,6 @@ export class Brainy implements BrainyInterface { this._generationStampingActive = true } - // LOG-AUTHORITY SWITCH (checked at open only). A STORED artifact - // always wins: an already-flipped brain runs durable-at-ack; an - // explicitly-recorded tree posture is honored. With NO artifact, the - // 10.0.0 FLEET DEFAULT is ADOPT-AT-OPEN (config logAuthority: - // 'adopt'): the verification oracle gates the flip — curable - // divergences are baseline-backfilled, the brain flips ONLY on green, - // and a brain that cannot go green STAYS tree-authoritative LOUDLY - // with the refusal recorded (cheap subsequent opens; an operator - // re-runs adoptLogAuthority() after fixing the divergence). - // 'defer' is the documented opt-out: no automatic adoption. - if (!this.isReadOnly) { - const storedArtifact = await this.storage - .readRawObject(LOG_AUTHORITY_PATH) - .catch(() => null) - const authority = await readLogAuthority(this.storage) - this._logAuthority = authority - if (authority.authority === 'log') { - this.generationStore.setLogDurability('at-ack') - prodLog.info('[Brainy] storage authority: generation log (durable-at-ack enabled)') - } else if ( - storedArtifact === null && - this.config.logAuthority === 'adopt' && - this.generationStore.getFactLog() !== null - ) { - try { - await this.adoptLogAuthority() - prodLog.info( - '[Brainy] storage authority adopted at open: generation log ' + - '(fleet default; oracle green; durable-at-ack enabled)' - ) - } catch (err) { - // The guarded ruling: a brain that cannot verify STAYS tree, - // loudly, with the refusal recorded so subsequent opens are - // cheap. Never a silent half-state; never a failed open. - const reason = (err as Error).message - prodLog.warn( - `[Brainy] log-authority adoption REFUSED at open — this brain stays ` + - `tree-authoritative until an operator resolves the divergence and ` + - `re-runs adoptLogAuthority(). Reason: ${reason}` - ) - try { - const refusal: LogAuthorityRecord = { - authority: 'tree', - adoptRefusal: { at: Date.now(), reason: reason.slice(0, 500) } - } - await this.storage.writeRawObject(LOG_AUTHORITY_PATH, refusal) - this._logAuthority = refusal - } catch { - // Unrecordable refusal = the next open retries the oracle — - // the conservative outcome. - } - } - } - } - - // MT5 crash recovery — REPLAY, NOT LISTING: the pending-embed markers - // live IN the generation log (embed.pending rides the deferred write's - // own fact; embed.landed rides the landing commit), so recovery folds - // the log's marker records back into the in-memory set — after the - // one-time bridge migrates any sidecar files a pre-log build left - // behind — and resumes the worker in the background. A crash between - // a deferred write's ack and its background embed DELAYED a vector; - // this is where it lands. - if (!this.isReadOnly) { - try { - await this.bridgeLegacyPendingEmbedSidecars() - await this.recoverPendingEmbedsFromLog() - if (this._pendingEmbedIds.size > 0) { - prodLog.info( - `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + - `session — resuming in the background` - ) - const t = setTimeout(() => this.kickEmbedWorker(), 0) - ;(t as { unref?: () => void }).unref?.() - } - } catch (err) { - prodLog.warn( - `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` + - `the log's markers remain durable; recovery retries next open` - ) - } - } - // Eager embedding initialization. // // Adaptive default (8.0): the WASM embedding engine eagerly initializes @@ -2019,466 +1827,12 @@ export class Brainy implements BrainyInterface { * deletes — the before-image + per-id-chain set. * @param run - The single-op's existing operation batch builder (the * `tx => {…}` body previously passed straight to `executeTransaction`). - * @param precommit - Optional CAS precondition, run under the commit mutex. - * @param pendingEvents - Change-feed events to stamp and emit post-commit. - * @param records - Optional v2 marker records (e.g. the deferred-embedding - * lifecycle markers) riding this write's commit fact — same generation, - * one atomic append. Refused on generation-less bootstrap writes. */ - /** - * Storage-root-relative prefix of the RETIRED sidecar pending-embed marker - * files (pre-log builds persisted one raw object per pending embed here). - * The markers live IN the generation log now (`embed.pending` / - * `embed.landed` records); this prefix survives ONLY for the one-time - * migration bridge ({@link bridgeLegacyPendingEmbedSidecars}) — no other - * code path writes, lists, or deletes it. - */ - private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/' - - /** - * @description Mark a deferred embed pending (MT5): the id joins the - * in-memory fast-path set and the returned `embed.pending` record is - * threaded onto the deferred write's OWN commit fact — same generation, - * same atomic append, and (in at-ack log durability) the same covering - * fsync as the write itself. The marker can never be orphaned from its - * write nor the write from its marker: a failed commit appends no fact, - * so no durable marker exists either (the in-memory entry is harmless - * and reaped by the worker). Recovery folds the marker back out of the - * log at open ({@link recoverPendingEmbedsFromLog}). - */ - private enqueuePendingEmbed(id: string): FactMarkerRecord { - this._pendingEmbedIds.add(id) - return { type: 'embed.pending', id, enqueuedAt: Date.now() } - } - - /** - * @description Clear a pending embed from the in-memory set. The DURABLE - * clear is the `embed.landed` record riding the landing commit's own fact - * (or, for a row deleted before its embed landed, the row's tombstone - * fact) — the recovery fold consumes those; nothing here touches storage. - * One honest residue: a pending row whose entity still exists but carries - * no data is reaped in memory only, so it re-folds at the next open and - * is re-reaped there — a bounded no-op, never a lost vector. - */ - private clearPendingEmbed(id: string): void { - this._pendingEmbedIds.delete(id) - } - - /** - * @description Rebuild the pending-embed set by REPLAYING the generation - * log's marker records (recovery = replay, not listing): `embed.pending` - * arms an id, `embed.landed` disarms it, and a noun tombstone disarms it - * too (a row deleted before its embed landed owes no vector). What - * survives the fold is exactly the set of acknowledged deferred writes - * whose vectors have not landed. - * - * BOUND (honest): no durable low-water mark exists for the earliest - * unconsumed pending, so the fold scans the log's committed facts from - * generation 1 — a sequential read of the log at open, O(log bytes). - * It is SKIPPED WHOLESALE when the log has never had a v2 tail - * ({@link FactLog.hasV2History} — v1 facts cannot carry marker records), - * so pre-cutover brains pay nothing; on a mixed log the scan still reads - * the v1 segments (a segment's format is only known from its bytes) but - * they fold to nothing, so the DECODE cost is bounded by v2 history. - * Storage without a fact log hosts no durable markers at all — the - * pending set is session-local there, matching that storage's overall - * durability posture. - */ - private async recoverPendingEmbedsFromLog(): Promise { - const log = this.generationStore.getFactLog() - if (!log || !log.hasV2History()) return - const scan = log.scanFacts({ fromGeneration: 1 }) - for await (const batch of scan.batches()) { - for (const fact of batch.facts) { - for (const record of fact.records ?? []) { - if (record.type === 'embed.pending') { - this._pendingEmbedIds.add(record.id) - } else if (record.type === 'embed.landed') { - this._pendingEmbedIds.delete(record.id) - } - } - for (const op of fact.ops) { - if (op.kind === 'noun' && op.record === null) { - this._pendingEmbedIds.delete(op.id) - } - } - } - } - } - - /** - * @description ONE-TIME LEGACY BRIDGE: a brain that deferred embeds under - * a pre-log build persisted one sidecar marker file per pending embed - * under {@link PENDING_EMBED_PREFIX}. At open, fold those ids into the - * pending set AND migrate them: commit ONE fact carrying their - * `embed.pending` records (the log is the markers' durable home now), - * then delete the sidecar files — in that order, so a crash between the - * two re-runs the bridge instead of losing a marker (a re-migrated - * duplicate folds idempotently; at worst an already-landed embed re-runs - * once — idempotent, never lost). Narrated loudly. Storage without a - * fact log keeps its sidecars in place (there is no log to migrate into) - * and folds them into memory only, exactly as loud. - */ - private async bridgeLegacyPendingEmbedSidecars(): Promise { - const markerPaths = await this.storage.listRawObjects(Brainy.PENDING_EMBED_PREFIX) - if (markerPaths.length === 0) return - const ids: string[] = [] - for (const path of markerPaths) { - const id = path.slice(path.lastIndexOf('/') + 1) - if (id) ids.push(id) - } - if (ids.length === 0) return - for (const id of ids) this._pendingEmbedIds.add(id) - if (!this.generationStore.getFactLog()) { - prodLog.warn( - `[Brainy] ${ids.length} legacy pending-embed sidecar marker(s) found, but this ` + - `storage hosts no fact log to migrate them into — folded into memory; the ` + - `sidecar files remain the durable recovery source on this configuration` - ) - return - } - const enqueuedAt = Date.now() - const markers: FactMarkerRecord[] = ids.map((id) => ({ - type: 'embed.pending', - id, - enqueuedAt - })) - // One migration commit: a zero-op fact carrying every legacy marker - // (empty-ops facts are legal; the records leg makes this one visible). - await this.generationStore.commitSingleOp({ - touched: {}, - records: markers, - execute: async () => {} - }) - for (const id of ids) { - await this.storage.deleteRawObject(`${Brainy.PENDING_EMBED_PREFIX}${id}`).catch(() => {}) - } - prodLog.info( - `[Brainy] migrated ${ids.length} legacy pending-embed sidecar marker(s) into the ` + - `generation log and removed the sidecar files (one-time bridge)` - ) - } - - /** - * @description Start (or skip into) the ONE deferred-embedding worker. - * Never awaited by write paths; failures are LOUD and markers survive for - * the next kick (next deferred write, or the next open's recovery). - */ - private kickEmbedWorker(): void { - if (this._embedWorkerFlight || this._pendingEmbedIds.size === 0 || this.isReadOnly) return - this._embedWorkerFlight = this.runEmbedWorker() - .catch((err) => { - prodLog.error( - `[Brainy] deferred-embed worker failed: ${(err as Error).message} — ` + - `markers retained; retries at the next deferred write or open` - ) - }) - .finally(() => { - this._embedWorkerFlight = null - if (this._pendingEmbedIds.size > 0) { - // New arrivals during the run: schedule (never recurse) the next pass. - const t = setTimeout(() => this.kickEmbedWorker(), 0) - ;(t as { unref?: () => void }).unref?.() - } - }) - } - - /** - * @description Drain the pending-embed set: embed each row's CURRENT data - * (a row updated again before its turn embeds the latest content — the - * marker set is idempotent per id) and swap the vector in ATOMICALLY - * (ReplaceInVectorIndex → the in-place update; the row is never absent - * from search). Orphans (row deleted, or no data) reap their markers. - */ - private async runEmbedWorker(): Promise { - const batch = Array.from(this._pendingEmbedIds) - for (const id of batch) { - try { - const entity = await this.get(id, { includeVectors: true }) - if (!entity || entity.data === undefined || entity.data === null) { - // Orphan reap: a deleted row's tombstone fact durably disarms the - // marker at the next recovery fold; a data-less-but-present row - // (edge case) re-folds and re-reaps — bounded, never a lost vector. - this.clearPendingEmbed(id) - continue - } - // Hang guard: a wedged embedder must not block every later pending - // embed forever — time out LOUDLY, keep the marker, move on. (A - // failure is retryable; an unbounded silent wait is the outlawed - // shape.) - const newVector = await Promise.race([ - this.embed(entity.data), - new Promise((_, reject) => { - const t = setTimeout( - () => reject(new Error('deferred embed timed out after 60s')), - 60_000 - ) - ;(t as { unref?: () => void }).unref?.() - }) - ]) - if (!this.dimensions) { - this.dimensions = newVector.length - } else if (newVector.length !== this.dimensions) { - throw new Error( - `deferred embed produced ${newVector.length} dimensions, store expects ${this.dimensions}` - ) - } - const oldVector = (entity.vector as number[] | undefined) ?? [] - // The landing commit's fact carries the embed.landed record (vector - // inline, per the v2 format) alongside the row's after-image — the - // durable "this pending is consumed" that recovery's fold reads. - await this.persistSingleOp( - { nouns: [id] }, - async (tx) => { - tx.addOperation( - new SaveNounOperation(this.storage, { - id, - vector: newVector, - connections: new Map(), - level: 0 - }) - ) - tx.addOperation( - new ReplaceInVectorIndexOperation(this.index, id, oldVector, newVector, this.indexWriteGeneration) - ) - }, - undefined, - undefined, - [{ type: 'embed.landed', id, vector: newVector }] - ) - this.clearPendingEmbed(id) - } catch (err) { - prodLog.warn( - `[Brainy] deferred embed for ${id} failed: ${(err as Error).message} — marker retained for retry` - ) - } - } - } - - /** - * @description The deferred-embedding BARRIER: resolves when every pending - * embed has landed (vector searchable) or been reaped. The eventual- - * vector-index contract's awaitable edge — tests and "must be searchable - * before I proceed" callers use this; nothing else ever needs to wait. - */ - public async awaitPendingEmbeds(): Promise { - while (this._pendingEmbedIds.size > 0 || this._embedWorkerFlight) { - this.kickEmbedWorker() - await (this._embedWorkerFlight ?? Promise.resolve()) - } - } - - /** The deferred-embedding backlog size (also on getIndexStatus().pendingEmbeds). */ - public pendingEmbedCount(): number { - return this._pendingEmbedIds.size - } - - /** - * THE READ BARRIER: wait until a projection — or every projection — has - * caught up to the CURRENT committed head, so a write-then-recall caller - * has ONE honest await instead of a sleep-and-hope. - * - * Legs: - * - `'semantic'` — waits for the deferred-embedding backlog to drain - * (delegates to {@link awaitPendingEmbeds}, which keeps working - * unchanged as this leg's engine). After it resolves, every previously - * acknowledged write is vector-searchable. - * - `'metadata'` / `'graph'` / `'aggregation'` — resolve IMMEDIATELY by - * design today: these projections are updated inside the write path, so - * by the time a write's promise resolves they already reflect it. Their - * asynchrony arrives with the log-authority read path; the door's shape - * freezes now so callers written against it keep working unchanged when - * those legs become real waits. - * - no argument — every projection at the head; today that reduces to the - * semantic drain (the only asynchronous projection in the current - * architecture). - * - * `opts.generation`: resolve as soon as the projection's watermark has - * reached that committed generation. The pending-embed set carries no - * generation stamps today, so the refinement is conservative — an empty - * backlog resolves immediately (the watermark is at the head, hence ≥ any - * committed generation); a non-empty backlog waits for the full drain, a - * SUPERSET of the requested wait, never a partial one. - * - * `opts.timeoutMs`: on expiry the promise REJECTS with - * {@link WaitForIndexedTimeoutError} — typed, carrying the leg and the - * still-pending embed count, and naming the gauge to check - * (`getIndexStatus().projections.semantic.pendingEmbeds`). Never a silent - * partial wait: a timeout means the projection has NOT caught up. - * - * @example Write, then semantically recall — no polling, no sleeps - * ```typescript - * const id = await brain.add({ - * data: 'quarterly revenue narrative', - * type: NounType.Document, - * deferEmbedding: true, - * metadata: { kind: 'report' } - * }) - * await brain.waitForIndexed('semantic') // the barrier: vector landed + indexed - * const hits = await brain.find({ query: 'revenue report', searchMode: 'semantic' }) - * // `id` is eligible to appear in `hits` — the recall is honest, not lucky. - * ``` - * - * @param path - The projection to wait on; omit to wait on all of them. - * @param opts - Optional `generation` watermark target and `timeoutMs` bound. - * @throws {WaitForIndexedTimeoutError} When `timeoutMs` expires before the - * projection catches up. - */ - public async waitForIndexed( - path?: IndexedProjectionPath, - opts?: WaitForIndexedOptions - ): Promise { - await this.ensureInitialized() - - // Synchronous projections: updated inside the write path today, so an - // acknowledged write is already reflected — resolve immediately BY - // DESIGN (honest, not a stub). When the log-authority read path makes - // these legs asynchronous, only this body changes; the door's shape is - // frozen now. - if (path === 'metadata' || path === 'graph' || path === 'aggregation') { - return - } - - // 'semantic' — or no-arg, which today reduces to it: the deferred-embed - // backlog is the only asynchronous projection in the current - // architecture. - - // Generation refinement (conservative — see JSDoc): an empty backlog - // means the semantic watermark is at the head, hence ≥ any committed G. - if (opts?.generation !== undefined && this._pendingEmbedIds.size === 0) { - return - } - - const timeoutMs = opts?.timeoutMs - const drained = this.awaitPendingEmbeds() - if (timeoutMs === undefined) { - return drained - } - - // Typed timeout: reject LOUDLY with the leg + the live backlog gauge. - // (`drained` never rejects — the worker catches its own failures — so - // abandoning it on timeout cannot leak an unhandled rejection; the - // backlog keeps draining in the background.) - let timer: ReturnType | undefined - try { - await Promise.race([ - drained, - new Promise((_, reject) => { - timer = setTimeout( - () => - reject( - new WaitForIndexedTimeoutError( - path ?? 'all', - timeoutMs, - this._pendingEmbedIds.size - ) - ), - timeoutMs - ) - ;(timer as { unref?: () => void }).unref?.() - }) - ]) - } finally { - if (timer !== undefined) clearTimeout(timer) - } - } - - /** - * @description The write-side persistence trigger (policy `'auto'`): count - * the committed write, kick a single-flight BACKGROUND flush when the - * write-count or interval threshold is crossed, and (re)arm the idle - * timer. Never awaited by the write path — the ack is already durable at - * the canonical layer; this schedules DERIVED-state persistence on the - * engine's own cadence (callers never call flush() in hot paths). - */ - private noteWriteForPersistence(): void { - const cfg = this.config.persistence - if (this.isReadOnly || cfg?.policy === 'manual') return - this._persistDirtyWrites++ - const every = cfg?.flushEveryWrites ?? 512 - const intervalMs = cfg?.flushIntervalMs ?? 30_000 - const idleMs = cfg?.flushOnIdleMs ?? 2_000 - - if ( - this._persistDirtyWrites >= every || - Date.now() - this._persistLastFlushAt >= intervalMs - ) { - this.kickBackgroundFlush('threshold') - } - - if (this._persistIdleTimer) clearTimeout(this._persistIdleTimer) - this.armIdleFlushTimer(idleMs, intervalMs) - } - - /** - * @description Arm the idle-flush timer — DEBOUNCED UNDER LOAD. The idle - * trigger exists to make a QUIET system durable fast; it must never add - * flush pressure to a BUSY one. When individual writes are slower than - * the idle window (a contended disk), every inter-write gap looks like - * "idle" and would fire a full flush per write — a measured 15-flush - * amplifier during 100 contended adds on a production-shaped box. The - * law: an idle fire landing within `intervalMs` of the last flush DEFERS - * (re-arms for the remaining interval) rather than flushing — deferred, - * never dropped, so a lone write on a then-quiet system still persists at - * the interval boundary without any further write arriving; a genuinely - * quiet system (last flush long past) flushes on idle exactly as before. - */ - private armIdleFlushTimer(idleMs: number, intervalMs: number, delayMs = idleMs): void { - // The idle-fire spacing floor: 10× the CONFIGURED idle window, capped by - // the interval — always derived from idleMs, never from a deferred - // re-arm delay (recomputing from the delay compounds into runaway - // deferral). Scales with intent — a caller configuring a tiny idle - // window gets fast idle-driven durability (small floor); default config - // (2s idle / 30s interval) gets a 20s floor, capping the contended-disk - // shape at ~1 idle flush per 20s instead of one per inter-write gap. - const floorMs = Math.min(intervalMs, idleMs * 10) - const timer = setTimeout(() => { - this._persistIdleTimer = null - if (this._persistDirtyWrites === 0) return - const sinceFlush = Date.now() - this._persistLastFlushAt - if (sinceFlush >= floorMs) { - this.kickBackgroundFlush('idle') - } else { - // Deferred, never dropped: land exactly at the floor boundary. - this.armIdleFlushTimer(idleMs, intervalMs, Math.max(idleMs, floorMs - sinceFlush)) - } - }, delayMs) - // Never hold the process open for a cadence timer. - ;(timer as { unref?: () => void }).unref?.() - this._persistIdleTimer = timer - } - - /** - * @description Start (or join) the ONE background flush. The dirty counter - * resets at kick time so writes landing during the flush re-accumulate - * toward the next trigger. A failure is LOUD and leaves the writes counted - * again — silence is not an option, and neither is a retry storm (the next - * trigger re-attempts). - */ - private kickBackgroundFlush(reason: 'threshold' | 'idle'): void { - if (this._persistBackgroundFlight) return - const counted = this._persistDirtyWrites - this._persistDirtyWrites = 0 - this._persistLastFlushAt = Date.now() - this._persistBackgroundFlight = this.flush() - .catch((err) => { - this._persistDirtyWrites += counted // re-arm the trigger honestly - prodLog.error( - `[Brainy] background flush (${reason}) FAILED: ${(err as Error).message} — ` + - `derived-state persistence retries at the next trigger; canonical data is unaffected` - ) - }) - .finally(() => { - this._persistBackgroundFlight = null - }) - } - private async persistSingleOp( touched: { nouns?: string[]; verbs?: string[] }, run: TransactionFunction, precommit?: (before: CommitBeforeImages) => void, - pendingEvents?: PendingChangeEvent[], - records?: FactMarkerRecord[] + pendingEvents?: PendingChangeEvent[] ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> { // Change-feed capture: when this write will emit, hold a reference to the // commit's before-images so `remove` events can carry the record's last @@ -2493,15 +1847,6 @@ export class Brainy implements BrainyInterface { : precommit if (!this._generationStampingActive) { - // Marker records ride a commit FACT — a generation-less bootstrap - // write has none to ride. No bootstrap path defers embeds today; - // refuse loudly rather than silently dropping a durable marker. - if (records && records.length > 0) { - throw new Error( - 'persistSingleOp: marker records require a generation-stamped commit — ' + - 'a bootstrap (generation-0) write cannot carry them' - ) - } // Init-time / infrastructure baseline write (e.g. the VFS root): apply // WITHOUT creating a generation. Generation 0 is the freshly-materialized // brain (bootstrap included); the first USER write is generation 1. @@ -2540,7 +1885,6 @@ export class Brainy implements BrainyInterface { receipt = await this.generationStore.commitSingleOp({ touched, precommit: captureAndCheck, - ...(records && records.length > 0 ? { records } : {}), execute: () => this.transactionManager.executeTransaction(run, { timeout: transactTimeoutBudget( @@ -2576,7 +1920,6 @@ export class Brainy implements BrainyInterface { ) } } - this.noteWriteForPersistence() return receipt } @@ -2752,26 +2095,15 @@ export class Brainy implements BrainyInterface { } // Get or compute vector - // MT5 deferred embedding: ack at durability with a stub vector and a - // pending marker riding the insert's OWN commit fact (same generation, - // one atomic append — a marker-less committed row, the silently-missing- - // vector shape, is structurally impossible). The background worker - // embeds + inserts. - const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed - ? [] - : params.vector || (await this.embed(params.data)) + const vector = params.vector || (await this.embed(params.data)) - // Ensure dimensions are set (a deferred-embed stub carries no dimension - // information — the worker's real vector goes through the same guard). - if (!deferringEmbed) { - if (!this.dimensions) { - this.dimensions = vector.length - } else if (vector.length !== this.dimensions) { - throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` - ) - } + // Ensure dimensions are set + if (!this.dimensions) { + this.dimensions = vector.length + } else if (vector.length !== this.dimensions) { + throw new Error( + `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` + ) } // Prepare metadata for storage: a v2 nested-bag record — engine fields @@ -2851,14 +2183,6 @@ export class Brainy implements BrainyInterface { } : undefined - // MT5: the pending marker RIDES the insert's own commit fact (same - // generation, one atomic append) — threaded to persistSingleOp below. - // A failed commit appends nothing, so no orphaned durable marker can - // exist; the in-memory entry is harmless and reaped by the worker. - const embedMarkers: FactMarkerRecord[] | undefined = deferringEmbed - ? [this.enqueuePendingEmbed(id)] - : undefined - const runInsert: TransactionFunction = async (tx) => { // Operation 1: Save metadata FIRST (TypeAwareStorage caching) // isNew=true: skip pre-read for rollback (entity doesn't exist yet) @@ -2877,18 +2201,14 @@ export class Brainy implements BrainyInterface { }, true) ) - // Operation 3: Add to HNSW index (after entity saved). A deferred - // embed has nothing to index yet — the worker's atomic update - // inserts the real vector. - if (!deferringEmbed) { - tx.addOperation( - new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration) - ) - } + // Operation 3: Add to HNSW index (after entity saved) + tx.addOperation( + new AddToVectorIndexOperation(this.index, id, vector) + ) // Operation 4: Add to metadata index tx.addOperation( - new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration) + new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) ) } @@ -2918,7 +2238,7 @@ export class Brainy implements BrainyInterface { const MAX_UPSERT_ATTEMPTS = 10 for (let attempt = 0; ; attempt++) { try { - await this.persistSingleOp({ nouns: [id] }, runInsert, insertPrecommit, addEvents, embedMarkers) + await this.persistSingleOp({ nouns: [id] }, runInsert, insertPrecommit, addEvents) break } catch (err) { if (!(err instanceof InsertPreconditionExistsSignal)) { @@ -2952,7 +2272,6 @@ export class Brainy implements BrainyInterface { this._aggregationIndex.onEntityAdded(id, entityForIndexing) } - if (deferringEmbed) this.kickEmbedWorker() return id } @@ -3438,11 +2757,6 @@ export class Brainy implements BrainyInterface { // new `data`); otherwise new `data` re-embeds; otherwise the existing // vector is kept. Any vector change re-indexes HNSW below. let vector = existing.vector - // MT5 deferred re-embedding: the OLD vector keeps serving semantic - // search — stale-but-present, never absent (the flicker law) — until - // the background worker embeds the new data and swaps it atomically. - const deferringEmbed = - params.deferEmbedding === true && Boolean(params.data) && !params.vector if (params.vector) { if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( @@ -3450,14 +2764,10 @@ export class Brainy implements BrainyInterface { ) } vector = params.vector - } else if (params.data && !deferringEmbed) { + } else if (params.data) { vector = await this.embed(params.data) } - // A deferred data change does NOT reindex now (the vector is unchanged; - // the worker's atomic swap carries the real reindex later). - const needsReindexing = Boolean( - (params.data && !deferringEmbed) || params.type || params.vector - ) + const needsReindexing = Boolean(params.data || params.type || params.vector) // Always update the noun with new metadata const newMetadata = params.merge !== false @@ -3544,12 +2854,6 @@ export class Brainy implements BrainyInterface { updatedMetadata._rev = authoritativeRev + 1 } - // MT5: the pending marker rides the update's own commit fact (same - // generation, one atomic append) — threaded to persistSingleOp below. - const embedMarkers: FactMarkerRecord[] | undefined = deferringEmbed - ? [this.enqueuePendingEmbed(params.id)] - : undefined - // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -3575,16 +2879,11 @@ export class Brainy implements BrainyInterface { level: 0 }) ) - // ONE atomic vector-index leg: the historical Remove→Add pair was - // two separately-awaited operations — between them the row was in - // NEITHER index (dark to semantic recall, visible to metadata - // reads). ReplaceInVectorIndexOperation goes through the provider's - // in-place updateItem when available (row never absent; an - // element-wise UNCHANGED vector — the type-only-update shape that - // flickered in production — is a pure no-op), else remove+add - // adjacent within the single op. tx.addOperation( - new ReplaceInVectorIndexOperation(this.index, params.id, existing.vector, vector, this.indexWriteGeneration) + new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector) + ) + tx.addOperation( + new AddToVectorIndexOperation(this.index, params.id, vector) ) } @@ -3614,10 +2913,10 @@ export class Brainy implements BrainyInterface { metadata: existing.metadata // CRITICAL: keep as nested 'metadata' property! } tx.addOperation( - new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration) + new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata) ) tx.addOperation( - new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration) + new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) ) }, casPrecommit, this._changeFeed.hasListeners ? [ @@ -3638,7 +2937,7 @@ export class Brainy implements BrainyInterface { } } ] - : undefined, embedMarkers) + : undefined) // Aggregation hook (outside transaction — derived data). `existing` is // the full get() view — every reserved field top-level — and must be @@ -3651,8 +2950,6 @@ export class Brainy implements BrainyInterface { existing as unknown as Record ) } - - if (deferringEmbed) this.kickEmbedWorker() } /** @@ -3702,14 +2999,14 @@ export class Brainy implements BrainyInterface { // Operation 1: Remove from vector index if (noun) { tx.addOperation( - new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration) + new RemoveFromVectorIndexOperation(this.index, id, noun.vector) ) } // Operation 2: Remove from metadata index if (metadata) { tx.addOperation( - new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration) + new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata) ) } @@ -3766,20 +3063,12 @@ export class Brainy implements BrainyInterface { // Aggregation hook (outside transaction — derived data). The view must // carry EVERY reserved field top-level (not a subset): a groupBy on // subtype/visibility/etc. otherwise decrements a nonexistent group and - // the real count never comes down. A delete whose before-image is - // unavailable can no longer SKIP the hook silently (the gated skip let - // counts drift upward forever) — it flags an exact rescan, loudly. - if (this._aggregationIndex) { - if (metadata) { - this._aggregationIndex.onEntityDeleted( - id, - this.entityForAggFromRawRecord(metadata as Record) - ) - } else { - this._aggregationIndex.flagAllForRescan( - `delete of ${id} carried no before-image metadata — contribution unknowable` - ) - } + // the real count never comes down. + if (this._aggregationIndex && metadata) { + this._aggregationIndex.onEntityDeleted( + id, + this.entityForAggFromRawRecord(metadata as Record) + ) } } @@ -3813,14 +3102,8 @@ export class Brainy implements BrainyInterface { verb: Pick & { sourceInt?: bigint; targetInt?: bigint } ): { sourceInt: bigint; targetInt: bigint } { const idMapper = this.metadataIndex.getIdMapper() - // Thread the write generation into any mint: a native mapper stamps the - // assignment record with the real watermark instead of a literal 0. - // Evaluated HERE (mint time) — at execute time inside a batch this is the - // in-flight commit generation; at plan time it is the pre-batch watermark - // (truthful: the mint happened before the batch committed). - const generation = this.indexWriteGeneration() - const sourceInt = BigInt(idMapper.getOrAssign(verb.sourceId, generation)) - const targetInt = BigInt(idMapper.getOrAssign(verb.targetId, generation)) + const sourceInt = BigInt(idMapper.getOrAssign(verb.sourceId)) + const targetInt = BigInt(idMapper.getOrAssign(verb.targetId)) verb.sourceInt = sourceInt verb.targetInt = targetInt return { sourceInt, targetInt } @@ -6518,24 +5801,6 @@ export class Brainy implements BrainyInterface { } } - // MATCH-ALL NORMALIZATION (served-or-refused law): an empty `where: {}` - // carries zero predicates, so it MUST route exactly like an absent `where`. - // Left in place it reads as "filter criteria present" below, builds an - // empty index filter, and `getIdsForFilter({})` answers `[]` by contract — - // a silent empty on a query that semantically matches everything (worst on - // a freshly reopened brain, where it masquerades as data loss; on the - // vector path it short-circuits `find({ query, where: {} })` to `[]`). - // Dropped here, ONCE, before branch selection: the query takes the - // unfiltered match-all branch below, which serves from truth-complete - // sources — a storage page bounded to the offset+limit window (never a - // full walk), or the column store's top-K sort when orderBy is present. - // Every delegating surface (Db pins via host.find, pagination.find, - // streaming.search, subgraph query seeding) inherits this routing. - if (params.where !== undefined && !whereConstrains(params.where)) { - const { where: _emptyWhere, ...rest } = params - params = rest as FindParams - } - // Zero-config validation (static import for performance) validateFindParams(params) @@ -7456,18 +6721,6 @@ export class Brainy implements BrainyInterface { `An empty selector would silently delete nothing — refusing.` ) } - // An empty `where: {}` carries zero predicates. find() serves it as - // MATCH-ALL (the served-or-refused law), which on this destructive path - // would silently become "delete up to `limit` arbitrary rows". A bulk - // delete of everything must be asked for explicitly (type selector, real - // predicates, or ids) — refuse the ambiguous shape loudly. - if (params.where && !params.ids && !params.type && !whereConstrains(params.where)) { - throw new Error( - `removeMany() received where: {} — an empty filter matches EVERYTHING, ` + - `and a match-all bulk delete must be explicit. Pass real predicates, ` + - `a { type }, or { ids }; to clear the store use clear().` - ) - } if (params.ids && params.ids.length === 0) { throw new Error( `removeMany() received ids: [] — an empty id list deletes nothing. ` + @@ -7562,13 +6815,13 @@ export class Brainy implements BrainyInterface { // Add delete operations to transaction if (noun) { tx.addOperation( - new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration) + new RemoveFromVectorIndexOperation(this.index, id, noun.vector) ) } if (metadata) { tx.addOperation( - new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration) + new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata) ) } @@ -8186,156 +7439,6 @@ export class Brainy implements BrainyInterface { return this.generationStore?.getFactLog()?.segmentPaths(options) ?? [] } - /** - * @description This brain's storage authority as read at open: `'tree'` - * (the canonical record tree is authoritative; the generation log is a - * complete dual-written journal — the default) or `'log'` (the log is - * authoritative; single-op acks are durable-at-ack). See - * {@link adoptLogAuthority} for the guarded flip. - */ - logAuthority(): LogAuthorityRecord { - return { ...this._logAuthority } - } - - /** - * @description Run the log-completeness VERIFICATION ORACLE (read-only): - * replay the generation log and diff the resulting per-id state against - * the canonical tree. Green = the log exactly reproduces canonical truth. - * Red NAMES every divergence class — `pre-log-record` rows (canonical - * history the log never saw) need a baseline backfill before this brain - * can ever flip. Safe at any time; walks are paged and memory-bounded - * (digests, never bodies). - */ - async verifyLogAuthority(): Promise { - await this.ensureInitialized() - return runLogCompletenessOracle({ - storage: this.storage as unknown as LogAuthorityStorage, - scanFacts: () => this.scanFacts(), - // Both sides normalize to ENTITY TRUTH before digesting: canonical - // wrappers denormalize HNSW residue (connections/level) the log never - // carries — digesting it would fake state-differs on any nonzero-level - // node (the residue has its own rebuild path; it is not entity state). - canonicalNounDigest: async (id: string) => { - const raw = await this.storage.readNounRaw(id) - if (raw.metadata === null && raw.vector === null) return null - return recordDigest(nounEntityTruth({ metadata: raw.metadata, vector: raw.vector })) - }, - factRecordDigest: (record: unknown) => - recordDigest(nounEntityTruth(record as { metadata: unknown; vector: unknown })) - }) - } - - /** - * @description THE GUARDED FLIP: run the oracle; on GREEN, persist the - * authority switch and enable durable-at-ack immediately (the rest of - * log-authoritative behavior engages at the next open — the switch is - * checked-at-open by law). On RED the flip REFUSES, naming the first - * divergence and the cure. One-directional unless an operator reverts - * the stored artifact explicitly. - * @returns The oracle report (green) — callers surface it as the flip receipt. - * @throws When the oracle is red; nothing is written. - */ - async adoptLogAuthority(): Promise { - await this.ensureInitialized() - this.assertWritable('adoptLogAuthority') - let report = await this.verifyLogAuthority() - - // BASELINE BACKFILL: curable divergences are rows whose CANONICAL truth - // simply never reached the log — pre-log records (e.g. the generation-0 - // VFS root, or a brain older than its log) and witness drift from - // maintenance that rewrote canonical outside a generation. The cure is - // an identity re-commit: any generational touch of the row makes the - // commit fact capture the CURRENT canonical bytes (the fact reads - // canonical back after execute), so the log converges on witness truth. - // Log-AHEAD divergences (log-live-canonical-absent / - // log-tombstone-canonical-present) are NOT curable by backfill — the - // log claims things the witness denies — and refuse loudly below. - let passes = 0 - while (report.verdict === 'red' && passes < 5) { - passes++ - const curable = report.mismatches.filter( - (m) => m.reason === 'pre-log-record' || m.reason === 'state-differs' - ) - const incurable = report.mismatches.filter( - (m) => m.reason !== 'pre-log-record' && m.reason !== 'state-differs' - ) - if (incurable.length > 0) { - throw new Error( - `adoptLogAuthority(): the log claims state the canonical witness denies ` + - `(${incurable.length} divergence(s); first: ${incurable[0].reason} on ` + - `${incurable[0].id}) — backfill cannot cure a log-ahead divergence. ` + - `Investigate before flipping; the witness remains authoritative.` - ) - } - if (curable.length === 0) break - prodLog.info( - `[Brainy] adoptLogAuthority: baseline backfill pass ${passes} — re-committing ` + - `${curable.length} row(s) whose canonical truth never reached the log` - ) - for (const m of curable) { - const raw = await this.storage.readNounRaw(m.id) - if (raw.metadata === null && raw.vector === null) continue // vanished since the scan - // LAW-SHAPE RE-COMMIT: rewrite canonical as EXACTLY the wrapper the - // log's reconstruction produces (the hydration law: denormalized - // enumeration fields derived from the metadata leg + the embedding - // floats). This is what makes the backfill actually CURE - // state-differs drift: rows written before the hydration law carry - // denormalized copies that disagree with their own metadata leg, and - // an as-is identity re-commit preserves that drift forever — the - // oracle re-flags it every pass and existing brains never flip. The - // metadata leg is the authority (denormalized fields are its - // projections, per the field-addressing law); nothing degrades: the - // floats ride through, adjacency residue has its own rebuild path. - const wrapper = - raw.vector !== null && typeof raw.vector === 'object' && !Array.isArray(raw.vector) - ? (raw.vector as Record) - : null - const vector = Array.isArray(raw.vector) - ? (raw.vector as number[]) - : Array.isArray(wrapper?.vector) - ? (wrapper!.vector as number[]) - : [] - const lawWrapper = reconstructNounWrapper(m.id, raw.metadata, vector) - const priorRaw = { metadata: raw.metadata, vector: raw.vector } - await this.persistSingleOp({ nouns: [m.id] }, async (tx) => { - tx.addOperation({ - name: 'BaselineLawShapeRewrite', - execute: async () => { - await this.storage.writeNounRaw(m.id, { - metadata: raw.metadata, - vector: lawWrapper - }) - return async () => { - await this.storage.writeNounRaw(m.id, priorRaw) - } - } - }) - }) - } - const next = await this.verifyLogAuthority() - if ( - next.verdict === 'red' && - next.mismatches.length >= report.mismatches.length && - !report.mismatchListTruncated - ) { - throw new Error( - `adoptLogAuthority(): baseline backfill made no progress ` + - `(${report.mismatches.length} → ${next.mismatches.length} mismatches; first: ` + - `${next.mismatches[0]?.reason} on ${next.mismatches[0]?.id}) — refusing to loop. ` + - `This is a divergence class the backfill cannot express; investigate.` - ) - } - report = next - } - - this._logAuthority = await flipToLogAuthority( - this.storage as unknown as LogAuthorityStorage, - report - ) - this.generationStore.setLogDurability('at-ack') - return report - } - /** * @description Read the reified transaction log — one entry per committed * generation, carrying the committed generation, the commit timestamp, and @@ -8565,7 +7668,6 @@ export class Brainy implements BrainyInterface { meta: options?.meta, ifAtGeneration: options?.ifAtGeneration, precommit: casPrecommit, - ...(plan.markerRecords.length > 0 ? { records: plan.markerRecords } : {}), execute: async () => { await this.transactionManager.executeTransaction( async (tx) => { @@ -8603,7 +7705,6 @@ export class Brainy implements BrainyInterface { // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) - this.noteWriteForPersistence() const receipt: TransactReceipt = { generation, timestamp, ids: plan.ids } return this.createPinnedDb({ generation, timestamp, receipt }) } @@ -9783,9 +8884,7 @@ export class Brainy implements BrainyInterface { } // 'absent' / vectorless / wrong-dim → skip (not vector-rankable at this gen). if (Array.isArray(vec) && vec.length === dim) { - // Mint-now fallback stamps the CURRENT committed watermark (the mint - // happens now, regardless of the historical G being materialized). - ints.push(BigInt(idMapper.getInt(id) ?? idMapper.getOrAssign(id, this.indexWriteGeneration()))) + ints.push(BigInt(idMapper.getInt(id) ?? idMapper.getOrAssign(id))) rows.push(vec) } } @@ -9823,8 +8922,7 @@ export class Brainy implements BrainyInterface { postCommit: [], casUpdates: [], createdNouns: new Set(), - changeEvents: [], - markerRecords: [] + changeEvents: [] } for (const op of ops) { @@ -9940,23 +9038,13 @@ export class Brainy implements BrainyInterface { } } - // MT5 deferred embedding: ack at durability with a stub vector and a - // DURABLE pending marker (written BEFORE the commit — an orphaned marker - // from a failed commit is harmless and reaped by the worker; a - // marker-less committed row would be a silently missing vector, which is - // the disallowed direction). The background worker embeds + inserts. - const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed - ? [] - : params.vector || (await this.embed(params.data)) - if (!deferringEmbed) { - if (!this.dimensions) { - this.dimensions = vector.length - } else if (vector.length !== this.dimensions) { - throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` - ) - } + const vector = params.vector || (await this.embed(params.data)) + if (!this.dimensions) { + this.dimensions = vector.length + } else if (vector.length !== this.dimensions) { + throw new Error( + `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` + ) } // isNew controls the operation's rollback strategy: a custom id may @@ -10019,20 +9107,11 @@ export class Brainy implements BrainyInterface { } } - if (deferringEmbed) { - // The pending marker rides the batch's ONE commit fact (same - // generation, one atomic append); the worker kicks post-commit via - // the plan hook. - plan.markerRecords.push(this.enqueuePendingEmbed(id)) - plan.postCommit.push(() => this.kickEmbedWorker()) - } plan.operations.push( new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), - ...(deferringEmbed - ? [] - : [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]), - new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration) + new AddToVectorIndexOperation(this.index, id, vector), + new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) ) plan.touchedNouns.push(id) plan.postCommit.push(() => { @@ -10206,15 +9285,13 @@ export class Brainy implements BrainyInterface { connections: new Map(), level: 0 }), - // ONE atomic vector-index leg — same law as update(): the row must - // never be absent from vector search during an update (see - // ReplaceInVectorIndexOperation). - new ReplaceInVectorIndexOperation(this.index, params.id, existing.vector, vector, this.indexWriteGeneration) + new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector), + new AddToVectorIndexOperation(this.index, params.id, vector) ) } plan.operations.push( - new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration), - new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration) + new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata), + new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) ) plan.touchedNouns.push(params.id) @@ -10294,10 +9371,10 @@ export class Brainy implements BrainyInterface { } if (noun) { - plan.operations.push(new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration)) + plan.operations.push(new RemoveFromVectorIndexOperation(this.index, id, noun.vector)) } if (metadata) { - plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration)) + plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata)) } // Pre-read metadata rides along: the count decrement must not depend on // re-reading the record being removed (see remove()). @@ -10349,14 +9426,6 @@ export class Brainy implements BrainyInterface { this._aggregationIndex.onEntityDeleted(id, entityForAgg) } }) - } else { - // Un-gated (mirror of remove()): a before-image-less delete flags an - // exact rescan instead of silently skipping the decrement. - plan.postCommit.push(() => { - this._aggregationIndex?.flagAllForRescan( - `transact delete of ${id} carried no before-image metadata — contribution unknowable` - ) - }) } state.nouns.delete(id) @@ -11311,22 +10380,6 @@ export class Brainy implements BrainyInterface { await this.generationStore.flushPendingSingleOps() // Flush all components in parallel for performance - // Watermark stamps ride every flush fan-out: stamp each projection with - // the committed generation BEFORE its flush persists (stamp-after-data - // holds inside each owner — the stamp is its LAST write; here we only - // hand the generation over). No committedGeneration capability = no - // stamp = the owner's verdict machinery treats the artifact as legacy. - { - const wmGen = this.storage?.committedGeneration?.() ?? null - if (wmGen !== null) { - // ALL THREE optional-chained: a replacement provider (the native - // pair swaps these managers) may not carry the stamp method — a - // missing stamp is a verdict-side rescan, never a flush crash. - ;(this.metadataIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - ;(this.index as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - ;(this.graphIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - } - } await Promise.all([ // 1. Flush storage adapter counts (entity/verb counts by type) (async () => { @@ -11350,22 +10403,7 @@ export class Brainy implements BrainyInterface { // 5. Persist the generation counter (8.0 MVCC — coalesced single-op // bumps become durable on every explicit flush) - this.generationStore.persistCounterNow(), - - // 6. Persist aggregation state, stamped at the committed generation - // (BRAINY-PROD-LATENCY-TRIAD / SELF-ENGINE-LIFECYCLE-SPRINT ask (a)): - // aggregation used to persist ONLY at close(), so a long-lived - // writer that flushes but never closes — the primary production - // shape — left its stamp behind after every write window, and any - // unclean exit forced a WHOLE-STORE backfill walk on the next - // first stats call (measured >60s and door-starving on a 9k-row - // production brain). Flushing here keeps the stamp current, so a - // reopen adopts (or incrementally catches up) instead of rescanning. - (async () => { - if (this._aggregationIndex) { - await this._aggregationIndex.flush() - } - })() + this.generationStore.persistCounterNow() ]) // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY @@ -11506,32 +10544,6 @@ export class Brainy implements BrainyInterface { return this.storage.requestFlushOverFilesystem(timeoutMs) } - /** - * @description The per-projection catch-up gauges served on - * `getIndexStatus().projections` (both the initialized and the - * pre-init snapshot — the numbers are safe to read at any lifecycle - * stage). Semantic reports the live deferred-embed backlog; metadata and - * graph are synchronous today (updated inside the write path); - * aggregation reports its rescan/catch-up backlogs (zero when the - * aggregation engine was never engaged). - */ - private projectionGauges(): { - semantic: { pendingEmbeds: number } - metadata: { synchronous: true } - graph: { synchronous: true } - aggregation: { pendingBackfills: number; pendingCatchUps: number } - } { - return { - semantic: { pendingEmbeds: this._pendingEmbedIds.size }, - metadata: { synchronous: true }, - graph: { synchronous: true }, - aggregation: { - pendingBackfills: this._aggregationIndex?.getPendingBackfills().length ?? 0, - pendingCatchUps: this._aggregationIndex?.getPendingCatchUps().length ?? 0 - } - } - } - /** * Get index loading status (Diagnostic for lazy loading) * @@ -11544,35 +10556,12 @@ export class Brainy implements BrainyInterface { * console.log(`HNSW Index: ${status.hnswIndex.size} entities`) * console.log(`Metadata Index: ${status.metadataIndex.entries} entries`) * console.log(`Graph Index: ${status.graphIndex.relationships} relationships`) - * console.log(`Pending embeds: ${status.projections.semantic.pendingEmbeds}`) * console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`) * ``` */ async getIndexStatus(): Promise<{ initialized: boolean lazyRebuildCompleted: boolean - /** Deferred embeds not yet landed (MT5) — the eventual-vector-index backlog. */ - pendingEmbeds: number - /** Per-projection catch-up gauges — the honest numbers behind - * {@link waitForIndexed}. `synchronous: true` marks projections updated - * inside the write path today: their barrier leg resolves immediately by - * design, and the flag becomes a real backlog gauge when the - * log-authority read path makes them asynchronous. */ - projections: { - /** The deferred-embedding backlog (same number as the top-level - * `pendingEmbeds`, which stays for compat). */ - semantic: { pendingEmbeds: number } - metadata: { synchronous: true } - graph: { synchronous: true } - aggregation: { - /** Aggregates flagged for a full rescan of existing entities - * (drained on the next aggregate query). */ - pendingBackfills: number - /** Aggregates adopted behind the watermark, with exact missing - * windows still to reconcile. */ - pendingCatchUps: number - } - } disableAutoRebuild: boolean /** `true` while a native provider runs the one-time 7.x → 8.0 rebuild LOCK. * A readiness probe should map this to HTTP 503 + Retry-After (transiently @@ -11618,8 +10607,6 @@ export class Brainy implements BrainyInterface { return { initialized: false, lazyRebuildCompleted: this.lazyRebuildCompleted, - pendingEmbeds: this._pendingEmbedIds.size, - projections: this.projectionGauges(), disableAutoRebuild: this.config.disableAutoRebuild || false, migrating: false, rebuildFailed: this._indexRebuildFailed != null, @@ -11662,8 +10649,6 @@ export class Brainy implements BrainyInterface { return { initialized: this.initialized, lazyRebuildCompleted: this.lazyRebuildCompleted, - pendingEmbeds: this._pendingEmbedIds.size, - projections: this.projectionGauges(), disableAutoRebuild: this.config.disableAutoRebuild || false, // A non-fatal index-rebuild failure recorded at init(), or adopt-forward // degraded ids, are degraded states (queries may be incomplete) — surface @@ -12106,26 +11091,21 @@ export class Brainy implements BrainyInterface { // Get total count for pagination UI (O(1) when possible) count: async (params: Omit, 'limit' | 'offset'>) => { - // Match-all normalization (shared with find()): an empty `where: {}` - // carries no predicates. Counting it as a filter would route through - // getIdsForFilter({}) → [] → a silent count of 0 while rows exist. - const constrainingWhere = whereConstrains(params.where) ? params.where : undefined - // For simple type queries, use O(1) index counting - if (params.type && !params.subtype && !params.query && !constrainingWhere && !params.connected) { + if (params.type && !params.subtype && !params.query && !params.where && !params.connected) { const types = Array.isArray(params.type) ? params.type : [params.type] return types.reduce((sum, type) => sum + this.metadataIndex.getEntityCountByType(type), 0) } // For complex queries, use metadata index for efficient counting - if (constrainingWhere || params.subtype || params.service) { + if (params.where || params.subtype || params.service) { let filter: any = {} - if (constrainingWhere) { + if (params.where) { // Where keys pass through UNTOUCHED — the one addressing law // parses them at the index boundary (bare = user metadata, // system.* = engine scalars). The old where.type→noun alias is // dead: a bare 'type' is the user's own field now. - Object.assign(filter, constrainingWhere) + Object.assign(filter, params.where) } if (params.service) filter['system.service'] = params.service if (params.subtype !== undefined) { @@ -12186,18 +11166,13 @@ export class Brainy implements BrainyInterface { return { // Stream all entities with optional filtering entities: async function* (this: Brainy, filter?: Partial>) { - // Match-all normalization (shared with find()): an empty `where: {}` - // carries no predicates — routing it through getIdsForFilter({}) - // would stream NOTHING while storage holds rows. Treat it as absent - // so it falls to the unfiltered storage-paginated walk below. - const constrainingWhere = whereConstrains(filter?.where) ? filter!.where : undefined - if (filter && (filter.type || filter.subtype || constrainingWhere || filter.service)) { + if (filter?.type || filter?.subtype || filter?.where || filter?.service) { // Use MetadataIndexManager for efficient filtered streaming let filterObj: any = {} - if (constrainingWhere) { + if (filter.where) { // Where keys pass through — the addressing law parses them at // the index boundary; the type→noun alias is dead. - Object.assign(filterObj, constrainingWhere) + Object.assign(filterObj, filter.where) } if (filter.service) filterObj['system.service'] = filter.service if (filter.subtype !== undefined) { @@ -14334,20 +13309,15 @@ export class Brainy implements BrainyInterface { service?: string excludeVFS?: boolean }): any | null { - // An empty `where: {}` carries no predicates — it is NOT structured - // criteria (see whereConstrains). Counting it would produce an empty - // filter object, and getIdsForFilter({}) / getIdSetForFilter({}) answer - // the empty set by contract — silently emptying a match-all query. - const constrainingWhere = whereConstrains(params.where) ? params.where : undefined - if (!(constrainingWhere || params.type || params.subtype || params.service || params.excludeVFS)) { + if (!(params.where || params.type || params.subtype || params.service || params.excludeVFS)) { return null } let filter: any = {} - if (constrainingWhere) { + if (params.where) { // Where keys pass through UNTOUCHED — the one addressing law parses // them at the index boundary (bare = user metadata, system.* = engine // scalars, typed refusal otherwise). The old type→noun alias is dead. - Object.assign(filter, constrainingWhere) + Object.assign(filter, params.where) } if (params.service) filter['system.service'] = params.service if (params.excludeVFS === true) { @@ -15855,11 +14825,7 @@ export class Brainy implements BrainyInterface { requireSubtype: config?.requireSubtype ?? true, // Multi-process safety mode: config?.mode ?? 'writer', - force: config?.force ?? false, - // Engine-owned persistence cadence — defaults resolve at the trigger - // site (policy 'auto': 512 writes / 30s interval / 2s idle). - persistence: config?.persistence, - logAuthority: config?.logAuthority ?? 'adopt' + force: config?.force ?? false } } @@ -15887,30 +14853,14 @@ export class Brainy implements BrainyInterface { } // If indexes already populated AND honestly serving, mark complete and skip. - // Honest gate: when a provider exposes isReady(), that REPLACES the size()>0 + // Honest gate: when the provider exposes isReady(), that REPLACES the size()>0 // proxy (a native index can report a non-zero size while its serving structure // is not loaded — the silent-empty cold-load class). A not-ready provider falls // through so the rebuild path can load it; verifyVectorLive() is the query-time // backstop either way. Providers without isReady() keep the size() heuristic // (the JS index's size()>0 genuinely means loaded). - // - // ALL THREE providers vote (fleet-adoption find, SELF-ENGINE-PAIR-STANDARD): - // this gate used to assess ONLY the vector index, so a not-ready native - // METADATA provider (its strand report) never blocked the completion latch - // — under disableAutoRebuild the promised lazy first-query rebuild never - // fired and every find() silently returned [] on a populated store. A - // not-ready report from ANY provider now falls through to the rebuild. const vectorReadiness = assessIndexReadiness(this.index) - const metadataReadiness = assessIndexReadiness(this.metadataIndex) - const graphReadiness = assessIndexReadiness(this.graphIndex) - const anyProviderNotReady = - vectorReadiness === 'not-ready' || - metadataReadiness === 'not-ready' || - graphReadiness === 'not-ready' - if ( - !anyProviderNotReady && - (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) - ) { + if (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) { this.lazyRebuildCompleted = true return } @@ -17155,20 +16105,6 @@ export class Brainy implements BrainyInterface { // persisted state is NOT listed — no walk at all on a clean reopen). await index.ready() - // Behind-stamp catch-up FIRST (SELF-ENGINE-LIFECYCLE-SPRINT ask (b)): - // adopted-but-behind state reconciles its exact missing window - // incrementally — bounded by that window's affected entities — instead - // of the whole-store rescan an unclean exit used to force. Single-flight - // like the walk below; a failed catch-up demotes to a LOUD rescan. - if (index.getPendingCatchUps().length > 0) { - if (!this._aggregationCatchUpFlight) { - this._aggregationCatchUpFlight = this.runAggregationCatchUp().finally(() => { - this._aggregationCatchUpFlight = null - }) - } - await this._aggregationCatchUpFlight - } - // Single-flight: concurrent queries share ONE walk instead of each wiping // the others' partial state and starting their own (the stampede that kept // a busy store from ever converging). The loop covers the rare case where @@ -17197,128 +16133,6 @@ export class Brainy implements BrainyInterface { } } - /** - * @description Build the aggregation view of a LIVE entity — top-level - * engine fields + the user bag, the same shape `entityForIndexing` and - * `entityForAggFromRawRecord` produce, so group keys and source filters - * resolve identically whichever door an entity arrives through. - */ - private aggViewFromEntity(e: Entity): Record { - return { - type: e.type, - ...(e.subtype !== undefined && { subtype: e.subtype }), - ...((e as unknown as Record).visibility !== undefined && { - visibility: (e as unknown as Record).visibility - }), - ...(e.confidence !== undefined && { confidence: e.confidence }), - ...(e.weight !== undefined && { weight: e.weight }), - createdAt: e.createdAt, - updatedAt: e.updatedAt, - ...(e.service !== undefined && { service: e.service }), - ...(e.data !== undefined && { data: e.data }), - ...(e.createdBy !== undefined && { createdBy: e.createdBy }), - metadata: e.metadata ?? {} - } - } - - /** Cap on a catch-up window's affected-entity count before demoting to a rescan. */ - private static readonly AGGREGATION_CATCHUP_MAX_AFFECTED = 5000 - - /** - * Reconcile every behind-stamp aggregate's exact missing window - * `(from, to]` using the fact log for the AFFECTED ID SET and time-travel - * reads for exact before/after states — cost bounded by writes since the - * last flush, never store size. Reconciliation targets the FIXED window - * end (`to` = the committed generation at adoption), so live write hooks - * compose exactly: every application on both paths is a precise old/new - * delta pair, and interleaving cannot drift totals. Any failure or an - * oversized window demotes to the announced full rescan — never a silent - * partial serve. - */ - private async runAggregationCatchUp(): Promise { - const index = this._aggregationIndex! - const catchups = index.getPendingCatchUps() - if (catchups.length === 0) return - - const startedAt = Date.now() - try { - // One fact scan covers every window (they share flush boundaries in - // practice); per-name windows filter per id below. - const from = Math.min(...catchups.map(c => c.from)) - const to = Math.max(...catchups.map(c => c.to)) - const scan = this.scanFacts({ fromGeneration: from + 1, toGeneration: to, kinds: ['noun'] }) - if (!scan) { - for (const c of catchups) { - index.demoteCatchUpToBackfill(c.name, 'no fact log on this store — window unreadable') - } - return - } - - // id → generations it changed at, inside the union window. - const affected = new Map() - for await (const batch of scan.batches()) { - for (const fact of batch.facts) { - for (const op of fact.ops) { - if (op.kind !== 'noun') continue - const gens = affected.get(op.id) - if (gens) gens.push(fact.generation) - else affected.set(op.id, [fact.generation]) - } - } - if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) break - } - if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) { - for (const c of catchups) { - index.demoteCatchUpToBackfill( - c.name, - `window touches >${Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED} entities — a rescan is cheaper` - ) - } - return - } - - // Exact before/after views per unique generation bound, via time travel. - const dbCache = new Map>() - const dbAt = async (gen: number): Promise> => { - let db = dbCache.get(gen) - if (!db) { - db = await this.asOf(gen) - dbCache.set(gen, db) - } - return db - } - try { - for (const c of catchups) { - const beforeDb = await dbAt(c.from) - const afterDb = await dbAt(c.to) - let reconciled = 0 - for (const [id, gens] of affected) { - if (!gens.some(g => g > c.from && g <= c.to)) continue - const [before, after] = await Promise.all([beforeDb.get(id), afterDb.get(id)]) - index.reconcileEntity( - c.name, - id, - before ? this.aggViewFromEntity(before) : null, - after ? this.aggViewFromEntity(after) : null - ) - reconciled++ - } - index.finishCatchUp(c.name) - prodLog.info( - `[Aggregation] '${c.name}': caught up generations ${c.from}→${c.to} — ` + - `${reconciled} entit${reconciled === 1 ? 'y' : 'ies'} reconciled in ${Date.now() - startedAt}ms (no store rescan)` - ) - } - } finally { - await Promise.all(Array.from(dbCache.values(), db => db.release().catch(() => {}))) - } - } catch (err) { - for (const c of index.getPendingCatchUps()) { - index.demoteCatchUpToBackfill(c.name, `catch-up failed: ${(err as Error).message}`) - } - } - } - /** * One store walk fills EVERY aggregate currently pending backfill — M pending * aggregates cost one enumeration, not M. Only reached when an aggregate @@ -17335,16 +16149,6 @@ export class Brainy implements BrainyInterface { const startedAt = Date.now() for (const n of names) index.beginBackfill(n) - // SELF-ENGINE-LIFECYCLE-SPRINT ask (c): when the native provider offers - // the parallel whole-rebuild (`rebuildAggregate` — on the contract since - // 8.x but never invoked), collect the walk's views and hand them over in - // ONE call per aggregate instead of a per-entity FFI stream. Memory note: - // the collected views are metadata-only records (no vectors); at the - // scales where this walk is even reached the array is the cheap part — - // the per-entity FFI round-trips were the measured cost. - const useProviderRebuild = index.hasProviderRebuild() - const collected: Array> = [] - let scanned = 0 try { const PAGE = 500 @@ -17356,12 +16160,8 @@ export class Brainy implements BrainyInterface { }) for (const noun of page.items) { const record = noun as unknown as Record - if (useProviderRebuild) { - collected.push(record) - } else { - for (const n of names) { - index.backfillEntity(n, record) - } + for (const n of names) { + index.backfillEntity(n, record) } } scanned += page.items.length @@ -17394,21 +16194,10 @@ export class Brainy implements BrainyInterface { throw err } - if (useProviderRebuild) { - for (const n of names) { - if (!index.rebuildWithProvider(n, collected)) { - // Provider refused/absent for this one — stream it the JS way. - for (const record of collected) index.backfillEntity(n, record) - index.finishBackfill(n) - } - } - } else { - for (const n of names) index.finishBackfill(n) - } + for (const n of names) index.finishBackfill(n) this._aggregationBackfillFailure = null prodLog.info( - `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) ` + - `in ${Date.now() - startedAt}ms${useProviderRebuild ? ' (native parallel rebuild)' : ''}` + `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms` ) } @@ -17419,16 +16208,6 @@ export class Brainy implements BrainyInterface { * This ensures deferred persistence mode data is saved */ async close(): Promise { - // Persistence cadence teardown: no background flush may fire after close - // begins (close() runs its own final flush). - if (this._persistIdleTimer) { - clearTimeout(this._persistIdleTimer) - this._persistIdleTimer = null - } - if (this._persistBackgroundFlight) { - await this._persistBackgroundFlight.catch(() => {}) - } - // Cancel any pending post-import background deduplication FIRST — it is a // writer (merge-deletes), and no delete pass may start mid- or post-close. this._backgroundDedup?.cancelPending() @@ -17596,26 +16375,6 @@ export class Brainy implements BrainyInterface { } } -/** - * @description Whether a `where` clause actually constrains the result set — - * i.e. it is a non-null object carrying at least one predicate key. An empty - * `where: {}` carries ZERO predicates and must behave exactly like an absent - * `where` everywhere it is consulted; treating it as "a filter is present" - * routes the query into the index-filter path, where `getIdsForFilter({})` - * answers `[]` by contract — a silent empty on a match-all query (the - * forbidden answer class: served-or-refused, never silently nothing). - * @param where - The raw `where` value from a query/selector params object. - * @returns `true` when `where` holds at least one predicate. - */ -function whereConstrains(where: unknown): where is Record { - return ( - where !== null && - typeof where === 'object' && - !Array.isArray(where) && - Object.keys(where).length > 0 - ) -} - /** * @description Extract the entity/relationship id from a canonical storage * path of the form `entities/(nouns|verbs)///metadata.json`. diff --git a/src/db/factLog.ts b/src/db/factLog.ts index 82949fb6..94e79700 100644 --- a/src/db/factLog.ts +++ b/src/db/factLog.ts @@ -41,57 +41,10 @@ * terminal-readable) is the single source of truth for the segment SET; * rotation flips it atomically (write-new → fsync → rename) BEFORE the new * tail's first byte exists, so no segment file is ever unaccounted for. - * - * ## Mixed-version logs (the v2 live-write cutover) - * - * The segment header's `formatVersion` selects the decoder PER SEGMENT: - * v1 segments (ops-shaped facts, the format above) stay readable forever and - * are NEVER rewritten; a NEW tail segment writes the v2 format - * (`src/db/factLogFormat.ts` — record envelope, minted dense ints, genesis, - * sector seals) whenever the int minter is installed ({@link FactLog.setIntMinter} — - * the brain wires it from the metadata index's id mapper right after init). - * A bare `FactLog` with no minter keeps writing v1 (there is no authority - * that could reproduce int assignments, and 0 is never written). Cutover - * mechanics on an existing v1 log: an EMPTY v1 tail is re-headed to v2 in - * place; a non-empty v1 tail is sealed by an immediate rotation and the new - * tail is v2. Decoded v2 facts map back to the SAME {@link CommitFact} shape - * v1 consumers read (noun/verb ops with `{metadata, vector} | null` records) — - * the vector wrapper object is reconstructed from the record's metadata leg - * through the reserved-field hydration law (see `commitFactFromV2`). - * - * V2 tails additionally: write the `log.genesis` record (id-space width 64 + - * the brain id, minted once into the manifest's additive `brainId` field) as - * the first record of the FIRST fact of a brand-new log, and seal every - * `sync()` to the header-declared sector size with pad frames that are - * invisible to readers (torn-page defense at group-commit boundaries). */ import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack' import { crc32c } from '../utils/crc32c.js' import { prodLog } from '../utils/logger.js' -import { - FACT_LOG_FORMAT_V1, - FACT_LOG_FORMAT_V2, - DEFAULT_SEAL_SIZE, - parseSegmentHeader, - encodeSegmentHeaderV2, - encodeFactV2, - decodeFact as decodeFormatFact, - decodeGroupV2, - encodePadFrame, - minPadFrameBytes, - type CommitFactV2, - type LogRecord, - type EmbedPendingRecord, - type EmbedLandedRecord, - type BlobManifestRecord, - type BootstrapBaselineRecord, - type ProjectionNoteRecord -} from './factLogFormat.js' -import { - splitNounMetadataRecord -} from '../types/reservedFields.js' -import { NounType } from '../types/graphTypes.js' -import { v4 as uuidv4 } from '../universal/uuid.js' // Swappable msgpack implementation — defaults to the JS codec; a native // provider (registered via the plugin registry's 'msgpack' key) may replace @@ -112,12 +65,7 @@ export function setFactCodec(impl: { export const FACTS_PREFIX = '_generations/facts' /** The facts manifest path (JSON). */ export const FACTS_MANIFEST_PATH = `${FACTS_PREFIX}/manifest.json` -/** - * The v1 segment format version — the MANIFEST's formatVersion gate and the - * header value of v1 (minter-less) tails. NOT the live-write ceiling: new - * tails write `FACT_LOG_FORMAT_V2` (src/db/factLogFormat.ts) whenever the - * int minter is installed; both versions are read forever, per segment. - */ +/** Current segment format version (header field; additive-only within a major). */ export const FACTS_FORMAT_VERSION = 1 /** Rotation threshold: seal the tail segment once it exceeds this many bytes. */ const SEGMENT_ROTATE_BYTES = 8 * 1024 * 1024 @@ -135,31 +83,6 @@ export interface FactOp { record: { metadata: unknown | null; vector: unknown | null } | null } -/** - * V2-native records beyond noun/verb ops that a fact may carry through the - * ENCODER (types 6/7/8/9/10 of the v2 registry: embed markers, blob - * manifests, projection notes, bootstrap baselines). The deferred-embedding - * lifecycle PRODUCES types 6/7 today: `embed.pending` rides the deferred - * write's own commit fact and `embed.landed` rides the background worker's - * landing commit (recovery folds the pair back out of the log at open). The - * blob lifecycle remodels onto type 8 in a later leg. - */ -export type FactMarkerRecord = - | EmbedPendingRecord - | EmbedLandedRecord - | BlobManifestRecord - | ProjectionNoteRecord - | BootstrapBaselineRecord - -/** - * Mints the dense integer handle for an entity/verb id at fact-append time — - * REQUIRED to be reproducible: a rebuilt id mapper must reproduce the same - * assignments exactly, so the only legal implementation delegates to the - * metadata index's id mapper (`getOrAssign`). Returns a POSITIVE bigint; a - * minter that cannot resolve its mapper throws — an int of 0 is never written. - */ -export type FactIntMinter = (kind: 'noun' | 'verb', id: string) => bigint - /** One committed generation, as scanned back out of the log. */ export interface CommitFact { generation: number @@ -167,12 +90,6 @@ export interface CommitFact { ops: FactOp[] meta?: Record blobHashes?: string[] - /** - * V2-native marker records riding this fact (see {@link FactMarkerRecord}). - * Optional and additive: absent on every v1 fact and on every fact the - * current writers produce; requires a v2 tail to encode. - */ - records?: FactMarkerRecord[] } /** The telemetry a scan batch carries (frozen shape). */ @@ -226,12 +143,6 @@ interface FactsManifest { /** The append target. Its true content is established by scanning (crash tolerance). */ tailSegment: string | null updatedAt: string - /** - * This brain's stable id (additive, v2 cutover): minted as a uuid at the - * first v2 tail creation and never changed; the `log.genesis` record - * carries it. Absent on logs that have never had a v2 tail. - */ - brainId?: string } /** The narrow byte-level storage surface the fact log rides. */ @@ -340,339 +251,40 @@ function decodeFact(payload: Uint8Array): CommitFact { } } -/** - * Deep-normalize a decoded v2 JSON position (metadata legs, meta maps, - * notes) back to plain-JSON values: the v2 codec decodes msgpack int64/uint64 - * as `bigint` (its u64 wire discipline), but canonical records are JSON — a - * metadata timestamp like `createdAt: 1786…` must come back as the NUMBER it - * was encoded from. Safe-range bigints narrow exactly; anything beyond the - * safe-integer range in a JSON position refuses loudly (it cannot have come - * from a JSON write). - */ -function normalizeWireJson(value: unknown): unknown { - if (typeof value === 'bigint') { - if (value > BigInt(Number.MAX_SAFE_INTEGER) || value < -BigInt(Number.MAX_SAFE_INTEGER)) { - throw new Error( - `fact log v2: decoded integer ${value} exceeds the JS safe-integer range in a JSON position` - ) - } - return Number(value) - } - if (Array.isArray(value)) return value.map(normalizeWireJson) - if (value && typeof value === 'object' && !(value instanceof Uint8Array)) { - const out: Record = {} - for (const [k, v] of Object.entries(value)) out[k] = normalizeWireJson(v) - return out - } - return value -} - -/** - * JSON-serialization equivalence for a v2 ENCODE-side JSON position: drop - * undefined-valued object keys and map undefined array elements to null — - * exactly what `JSON.stringify` does when canonical records are persisted. - * Commit facts are built from write-cache-WARM objects that may still carry - * undefined-valued engine keys (`service: undefined`, …) which the durable - * JSON never had; msgpack would preserve them as nil (the v1 capture's known - * wart), so the v2 capture — the future storage authority — sanitizes to the - * DURABLE truth instead. - */ -function toJsonSafe(value: unknown): unknown { - if (value === undefined) return null - if (Array.isArray(value)) return value.map((v) => (v === undefined ? null : toJsonSafe(v))) - if (value && typeof value === 'object' && !(value instanceof Uint8Array)) { - const out: Record = {} - for (const [k, v] of Object.entries(value)) { - if (v === undefined) continue - out[k] = toJsonSafe(v) - } - return out - } - return value -} - -/** Mirror of the storage layer's stored-timestamp normalization, minus its - * `Date.now()` fallback (a DECODER must be deterministic — an unreadable - * timestamp is omitted, and the divergence surfaces via the oracle). */ -function reconstructTimestamp(value: unknown): number | undefined { - if (typeof value === 'number' && value > 0) return value - if ( - value !== null && - typeof value === 'object' && - typeof (value as { seconds?: unknown }).seconds === 'number' - ) { - return (value as { seconds: number }).seconds * 1000 - } - return undefined -} - -/** - * Rebuild a noun's canonical VECTOR-FILE wrapper from a v2 after-image — - * the read-side of the hydration law. Canonical noun vector files hold the - * denormalized enumerable entity (`{id, vector, connections, level, type, - * …reserved fields…, metadata}` — the write path's composition); the v2 - * record deliberately carries only the ENTITY state (metadata leg + embedding - * floats), because connections/level are derived HNSW residue with their own - * rebuild paths (empty in every 8.x write) and the denormalized top-level - * fields are projections of the metadata leg. This reconstruction applies - * the SAME split/hydrate law the storage layer uses - * (`splitNounMetadataRecord` — the single source of truth in - * src/types/reservedFields.ts; field map mirrors - * `BaseStorage.hydrateNounWithMetadata`, undefined keys omitted exactly as - * JSON serialization omits them), so in the no-drift case the reconstructed - * wrapper digests byte-equal to canonical. A drifted denormalized copy - * surfaces as an oracle `state-differs` — named, never silently absorbed. - */ -export function reconstructNounWrapper( - id: string, - metadataLeg: unknown, - floats: number[] -): Record { - const { reserved, custom } = splitNounMetadataRecord( - (metadataLeg ?? null) as Record | null - ) - const wrapper: Record = { - id, - vector: floats, - connections: {}, - level: 0, - type: (reserved.noun as string) || NounType.Thing - } - if (reserved.subtype !== undefined) wrapper.subtype = reserved.subtype - if (reserved.visibility !== undefined) wrapper.visibility = reserved.visibility - const createdAt = reconstructTimestamp(reserved.createdAt) - if (createdAt !== undefined) wrapper.createdAt = createdAt - const updatedAt = reconstructTimestamp(reserved.updatedAt) - if (updatedAt !== undefined) wrapper.updatedAt = updatedAt - if (reserved.confidence !== undefined) wrapper.confidence = reserved.confidence - if (reserved.weight !== undefined) wrapper.weight = reserved.weight - if (reserved.service !== undefined) wrapper.service = reserved.service - if (reserved.data !== undefined) wrapper.data = reserved.data - if (reserved.createdBy !== undefined) wrapper.createdBy = reserved.createdBy - wrapper._rev = typeof reserved._rev === 'number' ? reserved._rev : 1 - wrapper.metadata = custom - return wrapper -} - -/** Coerce a candidate embedding to `number[]`: plain arrays pass through - * (element-checked); numeric typed arrays (the JS HNSW rebuild path stores - * `Float32Array` vectors on the memory adapter) widen via `Array.from`. */ -function floatsOf(candidate: unknown, context: string): number[] | undefined { - if (Array.isArray(candidate)) { - for (const el of candidate) { - if (typeof el !== 'number') { - throw new Error(`fact log v2: ${context} vector carries a non-number element`) - } - } - return candidate as number[] - } - if (ArrayBuffer.isView(candidate) && !(candidate instanceof DataView)) { - return Array.from(candidate as unknown as ArrayLike) - } - return undefined -} - -/** Extract the embedding float array from a canonical vector value: a bare - * float array (or numeric typed array) passes through; a wrapper object - * yields its `vector` floats; `null` stays `null`; anything else refuses - * loudly. */ -function embeddingLegOf(value: unknown, context: string): number[] | null { - if (value === null || value === undefined) return null - const direct = floatsOf(value, context) - if (direct !== undefined) return direct - if (typeof value === 'object') { - const nested = floatsOf((value as { vector?: unknown }).vector, context) - if (nested !== undefined) return nested - } - throw new Error( - `fact log v2: ${context} has a canonical vector record with no float vector — ` + - `cannot encode its after-image` - ) -} - -/** - * Map one decoded v2 fact to the {@link CommitFact} shape every consumer - * already reads: noun/verb after-images and tombstones become ops (vector - * wrappers reconstructed — see {@link reconstructNounWrapper}); a - * `batch.meta` record becomes `meta` when the fact position carries none; - * `log.genesis` is log-level metadata (its width was verified at decode) and - * is not an op; marker records surface on the additive `records` field so - * nothing is silently dropped. Decoded JSON positions are normalized back - * from the codec's bigint discipline ({@link normalizeWireJson}). - */ -function commitFactFromV2(f: CommitFactV2): CommitFact { - const ops: FactOp[] = [] - const markers: FactMarkerRecord[] = [] - let batchMeta: Record | undefined - for (const r of f.records) { - switch (r.type) { - case 'noun.afterImage': { - const metadata = normalizeWireJson(r.metadata) ?? null - let vector: unknown | null = null - if (r.vectorLeg !== null) { - if (!Array.isArray(r.vectorLeg)) { - throw new Error( - `fact log v2: noun.afterImage ${r.id} carries a vector ref — this reader ` + - `resolves inline vectors only (refs are a later leg); refusing` - ) - } - vector = reconstructNounWrapper(r.id, metadata, r.vectorLeg) - } - ops.push({ kind: 'noun', id: r.id, record: { metadata, vector } }) - break - } - case 'noun.tombstone': - ops.push({ kind: 'noun', id: r.id, record: null }) - break - case 'verb.afterImage': { - const metadata = normalizeWireJson(r.metadata) ?? null - if (r.vectorLeg !== null && !Array.isArray(r.vectorLeg)) { - throw new Error( - `fact log v2: verb.afterImage ${r.id} carries a vector ref — this reader ` + - `resolves inline vectors only (refs are a later leg); refusing` - ) - } - // The canonical verb vector-file wrapper: endpoints + verb name ride - // as first-class v2 wire fields precisely so this reconstruction is - // exact ({id, vector, connections:{}, verb, sourceId, targetId} — - // verbs carry no `level`). - const vector: Record = { - id: r.id, - vector: r.vectorLeg ?? [], - connections: {}, - verb: r.verb, - sourceId: r.sourceId, - targetId: r.targetId - } - ops.push({ kind: 'verb', id: r.id, record: { metadata, vector } }) - break - } - case 'verb.tombstone': - ops.push({ kind: 'verb', id: r.id, record: null }) - break - case 'batch.meta': - batchMeta = normalizeWireJson(r.meta) as Record - break - case 'log.genesis': - break // the log's birth certificate — log-level metadata, not an op - case 'projection.note': - markers.push({ ...r, note: normalizeWireJson(r.note) as Record }) - break - case 'bootstrap.baseline': - markers.push({ ...r, metadata: normalizeWireJson(r.metadata) }) - break - default: - // embed.pending / embed.landed / blob.manifest carry no loose JSON maps. - markers.push(r) - break - } - } - const meta = f.meta ? (normalizeWireJson(f.meta) as Record) : batchMeta - return { - generation: f.generation, - timestamp: f.timestamp, - ops, - ...(meta ? { meta } : {}), - ...(f.blobHashes && f.blobHashes.length > 0 ? { blobHashes: f.blobHashes } : {}), - ...(markers.length > 0 ? { records: markers } : {}) - } -} - -/** One intact v2 frame's extent inside a segment (byte-slicing support). */ -interface V2FrameExtent { - /** Byte offset just past this frame. */ - end: number - /** The frame's generation (0 for pad filler). */ - generation: number - /** True when the frame is a pad (invisible filler). */ - isPad: boolean -} - -/** Walk a v2 segment's intact frames (torn-tail terminated), returning each - * frame's extent — the byte-level view truncation slices against, so kept - * frames are never re-encoded (byte-immutability of CRC-covered frames). */ -function walkV2Frames(bytes: Uint8Array): V2FrameExtent[] { - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - const extents: V2FrameExtent[] = [] - let offset = HEADER_BYTES - while (offset + FRAME_PREFIX_BYTES <= bytes.length) { - const length = view.getUint32(offset, true) - const expectedCrc = view.getUint32(offset + 4, true) - const start = offset + FRAME_PREFIX_BYTES - const end = start + length - if (end > bytes.length) break // torn tail - const payload = bytes.subarray(start, end) - if (crc32c(payload) !== expectedCrc) break // torn tail - const fact = decodeFormatFact(payload, FACT_LOG_FORMAT_V2, { - expectedIdSpaceWidth: 64 - }) as CommitFactV2 - extents.push({ end, generation: fact.generation, isPad: fact.records.length === 0 }) - offset = end - } - return extents -} - -/** - * The byte offset a v2 segment is cut at to keep exactly the facts with - * `generation ≤ keepThrough`: the end of the last kept FACT frame (pads - * between kept facts sit inside the retained span; pads after the cut are - * dropped and re-sealed at the next sync). When nothing is dropped the cut - * lands after the last intact frame — trailing pads retained, only a torn - * suffix (if any) removed. - */ -function v2CutOffset(extents: V2FrameExtent[], keepThrough: number): number { - let cut = HEADER_BYTES - let lastIntactEnd = HEADER_BYTES - for (const e of extents) { - lastIntactEnd = e.end - if (e.isPad) continue - if (e.generation <= keepThrough) { - cut = e.end - } else { - return cut // first beyond-keep fact: everything from here (pads included) goes - } - } - return lastIntactEnd -} - /** * Parse a segment's bytes: verify the header, then walk frames until the end * or a torn tail (length overrun / CRC mismatch), which terminates the walk — - * everything before it is intact. The header's formatVersion selects the - * decoder: the v1 walk below is byte-identical to the original v1 reader; - * v2 segments decode through the reference codec (`decodeGroupV2`, pads - * invisible, id-space width verified at 64 — a disagreeing genesis throws - * the codec's typed `GenesisWidthMismatchError`). Returns the decoded facts - * plus the byte length of the VALID prefix (header + intact frames), which - * reconciliation uses to cut a torn tail without re-encoding. + * everything before it is intact. Returns the decoded facts plus the byte + * length of the VALID prefix (header + intact frames), which reconciliation + * uses to cut a torn tail without re-encoding. */ function parseSegment( file: string, bytes: Uint8Array -): { facts: CommitFact[]; validBytes: number; formatVersion: number; sealSize?: number } { +): { facts: CommitFact[]; validBytes: number } { if (bytes.length < HEADER_BYTES) { prodLog.warn(`[FactLog] segment ${file} shorter than its header — treating as empty`) - return { facts: [], validBytes: 0, formatVersion: 0 } + return { facts: [], validBytes: 0 } } - let header: { formatVersion: number; sealSize?: number } - try { - header = parseSegmentHeader(bytes.subarray(0, HEADER_BYTES)) - } catch (err) { - throw new Error(`fact log: segment ${file}: ${(err as Error).message}`) + for (let i = 0; i < MAGIC.length; i++) { + if (bytes[i] !== MAGIC[i]) { + throw new Error(`fact log: segment ${file} has a bad magic — not a fact segment`) + } } - - if (header.formatVersion === FACT_LOG_FORMAT_V2) { - const group = decodeGroupV2(bytes.subarray(HEADER_BYTES), { expectedIdSpaceWidth: 64 }) - return { - facts: group.facts.map(commitFactFromV2), - validBytes: HEADER_BYTES + group.validBytes, - formatVersion: FACT_LOG_FORMAT_V2, - sealSize: header.sealSize + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const version = view.getUint32(8, true) + if (version !== FACTS_FORMAT_VERSION) { + throw new Error( + `fact log: segment ${file} has formatVersion ${version}; this build reads ${FACTS_FORMAT_VERSION}` + ) + } + for (let i = 20; i < HEADER_BYTES; i++) { + if (bytes[i] !== 0) { + // Non-zero reserved bytes = a future format this build cannot verify. + throw new Error(`fact log: segment ${file} has non-zero reserved header bytes — unverifiable`) } } - // v1 walk — byte-identical to the original reader. - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) const facts: CommitFact[] = [] let offset = HEADER_BYTES while (offset + FRAME_PREFIX_BYTES <= bytes.length) { @@ -686,7 +298,7 @@ function parseSegment( facts.push(decodeFact(payload)) offset = end } - return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 } + return { facts, validBytes: offset } } /** @@ -706,54 +318,23 @@ export class FactLog { } /** Decoded facts of the TAIL segment (bounded by the rotation threshold). */ private tailFacts: CommitFact[] = [] - /** Byte size of the tail segment file (valid prefix, pads included — - * pads count toward bytes but NEVER toward facts). */ + /** Byte size of the tail segment file (valid prefix). */ private tailBytes = 0 /** Highest generation in the log (0 = empty). */ private head = 0 /** Segment paths appended since the last sync (the fsync batch). */ private readonly dirtySegments = new Set() - /** The TAIL segment's on-disk format version (selects the live encoder). */ - private tailVersion: number = FACT_LOG_FORMAT_V1 - /** The tail's sector-seal size (v2 tails; from its header on reopen). */ - private tailSealSize: number = DEFAULT_SEAL_SIZE - /** The v2 int minter (see {@link FactIntMinter}); null = v1 live writes. */ - private intMinter: FactIntMinter | null = null constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) { this.storage = storage this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES } - /** - * Install the v2 int minter — the capability gate for v2 LIVE WRITES. - * With a minter installed, every NEW tail segment writes the v2 format and - * after-image records carry minted dense ints; without one, live writes - * stay v1 (no authority could reproduce int assignments, and 0 is never - * written). The brain wires this from the metadata index's id mapper right - * after the index is ready; an existing v1 tail cuts over on the next - * append (empty tail: re-headed in place; non-empty: sealed by rotation). - */ - setIntMinter(mint: FactIntMinter): void { - this.intMinter = mint - } - /** The highest committed generation the log holds (0 = empty). */ headGeneration(): number { return this.head } - /** - * True when this log has EVER had a v2 tail — the manifest's `brainId` is - * minted at every v2 tail creation seam and never removed (the tail-version - * check is a belt-and-braces second signal). Only v2 facts can carry marker - * records, so marker folds (e.g. the deferred-embed recovery scan) skip - * v1-only logs WHOLESALE on this one cheap check — no segment is read. - */ - hasV2History(): boolean { - return this.manifest.brainId !== undefined || this.tailVersion === FACT_LOG_FORMAT_V2 - } - /** * Open the log and reconcile it to committed truth: read the manifest, * establish the tail's intact content (torn-tail scan), then TRUNCATE any @@ -761,34 +342,6 @@ export class FactLog { * crash between fact-append and the commit point). After open, the log is * exactly the committed prefix. */ - /** - * Read (without truncating) every intact fact ABOVE a generation — the - * log-authority recovery surface: after a crash, facts beyond the - * manifest watermark that survived with valid CRCs are ACKED writes in - * durable-at-ack mode, and the owner REPLAYS them instead of letting - * open() truncate them. Must be called BEFORE open() (it reads the raw - * segments directly; the torn tail's invalid suffix is ignored exactly - * like open() would). - */ - async peekFactsAbove(committedGeneration: number): Promise { - const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null - if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return [] - if (stored.formatVersion !== FACTS_FORMAT_VERSION) return [] - const out: CommitFact[] = [] - const files = [...stored.segments.map((s) => s.file)] - if (stored.tailSegment) files.push(stored.tailSegment) - for (const file of files) { - const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) - if (bytes === null) continue - const { facts } = parseSegment(file, bytes) - for (const f of facts) { - if (f.generation > committedGeneration) out.push(f) - } - } - out.sort((a, b) => a.generation - b.generation) - return out - } - async open(committedGeneration: number): Promise { const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) { @@ -831,17 +384,11 @@ export class FactLog { const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` const bytes = await this.storage.readRawBytes(tailPath) if (bytes === null) { - // Manifest named a tail whose first byte never landed — an empty - // tail. Its header (and format version) is established at the next - // append (see the tail-provisioning ladder there). + // Manifest named a tail whose first byte never landed — an empty tail. this.tailFacts = [] this.tailBytes = 0 } else { - const parsed = parseSegment(this.manifest.tailSegment, bytes) - const { facts, validBytes } = parsed - this.tailVersion = - parsed.formatVersion === FACT_LOG_FORMAT_V2 ? FACT_LOG_FORMAT_V2 : FACT_LOG_FORMAT_V1 - this.tailSealSize = parsed.sealSize ?? DEFAULT_SEAL_SIZE + const { facts, validBytes } = parseSegment(this.manifest.tailSegment, bytes) const kept = facts.filter((f) => f.generation <= committedGeneration) if (kept.length !== facts.length || validBytes !== bytes.length) { const dropped = facts.length - kept.length @@ -851,16 +398,7 @@ export class FactLog { `${committedGeneration} from the tail (never committed)` ) } - if (this.tailVersion === FACT_LOG_FORMAT_V2) { - // V2: byte-slice at frame boundaries — CRC-covered frames are - // byte-immutable; a truncation never re-encodes what it keeps. - const cut = v2CutOffset(walkV2Frames(bytes), committedGeneration) - await this.storage.writeRawBytes(tailPath, bytes.subarray(0, cut)) - this.tailFacts = kept - this.tailBytes = cut - } else { - await this.rewriteTail(kept) - } + await this.rewriteTail(kept) } else { this.tailFacts = facts this.tailBytes = validBytes @@ -875,15 +413,6 @@ export class FactLog { * Append one committed generation's fact. NOT durable until {@link sync} — * the caller batches durability at its commit barrier (transact syncs in * the same call; Model-B group-commit syncs at flush). - * - * Tail provisioning (in order): a missing tail starts one; a named tail - * whose header never landed (manifest-first crash) gets its header now; an - * existing V1 tail cuts over to v2 once the minter is installed (empty: - * re-headed in place, non-empty: sealed by rotation — v1 segments are never - * rewritten); a full tail rotates. The frame then encodes in the TAIL's - * format: v2 tails carry after-image records with minted ints (and the - * genesis record on the very first fact of a brand-new log); v1 tails keep - * the v1 wire format byte-identically. */ async append(fact: CommitFact): Promise { if (fact.generation <= this.head) { @@ -893,42 +422,10 @@ export class FactLog { } if (this.manifest.tailSegment === null) { await this.startTail(fact.generation) - } else if (this.tailBytes === 0) { - await this.reinitializeTailHeader() - } else if (this.intMinter !== null && this.tailVersion === FACT_LOG_FORMAT_V1) { - if (this.tailFacts.length === 0 && this.tailBytes <= HEADER_BYTES) { - await this.upgradeEmptyTailToV2() - } else { - await this.rotate(fact.generation) - } } else if (this.tailBytes >= this.rotateBytes) { await this.rotate(fact.generation) } - - let frame: Uint8Array - if (this.tailVersion === FACT_LOG_FORMAT_V2) { - const records = this.buildV2Records(fact) - if (this.needsGenesis()) { - if (this.ensureBrainId()) await this.persistManifest() - records.unshift(this.genesisRecord()) - } - frame = encodeFactV2({ - generation: fact.generation, - timestamp: fact.timestamp, - records, - ...(fact.meta ? { meta: toJsonSafe(fact.meta) as Record } : {}), - ...(fact.blobHashes && fact.blobHashes.length > 0 ? { blobHashes: fact.blobHashes } : {}) - }) - } else { - if (fact.records && fact.records.length > 0) { - throw new Error( - `fact log: marker records (${fact.records.map((r) => r.type).join(', ')}) require a ` + - `v2 tail segment — this log's tail is v1 (no int minter installed); refusing rather ` + - `than silently dropping them` - ) - } - frame = encodeFrame(fact) - } + const frame = encodeFrame(fact) const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` await this.storage.appendRawBytes(tailPath, frame) this.tailFacts.push(fact) @@ -937,66 +434,14 @@ export class FactLog { this.dirtySegments.add(tailPath) } - /** - * Fsync every segment appended since the last sync. SEALS AT SYNC: a v2 - * tail is first padded to its sector-seal boundary (one pad frame, - * invisible to readers; a gap smaller than the smallest constructible pad - * frame pads through one extra sector — the codec's rule), so every - * durability barrier leaves the tail sector-aligned: a torn page can only - * tear INSIDE the group being written, never a previously-sealed one. - */ + /** Fsync every segment appended since the last sync. */ async sync(): Promise { - await this.padTailToSealBoundary() if (this.dirtySegments.size === 0) return const paths = [...this.dirtySegments] this.dirtySegments.clear() await this.storage.syncRawObjects(paths) } - // --- GROUP COMMIT ON THE LOG (durable-at-ack mode) ------------------------ - // Classic group commit: concurrent writers append, then join ONE fsync - // whose completion releases every covered ack. Two slots — the running - // sync and at most one queued behind it — give the covering guarantee: - // an append followed by ensureSynced() is always covered, because the - // sync it awaits STARTS after the append landed (a running sync that - // may have snapshotted earlier is never joined; the queued one is). - private syncRunning: Promise | null = null - private syncQueued: Promise | null = null - - /** - * Await a sync that covers every byte appended before this call. Many - * concurrent callers share one fsync (solo caller = immediate sync). The - * durability contract of an acked write in log-durable mode: this promise - * resolving means the caller's frames survive power loss. - */ - async ensureSynced(): Promise { - if (this.syncQueued) { - // A sync that has NOT started yet exists — it will snapshot after our - // append, so it covers us. - return this.syncQueued - } - if (this.syncRunning) { - // The running sync may have snapshotted before our append — queue the - // next one behind it and join that. - const queued = this.syncRunning - .catch(() => {}) - .then(() => { - // Promote: the queued sync becomes the running one. - this.syncQueued = null - this.syncRunning = this.sync().finally(() => { - this.syncRunning = null - }) - return this.syncRunning - }) - this.syncQueued = queued - return queued - } - this.syncRunning = this.sync().finally(() => { - this.syncRunning = null - }) - return this.syncRunning - } - /** * Open a scan over committed facts. The scan runs against a MANIFEST * SNAPSHOT (sealed segments + the tail's decoded facts at open) — exactly- @@ -1154,25 +599,7 @@ export class FactLog { `(head ${this.head}) — the fact to drop was already sealed; the log needs reopen` ) } - if (this.tailVersion === FACT_LOG_FORMAT_V2) { - // V2: byte-slice at frame boundaries (kept frames stay byte-identical; - // pads between kept facts are retained inside the prefix, trailing pads - // go and the next sync re-seals). The dropped frames may be unsynced — - // readRawBytes is read-after-write coherent over the append path. - const file = this.manifest.tailSegment - if (!file) return - const tailPath = `${FACTS_PREFIX}/${file}` - const bytes = await this.storage.readRawBytes(tailPath) - if (bytes === null) { - throw new Error(`fact log: dropAbove(${keepThrough}) cannot read the tail segment ${file}`) - } - const cut = v2CutOffset(walkV2Frames(bytes), keepThrough) - await this.storage.writeRawBytes(tailPath, bytes.subarray(0, cut)) - this.tailFacts = kept - this.tailBytes = cut - } else { - await this.rewriteTail(kept) - } + await this.rewriteTail(kept) this.head = this.computeHead() } @@ -1185,42 +612,25 @@ export class FactLog { return 0 } - /** The header bytes for a NEW tail: v2 whenever the minter is installed. */ - private newTailHeader(firstGeneration: number): Uint8Array { - return this.intMinter !== null - ? encodeSegmentHeaderV2(firstGeneration, DEFAULT_SEAL_SIZE) - : buildHeader(firstGeneration) - } - - /** Record the just-created tail's format in memory (mirrors its header). */ - private noteFreshTail(): void { - this.tailVersion = this.intMinter !== null ? FACT_LOG_FORMAT_V2 : FACT_LOG_FORMAT_V1 - this.tailSealSize = DEFAULT_SEAL_SIZE - } - /** Create the very first tail segment (manifest-first, then header bytes). */ private async startTail(firstGeneration: number): Promise { const file = segmentFileName(firstGeneration) this.manifest.tailSegment = file - if (this.intMinter !== null) this.ensureBrainId() await this.persistManifest() - await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, this.newTailHeader(firstGeneration)) + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, buildHeader(firstGeneration)) this.tailFacts = [] this.tailBytes = HEADER_BYTES - this.noteFreshTail() } /** * Seal the tail into the manifest and start a new one. Manifest-first: the * flip both seals the old tail AND names the new one atomically, so no - * segment file ever exists unaccounted for. The NEW tail's format follows - * the minter gate ({@link newTailHeader}) — this is also the v1→v2 cutover - * seam for a non-empty v1 tail (sealed as-is, never rewritten). + * segment file ever exists unaccounted for. */ private async rotate(nextGeneration: number): Promise { const sealedFile = this.manifest.tailSegment if (!sealedFile) return - // Seal what the tail actually holds (sync() also sector-seals a v2 tail). + // Seal what the tail actually holds. await this.sync() // sealed segments are always fully durable const entry: SegmentEntry = { file: sealedFile, @@ -1232,187 +642,10 @@ export class FactLog { const newFile = segmentFileName(nextGeneration) this.manifest.segments.push(entry) this.manifest.tailSegment = newFile - if (this.intMinter !== null) this.ensureBrainId() await this.persistManifest() - await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, this.newTailHeader(nextGeneration)) + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, buildHeader(nextGeneration)) this.tailFacts = [] this.tailBytes = HEADER_BYTES - this.noteFreshTail() - } - - /** - * The v1→v2 cutover for an EMPTY v1 tail: re-head it in place (nothing but - * the 32-byte header exists, so no v1 frame is ever rewritten). Also the - * cheapest cutover shape: brand-new brains whose first tail predates the - * minter installation converge here on their first post-install append. - */ - private async upgradeEmptyTailToV2(): Promise { - const file = this.manifest.tailSegment - if (!file) return - if (this.ensureBrainId()) await this.persistManifest() - const first = this.segmentFirstGenerationFromName(file) - const path = `${FACTS_PREFIX}/${file}` - await this.storage.writeRawBytes(path, encodeSegmentHeaderV2(first, DEFAULT_SEAL_SIZE)) - this.tailBytes = HEADER_BYTES - this.tailVersion = FACT_LOG_FORMAT_V2 - this.tailSealSize = DEFAULT_SEAL_SIZE - this.dirtySegments.add(path) - } - - /** - * A manifest-named tail whose header never landed (crash between the - * manifest flip and the first header byte — previously this appended - * frames into a headerless file the next open could not parse): write the - * header now, in the CURRENT format gate. - */ - private async reinitializeTailHeader(): Promise { - const file = this.manifest.tailSegment - if (!file) return - if (this.intMinter !== null && this.ensureBrainId()) await this.persistManifest() - const first = this.segmentFirstGenerationFromName(file) - const path = `${FACTS_PREFIX}/${file}` - await this.storage.writeRawBytes(path, this.newTailHeader(first)) - this.tailBytes = HEADER_BYTES - this.noteFreshTail() - this.dirtySegments.add(path) - } - - /** True when the NEXT appended fact is the first fact of a brand-new v2 - * log — the one that must open with the log.genesis record. */ - private needsGenesis(): boolean { - return ( - this.tailVersion === FACT_LOG_FORMAT_V2 && - this.manifest.segments.length === 0 && - this.tailFacts.length === 0 - ) - } - - /** Mint the brain id into the manifest if absent; true when it changed. */ - private ensureBrainId(): boolean { - if (this.manifest.brainId) return false - this.manifest.brainId = uuidv4() - return true - } - - /** The log's birth certificate (id-space width 64 — the only width this - * writer mints; a reader expecting another width refuses at decode). */ - private genesisRecord(): LogRecord { - const brainId = this.manifest.brainId - if (!brainId) { - throw new Error( - 'fact log v2: genesis requires a brainId in the facts manifest — invariant violated' - ) - } - return { type: 'log.genesis', idSpaceWidth: 64, brainId, createdAt: Date.now() } - } - - /** - * Convert one CommitFact's ops (+ optional marker records) to v2 wire - * records, MINTING ints at append time: entity/verb ints come from the - * injected minter (the metadata index's id mapper — the one authority a - * rebuild reproduces exactly). Verb endpoints and the verb name ride as - * first-class wire fields, lifted from the canonical verb vector wrapper. - * Every refusal here is loud — an after-image without a mintable int, a - * verb without endpoints, or a vector record without floats fails the - * WRITE, never writes a 0. - */ - private buildV2Records(fact: CommitFact): LogRecord[] { - const mint = (kind: 'noun' | 'verb', id: string): bigint => { - if (this.intMinter === null) { - throw new Error( - `fact log v2: no int minter is installed — cannot mint the ${kind} int for ${id}; ` + - `refusing to write a v2 after-image (an int of 0 is never written)` - ) - } - const minted = this.intMinter(kind, id) - // Reserved-root exemption: int 0 is legitimate for exactly one id — - // the all-zeros VFS root, minted 0 by construction at genesis on - // existing brains. Zero anywhere else is a corrupt mint. - const isReservedRoot = - minted === 0n && id === '00000000-0000-0000-0000-000000000000' - if (typeof minted !== 'bigint' || minted < 0n || (minted === 0n && !isReservedRoot)) { - throw new Error( - `fact log v2: the int minter returned ${String(minted)} for ${kind} ${id} — ` + - `minted ints are positive bigints (int 0 reserved for the VFS root alone); ` + - `refusing to write` - ) - } - return minted - } - - const records: LogRecord[] = [] - for (const op of fact.ops) { - if (op.kind === 'noun') { - if (op.record === null) { - records.push({ type: 'noun.tombstone', id: op.id }) - continue - } - records.push({ - type: 'noun.afterImage', - id: op.id, - entityInt: mint('noun', op.id), - metadata: toJsonSafe(op.record.metadata ?? null), - vectorLeg: embeddingLegOf(op.record.vector, `noun ${op.id}`) - }) - } else { - if (op.record === null) { - records.push({ type: 'verb.tombstone', id: op.id }) - continue - } - const wrapper = op.record.vector as Record | null - const verbName = wrapper?.verb - const sourceId = wrapper?.sourceId - const targetId = wrapper?.targetId - if ( - typeof verbName !== 'string' || - typeof sourceId !== 'string' || - typeof targetId !== 'string' - ) { - throw new Error( - `fact log v2: verb ${op.id} has no canonical endpoints (verb/sourceId/targetId ` + - `live in its vector record, which is missing or torn) — refusing to write an ` + - `after-image that could not be replayed` - ) - } - const floats = floatsOf(wrapper?.vector, `verb ${op.id}`) ?? [] - records.push({ - type: 'verb.afterImage', - id: op.id, - verbInt: mint('verb', op.id), - metadata: toJsonSafe(op.record.metadata ?? null), - vectorLeg: floats, - verb: verbName, - sourceId, - sourceInt: mint('noun', sourceId), - targetId, - targetInt: mint('noun', targetId) - }) - } - } - for (const marker of fact.records ?? []) records.push(marker) - return records - } - - /** - * Pad a v2 tail to its next sector-seal boundary with ONE pad frame — - * called from {@link sync} so alignment holds at every durability barrier. - * Pads count toward {@link tailBytes} but never toward facts (they are - * invisible to every reader); a gap smaller than the smallest constructible - * pad frame pads through one extra sector (the codec's rule). No-op for v1 - * tails, empty tails, and already-aligned tails. - */ - private async padTailToSealBoundary(): Promise { - if (this.tailVersion !== FACT_LOG_FORMAT_V2) return - const file = this.manifest.tailSegment - if (!file || this.tailBytes <= HEADER_BYTES) return - const remainder = this.tailBytes % this.tailSealSize - if (remainder === 0) return - let padBytes = this.tailSealSize - remainder - if (padBytes < minPadFrameBytes()) padBytes += this.tailSealSize - const tailPath = `${FACTS_PREFIX}/${file}` - await this.storage.appendRawBytes(tailPath, encodePadFrame(padBytes)) - this.tailBytes += padBytes - this.dirtySegments.add(tailPath) } /** Atomically persist the manifest (write-new → fsync → rename downstream). */ @@ -1441,24 +674,17 @@ export class FactLog { this.tailBytes = total } - /** Cut a SEALED segment back to `committedGeneration` (atomic replace). - * v2 segments byte-slice at frame boundaries (kept frames — pads - * included — are never re-encoded); the v1 re-encode path is unchanged. */ + /** Cut a SEALED segment back to `committedGeneration` (atomic replace). */ private async truncateSegmentTo(file: string, committedGeneration: number): Promise { const path = `${FACTS_PREFIX}/${file}` const bytes = await this.storage.readRawBytes(path) if (bytes === null) return - const { facts, formatVersion } = parseSegment(file, bytes) + const { facts } = parseSegment(file, bytes) const kept = facts.filter((f) => f.generation <= committedGeneration) prodLog.warn( `[FactLog] truncating sealed segment ${file} to generation ${committedGeneration} ` + `(${facts.length - kept.length} uncommitted fact(s) dropped)` ) - if (formatVersion === FACT_LOG_FORMAT_V2) { - const cut = v2CutOffset(walkV2Frames(bytes), committedGeneration) - await this.storage.writeRawBytes(path, bytes.subarray(0, cut)) - return - } const first = kept[0]?.generation ?? this.segmentFirstGenerationFromName(file) const parts: Uint8Array[] = [buildHeader(first)] for (const f of kept) parts.push(encodeFrame(f)) diff --git a/src/db/factLogFormat.ts b/src/db/factLogFormat.ts deleted file mode 100644 index 0ac93e1f..00000000 --- a/src/db/factLogFormat.ts +++ /dev/null @@ -1,1309 +0,0 @@ -/** - * @module db/factLogFormat - * @description Fact-log format v2 (record envelope + sector seals) — the pure - * encode/decode functions for the versioned on-disk fact-log byte format. - * No I/O and no storage dependencies live here: this module is the REFERENCE - * IMPLEMENTATION of the format, and a second (native) reader parses these - * exact bytes. Byte-level behavior is a two-implementation contract — bytes - * change only behind a format-version bump, never in place. - * - * ## Segment header (32 bytes, both versions) - * - * magic "BFACTS\0\0" (8B) | formatVersion:u32 LE | firstGeneration:u64 LE | - * v1: reserved 12B (ZEROED, verified) - * v2: sealSize:u16 LE at offset +20 | reserved 10B (ZEROED, verified) - * - * V1 segments remain readable forever via the v1 decode path — never rewritten. - * - * ## Frame (unchanged from v1) - * - * payloadLength:u32 LE | crc32c:u32 LE (of payload) | msgpack payload - * - * A bad length (overruns the buffer) or CRC mismatch is a TORN TAIL: it - * terminates the scan; everything before it is intact. - * - * ## V2 fact payload (msgpack, positional — same 5 positions as v1, but - * position 2 is `records`, not v1's `ops`) - * - * fact := [ generation:u64, timestamp:u64, records, meta|nil, blobHashes|nil ] - * record := [ recordType:u8, recordVersion:u8, cipherFlag:u8, keyId:bin16|nil, - * ...type-specific fields ] - * - * `cipherFlag`/`keyId` are RESERVED crypto envelope fields: `0`/`nil` (a - * plaintext record) is the ONLY legal combination this release writes or - * reads. Any nonzero cipherFlag or non-nil keyId refuses with the typed - * {@link UnknownLogRecordError} ("encrypted records need a newer reader") — - * so record-level encryption can land later without a format-version bump on - * the one compat surface. No crypto logic exists here; the bytes are reserved - * only. Pad records (type 0) are exempt: they are skipped WHOLESALE as - * length-only filler, so their fields beyond [type, version] are never - * inspected (this keeps pad frames byte-stable across the envelope change). - * - * Record type registry (all recordVersion = 1; type-specific fields listed — - * every record carries the 4-field envelope above first): - * - * 0 pad [] — length-only filler; readers SKIP; crc-covered - * 1 noun.afterImage [id bin16, entityInt u64, metadata, vectorLeg] - * 2 noun.tombstone [id bin16] - * 3 verb.afterImage [id bin16, verbInt u64, metadata, vectorLeg, - * verb str, sourceId bin16, sourceInt u64, - * targetId bin16, targetInt u64] - * 4 verb.tombstone [id bin16] - * 5 batch.meta [metaMap] — at most ONE per fact - * 6 embed.pending [id bin16, enqueuedAt u64] - * 7 embed.landed [id bin16, vector — INLINE float[] only] - * 8 blob.manifest [hash bin32, size u64, mimeType str, refOp u8 (0=add,1=release)] - * 9 projection.note [noteMap] — opaque map, reserved consumer - * 10 bootstrap.baseline [id bin16, kind u8 (0=noun,1=verb), metadata, vectorLeg] - * 11 log.genesis [idSpaceWidth u8 (32|64), brainId bin16, createdAt u64] - * — MUST be the first record of the first fact in a - * v2 log (first-record-of-fact is enforced here; the - * first-fact-of-log half belongs to the log layer) - * - * vectorLeg := float[] | ['ref', sameAsGeneration u64] | nil - * - * Integer wire discipline (reference encoder): every field declared u64 above - * rides as msgpack uint64 (0xcf, fixed 8 bytes); u8 fields ride as minimal - * msgpack uints (positive fixint). The decoder is liberal and accepts any - * msgpack unsigned-integer width for these fields. `entityInt`/`verbInt`/ - * `sourceInt`/`targetInt` surface as `bigint` (full u64 range); scalar - * counters and timestamps surface as `number` and refuse values beyond - * `Number.MAX_SAFE_INTEGER` loudly. - * - * ## Decoder law - * - * An unknown recordType, or a recordVersion newer than this reader knows, - * throws {@link UnknownLogRecordError} — NEVER skip-and-continue (type 0 pad - * is the sole exception: skipped by definition). A log.genesis whose - * idSpaceWidth disagrees with the caller's expected width throws - * {@link GenesisWidthMismatchError} naming both widths. - * - * ## Sector seals - * - * A "sealed group" is one or more frames padded to the next `sealSize` - * boundary with ONE pad frame — a frame whose fact is - * `[0, 0, [[0, 1, filler?]], nil, nil]` (generation 0 marks filler; real - * facts start at 1). Pad frames are invisible to readers. When the gap to the - * boundary is smaller than the smallest constructible pad frame, the group is - * padded through to the boundary AFTER next (one extra sealSize) — chosen as - * the simpler correct approach over rewriting the previous frame's payload: - * input frames stay byte-immutable, alignment still holds, and the cost is at - * most one sector on a rare (<1%) size coincidence. - */ -import { encode as msgpackEncode, decode as msgpackDecode } from '@msgpack/msgpack' -import { crc32c } from '../utils/crc32c.js' -import type { CommitFact } from './factLog.js' - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -/** Segment magic: ASCII "BFACTS" + two NULs (shared by v1 and v2 headers). */ -export const FACT_SEGMENT_MAGIC: Uint8Array = new Uint8Array([ - 0x42, 0x46, 0x41, 0x43, 0x54, 0x53, 0x00, 0x00 -]) - -/** Segment format version 1 (ops-shaped facts, 12 zeroed reserved bytes). */ -export const FACT_LOG_FORMAT_V1 = 1 - -/** Segment format version 2 (record envelope + sector seals). */ -export const FACT_LOG_FORMAT_V2 = 2 - -/** Segment header size in bytes (identical for v1 and v2). */ -export const SEGMENT_HEADER_BYTES = 32 - -/** Frame prefix size: payloadLength(4) + crc32c(4). */ -export const FRAME_PREFIX_BYTES = 8 - -/** Default sector-seal size (bytes) when the caller does not probe a device. */ -export const DEFAULT_SEAL_SIZE = 4096 - -/** The record version this reader knows (all registry types are version 1). */ -export const LOG_RECORD_VERSION = 1 - -/** - * The only legal `cipherFlag` value this release: plaintext. The encoder - * always writes it (with a nil keyId); the decoder refuses anything else - * with {@link UnknownLogRecordError} — encrypted records need a newer reader. - */ -export const LOG_RECORD_CIPHER_PLAINTEXT = 0 - -/** The v2 record-type registry — wire codes for every record type. */ -export const LOG_RECORD_TYPES = { - PAD: 0, - NOUN_AFTER_IMAGE: 1, - NOUN_TOMBSTONE: 2, - VERB_AFTER_IMAGE: 3, - VERB_TOMBSTONE: 4, - BATCH_META: 5, - EMBED_PENDING: 6, - EMBED_LANDED: 7, - BLOB_MANIFEST: 8, - PROJECTION_NOTE: 9, - BOOTSTRAP_BASELINE: 10, - LOG_GENESIS: 11 -} as const - -/** A wire code from the v2 record-type registry. */ -export type LogRecordTypeCode = (typeof LOG_RECORD_TYPES)[keyof typeof LOG_RECORD_TYPES] - -const U64_MAX = (1n << 64n) - 1n - -// --------------------------------------------------------------------------- -// Errors -// --------------------------------------------------------------------------- - -/** - * A record whose type or version this reader does not know. Thrown — never - * skipped — so an old reader can NEVER silently drop data written by a newer - * writer. Carries the offending type/version for programmatic handling. - */ -export class UnknownLogRecordError extends Error { - /** The wire recordType that was not understood. */ - public readonly recordType: number - /** The wire recordVersion that was not understood. */ - public readonly recordVersion: number - - constructor(recordType: number, recordVersion: number, message: string) { - super(message) - this.name = 'UnknownLogRecordError' - this.recordType = recordType - this.recordVersion = recordVersion - } -} - -/** - * A log.genesis record whose id-space width disagrees with the width the - * caller expects. Decoding across id-space widths is refused loudly — the - * error names both widths. - */ -export class GenesisWidthMismatchError extends Error { - /** The width the caller expected (32 or 64). */ - public readonly expectedWidth: number - /** The width the genesis record declares (32 or 64). */ - public readonly actualWidth: number - - constructor(expectedWidth: number, actualWidth: number) { - super( - `fact log v2: log.genesis declares a ${actualWidth}-bit id space but this reader ` + - `expected ${expectedWidth}-bit — refusing to decode across id-space widths` - ) - this.name = 'GenesisWidthMismatchError' - this.expectedWidth = expectedWidth - this.actualWidth = actualWidth - } -} - -// --------------------------------------------------------------------------- -// Record + fact types (the TS surface of the wire registry) -// --------------------------------------------------------------------------- - -/** A vector reference: "same vector as the one generation N carried inline". */ -export interface VectorRef { - /** The generation whose record carried the INLINE vector (single-hop only). */ - sameAsGeneration: number -} - -/** A record's vector leg: inline floats, a single-hop ref, or none. */ -export type VectorLeg = number[] | VectorRef | null - -/** Type 1 — the after-image of a noun: what the entity BECAME. */ -export interface NounAfterImageRecord { - type: 'noun.afterImage' - id: string - /** The entity's u64 integer handle (full range — hence bigint). */ - entityInt: bigint - metadata: unknown - vectorLeg: VectorLeg -} - -/** Type 2 — a body-less noun tombstone: the entity was removed. */ -export interface NounTombstoneRecord { - type: 'noun.tombstone' - id: string -} - -/** Type 3 — the after-image of a verb (relationship), endpoints included. */ -export interface VerbAfterImageRecord { - type: 'verb.afterImage' - id: string - /** The verb's u64 integer handle (full range — hence bigint). */ - verbInt: bigint - metadata: unknown - vectorLeg: VectorLeg - /** The verb name (relationship type). */ - verb: string - sourceId: string - sourceInt: bigint - targetId: string - targetInt: bigint -} - -/** Type 4 — a body-less verb tombstone: the relationship was removed. */ -export interface VerbTombstoneRecord { - type: 'verb.tombstone' - id: string -} - -/** Type 5 — batch-level metadata; at most ONE per fact. */ -export interface BatchMetaRecord { - type: 'batch.meta' - meta: Record -} - -/** Type 6 — an embedding was enqueued for the id (vector not yet available). */ -export interface EmbedPendingRecord { - type: 'embed.pending' - id: string - /** Enqueue time (epoch ms). */ - enqueuedAt: number -} - -/** Type 7 — a deferred embedding landed; carries the INLINE vector only. */ -export interface EmbedLandedRecord { - type: 'embed.landed' - id: string - /** The landed vector — inline floats only; refs are not allowed here. */ - vector: number[] -} - -/** Type 8 — a blob reference-count event (content-addressed by hash). */ -export interface BlobManifestRecord { - type: 'blob.manifest' - /** The blob's content hash — 64 lowercase hex chars (bin32 on the wire). */ - hash: string - size: number - mimeType: string - refOp: 'add' | 'release' -} - -/** Type 9 — an opaque note for a reserved projection consumer. */ -export interface ProjectionNoteRecord { - type: 'projection.note' - note: Record -} - -/** Type 10 — a bootstrap baseline row (initial-load after-image). */ -export interface BootstrapBaselineRecord { - type: 'bootstrap.baseline' - id: string - kind: 'noun' | 'verb' - metadata: unknown - vectorLeg: VectorLeg -} - -/** Type 11 — the log's birth certificate; first record of the first fact. */ -export interface LogGenesisRecord { - type: 'log.genesis' - /** The integer-handle width this log's records use. */ - idSpaceWidth: 32 | 64 - brainId: string - /** Creation time (epoch ms). */ - createdAt: number -} - -/** Any decodable v2 record (pads are skipped, never surfaced). */ -export type LogRecord = - | NounAfterImageRecord - | NounTombstoneRecord - | VerbAfterImageRecord - | VerbTombstoneRecord - | BatchMetaRecord - | EmbedPendingRecord - | EmbedLandedRecord - | BlobManifestRecord - | ProjectionNoteRecord - | BootstrapBaselineRecord - | LogGenesisRecord - -/** One committed generation in v2 shape: a record envelope, not v1 ops. */ -export interface CommitFactV2 { - generation: number - timestamp: number - records: LogRecord[] - meta?: Record - blobHashes?: string[] -} - -/** A parsed segment header (v1 has no sealSize; v2 always carries one). */ -export interface SegmentHeader { - formatVersion: number - firstGeneration: number - /** Sector-seal size (v2 only) — `undefined` on v1 headers. */ - sealSize?: number -} - -/** Options for {@link encodeFactV2}. */ -export interface EncodeFactV2Options { - /** - * Single-hop validator for vector refs: the set (or predicate) of - * generations whose records carried an INLINE vector. REQUIRED whenever any - * record carries a `VectorRef` — encoding an unverifiable ref is refused. - */ - inlineVectorGenerations?: Set | ((generation: number) => boolean) -} - -/** Options for the v2 decode path of {@link decodeFact}. */ -export interface DecodeFactV2Options { - /** - * The id-space width the caller expects. When set and the fact carries a - * log.genesis record, a disagreeing width throws - * {@link GenesisWidthMismatchError}. - */ - expectedIdSpaceWidth?: 32 | 64 -} - -/** The result of decoding a frame group: intact facts + valid byte length. */ -export interface DecodedFrameGroup { - facts: CommitFactV2[] - /** Byte length of the intact prefix (whole frames that decoded cleanly). */ - validBytes: number -} - -// --------------------------------------------------------------------------- -// msgpack wire helpers -// --------------------------------------------------------------------------- - -/** - * The v2 codec: `useBigInt64` makes bigints ride as fixed 8-byte uint64/int64 - * (the u64 wire discipline) while JS numbers keep exact-value round-trips - * (integers ≤ 32-bit ride minimal; larger numbers ride float64, which holds - * every safe integer exactly). - */ -const enc = (value: unknown): Uint8Array => msgpackEncode(value, { useBigInt64: true }) -const dec = (bytes: Uint8Array): unknown => msgpackDecode(bytes, { useBigInt64: true }) - -/** Coerce an encode-side u64 field to bigint, refusing out-of-range values. */ -function toWireU64(value: number | bigint, field: string): bigint { - let big: bigint - if (typeof value === 'bigint') { - big = value - } else if (Number.isSafeInteger(value)) { - big = BigInt(value) - } else { - throw new Error(`fact log v2: ${field} must be a safe integer or bigint; got ${value}`) - } - if (big < 0n || big > U64_MAX) { - throw new Error(`fact log v2: ${field} is out of u64 range: ${big}`) - } - return big -} - -/** Decode-side u64 → bigint (liberal: accepts any msgpack uint width). */ -function wireToBigint(value: unknown, field: string): bigint { - if (typeof value === 'bigint') { - if (value < 0n || value > U64_MAX) { - throw new Error(`fact log v2: ${field} is out of u64 range: ${value}`) - } - return value - } - if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) { - return BigInt(value) - } - throw new Error(`fact log v2: ${field} is not an unsigned integer`) -} - -/** Decode-side u64 → number, refusing values beyond safe-integer range. */ -function wireToNumber(value: unknown, field: string): number { - const big = wireToBigint(value, field) - if (big > BigInt(Number.MAX_SAFE_INTEGER)) { - throw new Error(`fact log v2: ${field} ${big} exceeds Number.MAX_SAFE_INTEGER`) - } - return Number(big) -} - -/** Decode-side u8 (record types, kinds, flags). */ -function wireToU8(value: unknown, field: string): number { - const n = typeof value === 'bigint' ? Number(value) : value - if (typeof n !== 'number' || !Number.isInteger(n) || n < 0 || n > 255) { - throw new Error(`fact log v2: ${field} is not a u8`) - } - return n -} - -/** uuid string → 16 raw bytes (bin16 on the wire). */ -function uuidToBytes(id: string): Uint8Array { - const hex = id.replace(/-/g, '') - if (hex.length !== 32 || /[^0-9a-fA-F]/.test(hex)) { - throw new Error(`fact log v2: id is not a uuid: ${id}`) - } - const bytes = new Uint8Array(16) - for (let i = 0; i < 16; i++) { - bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) - } - return bytes -} - -/** 16 raw bytes → canonical lowercase uuid string. */ -function bytesToUuid(bytes: unknown, field: string): string { - if (!(bytes instanceof Uint8Array) || bytes.length !== 16) { - throw new Error(`fact log v2: ${field} is not a bin16 id`) - } - let hex = '' - for (let i = 0; i < 16; i++) hex += bytes[i].toString(16).padStart(2, '0') - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` -} - -/** 64-hex-char content hash → 32 raw bytes (bin32 on the wire). */ -function hashToBytes(hash: string): Uint8Array { - if (typeof hash !== 'string' || !/^[0-9a-fA-F]{64}$/.test(hash)) { - throw new Error(`fact log v2: blob hash must be 64 hex chars; got ${String(hash).slice(0, 80)}`) - } - const bytes = new Uint8Array(32) - for (let i = 0; i < 32; i++) { - bytes[i] = parseInt(hash.slice(i * 2, i * 2 + 2), 16) - } - return bytes -} - -/** 32 raw bytes → 64-char lowercase hex content hash. */ -function bytesToHash(bytes: unknown): string { - if (!(bytes instanceof Uint8Array) || bytes.length !== 32) { - throw new Error('fact log v2: blob hash is not bin32') - } - let hex = '' - for (let i = 0; i < 32; i++) hex += bytes[i].toString(16).padStart(2, '0') - return hex -} - -/** True for a plain map object (not null/array/binary). */ -function isPlainMap(value: unknown): value is Record { - return ( - typeof value === 'object' && - value !== null && - !Array.isArray(value) && - !(value instanceof Uint8Array) - ) -} - -// --------------------------------------------------------------------------- -// Segment header (v1 read + v2 read/write) -// --------------------------------------------------------------------------- - -/** - * Build a v2 segment header: magic + formatVersion 2 + firstGeneration u64 LE - * + sealSize u16 LE at offset +20. The remaining 10 reserved bytes stay zero - * and are verified by every reader. - * - * @param firstGeneration - The first generation this segment will hold. - * @param sealSize - The sector-seal size groups in this segment align to - * (device atomic-write probing is the caller's business; default 4096). - */ -export function encodeSegmentHeaderV2( - firstGeneration: number, - sealSize: number = DEFAULT_SEAL_SIZE -): Uint8Array { - if (!Number.isSafeInteger(firstGeneration) || firstGeneration < 0) { - throw new Error(`fact log v2: firstGeneration must be a non-negative integer; got ${firstGeneration}`) - } - assertValidSealSize(sealSize) - const header = new Uint8Array(SEGMENT_HEADER_BYTES) - header.set(FACT_SEGMENT_MAGIC, 0) - const view = new DataView(header.buffer) - view.setUint32(8, FACT_LOG_FORMAT_V2, true) - view.setBigUint64(12, BigInt(firstGeneration), true) - view.setUint16(20, sealSize, true) - // bytes 22..31 stay zero (reserved, verified) - return header -} - -/** - * Parse a segment header — reads BOTH v1 (version 1, twelve zeroed reserved - * bytes, no sealSize) and v2 (version 2, sealSize u16 LE at +20, ten zeroed - * reserved bytes). Bad magic, non-zero reserved bytes, or an unknown version - * throw loudly; nothing is guessed. - * - * @param bytes - At least the first {@link SEGMENT_HEADER_BYTES} of a segment. - * @returns The parsed header; `sealSize` is `undefined` for v1 headers. - */ -export function parseSegmentHeader(bytes: Uint8Array): SegmentHeader { - if (bytes.length < SEGMENT_HEADER_BYTES) { - throw new Error( - `fact log: segment header needs ${SEGMENT_HEADER_BYTES} bytes; got ${bytes.length}` - ) - } - for (let i = 0; i < FACT_SEGMENT_MAGIC.length; i++) { - if (bytes[i] !== FACT_SEGMENT_MAGIC[i]) { - throw new Error('fact log: bad magic — not a fact segment') - } - } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - const formatVersion = view.getUint32(8, true) - const firstGenerationBig = view.getBigUint64(12, true) - if (firstGenerationBig > BigInt(Number.MAX_SAFE_INTEGER)) { - throw new Error(`fact log: firstGeneration ${firstGenerationBig} exceeds Number.MAX_SAFE_INTEGER`) - } - const firstGeneration = Number(firstGenerationBig) - - if (formatVersion === FACT_LOG_FORMAT_V1) { - assertReservedZero(bytes, 20) - return { formatVersion, firstGeneration } - } - if (formatVersion === FACT_LOG_FORMAT_V2) { - const sealSize = view.getUint16(20, true) - assertReservedZero(bytes, 22) - return { formatVersion, firstGeneration, sealSize } - } - throw new Error( - `fact log: segment formatVersion ${formatVersion}; this build reads 1 and 2 — ` + - `a newer reader is required` - ) -} - -/** Verify header bytes [from, 32) are zero — anything else is unverifiable. */ -function assertReservedZero(bytes: Uint8Array, from: number): void { - for (let i = from; i < SEGMENT_HEADER_BYTES; i++) { - if (bytes[i] !== 0) { - throw new Error('fact log: non-zero reserved header bytes — unverifiable') - } - } -} - -/** Refuse seal sizes the header cannot carry or a pad frame cannot fill. */ -function assertValidSealSize(sealSize: number): void { - if (!Number.isInteger(sealSize) || sealSize < 64 || sealSize > 0xffff) { - throw new Error( - `fact log v2: sealSize must be an integer in [64, 65535]; got ${sealSize}` - ) - } -} - -// --------------------------------------------------------------------------- -// Frames -// --------------------------------------------------------------------------- - -/** Wrap a msgpack payload in the frame envelope (length + crc32c + payload). */ -function buildFrame(payload: Uint8Array): Uint8Array { - const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) - const view = new DataView(frame.buffer) - view.setUint32(0, payload.length, true) - view.setUint32(4, crc32c(payload), true) - frame.set(payload, FRAME_PREFIX_BYTES) - return frame -} - -/** - * Verify a complete frame (exact length, CRC) and return its msgpack payload - * (a view into the frame — copy if you outlive the frame). The bridge between - * frame-level producers ({@link encodeFactV2}, {@link sealGroup}) and the - * payload-level {@link decodeFact}. - */ -export function framePayload(frame: Uint8Array): Uint8Array { - if (frame.length < FRAME_PREFIX_BYTES) { - throw new Error(`fact log: frame shorter than its ${FRAME_PREFIX_BYTES}-byte prefix`) - } - const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength) - const length = view.getUint32(0, true) - if (FRAME_PREFIX_BYTES + length !== frame.length) { - throw new Error( - `fact log: frame declares ${length} payload bytes but carries ${frame.length - FRAME_PREFIX_BYTES}` - ) - } - const payload = frame.subarray(FRAME_PREFIX_BYTES) - const expectedCrc = view.getUint32(4, true) - if (crc32c(payload) !== expectedCrc) { - throw new Error('fact log: frame payload fails its crc32c') - } - return payload -} - -// --------------------------------------------------------------------------- -// vectorLeg encode/decode -// --------------------------------------------------------------------------- - -/** Encode a vector leg; refs must pass the single-hop validator. */ -function encodeVectorLeg( - leg: VectorLeg | undefined, - options: EncodeFactV2Options | undefined, - context: string -): unknown { - if (leg === null || leg === undefined) return null - if (Array.isArray(leg)) { - for (const value of leg) { - if (typeof value !== 'number') { - throw new Error(`fact log v2: ${context} inline vector has a non-number element`) - } - } - return leg - } - if (isPlainMap(leg) && typeof (leg as VectorRef).sameAsGeneration === 'number') { - const target = (leg as VectorRef).sameAsGeneration - const validator = options?.inlineVectorGenerations - if (!validator) { - throw new Error( - `fact log v2: ${context} carries a vector ref to generation ${target} but no ` + - `single-hop validator was provided — refusing to encode an unverifiable ref` - ) - } - const targetIsInline = typeof validator === 'function' ? validator(target) : validator.has(target) - if (!targetIsInline) { - throw new Error( - `fact log v2: ${context} vector ref targets generation ${target}, which did not ` + - `carry an inline vector — refs must be single-hop` - ) - } - return ['ref', toWireU64(target, `${context} sameAsGeneration`)] - } - throw new Error(`fact log v2: ${context} has a malformed vector leg`) -} - -/** Decode a vector leg: floats, a single-hop ref, or null. */ -function decodeVectorLeg(wire: unknown, context: string): VectorLeg { - if (wire === null || wire === undefined) return null - if (Array.isArray(wire)) { - if (wire.length === 2 && wire[0] === 'ref') { - return { sameAsGeneration: wireToNumber(wire[1], `${context} sameAsGeneration`) } - } - return wire.map((value, i) => { - if (typeof value === 'number') return value - if (typeof value === 'bigint') return Number(value) - throw new Error(`fact log v2: ${context} vector element ${i} is not a number`) - }) - } - throw new Error(`fact log v2: ${context} has a malformed vector leg`) -} - -// --------------------------------------------------------------------------- -// Record encode/decode -// --------------------------------------------------------------------------- - -/** - * Encode one record into its positional wire array. Every record leads with - * the 4-field envelope [type, version, cipherFlag, keyId]; this release - * writes cipherFlag {@link LOG_RECORD_CIPHER_PLAINTEXT} and a nil keyId - * always (the fields are crypto-RESERVED, carrying no logic yet). - */ -function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefined): unknown[] { - const T = LOG_RECORD_TYPES - const V = LOG_RECORD_VERSION - const C = LOG_RECORD_CIPHER_PLAINTEXT - const K = null // keyId: nil until record-level encryption exists - switch (record.type) { - case 'noun.afterImage': - return [ - T.NOUN_AFTER_IMAGE, - V, - C, - K, - uuidToBytes(record.id), - toWireU64(record.entityInt, 'entityInt'), - record.metadata ?? null, - encodeVectorLeg(record.vectorLeg, options, `noun.afterImage ${record.id}`) - ] - case 'noun.tombstone': - return [T.NOUN_TOMBSTONE, V, C, K, uuidToBytes(record.id)] - case 'verb.afterImage': { - if (typeof record.verb !== 'string' || record.verb.length === 0) { - throw new Error(`fact log v2: verb.afterImage ${record.id} needs a non-empty verb name`) - } - return [ - T.VERB_AFTER_IMAGE, - V, - C, - K, - uuidToBytes(record.id), - toWireU64(record.verbInt, 'verbInt'), - record.metadata ?? null, - encodeVectorLeg(record.vectorLeg, options, `verb.afterImage ${record.id}`), - record.verb, - uuidToBytes(record.sourceId), - toWireU64(record.sourceInt, 'sourceInt'), - uuidToBytes(record.targetId), - toWireU64(record.targetInt, 'targetInt') - ] - } - case 'verb.tombstone': - return [T.VERB_TOMBSTONE, V, C, K, uuidToBytes(record.id)] - case 'batch.meta': - if (!isPlainMap(record.meta)) { - throw new Error('fact log v2: batch.meta requires a map') - } - return [T.BATCH_META, V, C, K, record.meta] - case 'embed.pending': - return [ - T.EMBED_PENDING, - V, - C, - K, - uuidToBytes(record.id), - toWireU64(record.enqueuedAt, 'enqueuedAt') - ] - case 'embed.landed': { - if (!Array.isArray(record.vector) || record.vector.some((v) => typeof v !== 'number')) { - throw new Error( - `fact log v2: embed.landed ${record.id} carries an INLINE float vector only — ` + - `refs and nil are not allowed here` - ) - } - return [T.EMBED_LANDED, V, C, K, uuidToBytes(record.id), record.vector] - } - case 'blob.manifest': { - if (typeof record.mimeType !== 'string') { - throw new Error('fact log v2: blob.manifest mimeType must be a string') - } - if (record.refOp !== 'add' && record.refOp !== 'release') { - throw new Error(`fact log v2: blob.manifest refOp must be 'add' or 'release'`) - } - return [ - T.BLOB_MANIFEST, - V, - C, - K, - hashToBytes(record.hash), - toWireU64(record.size, 'blob size'), - record.mimeType, - record.refOp === 'add' ? 0 : 1 - ] - } - case 'projection.note': - if (!isPlainMap(record.note)) { - throw new Error('fact log v2: projection.note requires a map') - } - return [T.PROJECTION_NOTE, V, C, K, record.note] - case 'bootstrap.baseline': { - if (record.kind !== 'noun' && record.kind !== 'verb') { - throw new Error(`fact log v2: bootstrap.baseline kind must be 'noun' or 'verb'`) - } - return [ - T.BOOTSTRAP_BASELINE, - V, - C, - K, - uuidToBytes(record.id), - record.kind === 'noun' ? 0 : 1, - record.metadata ?? null, - encodeVectorLeg(record.vectorLeg, options, `bootstrap.baseline ${record.id}`) - ] - } - case 'log.genesis': { - if (record.idSpaceWidth !== 32 && record.idSpaceWidth !== 64) { - throw new Error( - `fact log v2: log.genesis idSpaceWidth must be 32 or 64; got ${record.idSpaceWidth}` - ) - } - return [ - T.LOG_GENESIS, - V, - C, - K, - record.idSpaceWidth, - uuidToBytes(record.brainId), - toWireU64(record.createdAt, 'createdAt') - ] - } - default: { - // Pads are the sealer's business ({@link sealGroup}); anything else - // here is an unencodable record — refuse instead of writing bytes a - // reader would have to guess about. - const unknown = record as { type?: unknown } - throw new Error(`fact log v2: cannot encode record type ${String(unknown.type)}`) - } - } -} - -/** Exact wire arity per record type (envelope of 4 + type-specific fields). */ -const RECORD_ARITY: Record = { - [LOG_RECORD_TYPES.NOUN_AFTER_IMAGE]: 8, - [LOG_RECORD_TYPES.NOUN_TOMBSTONE]: 5, - [LOG_RECORD_TYPES.VERB_AFTER_IMAGE]: 13, - [LOG_RECORD_TYPES.VERB_TOMBSTONE]: 5, - [LOG_RECORD_TYPES.BATCH_META]: 5, - [LOG_RECORD_TYPES.EMBED_PENDING]: 6, - [LOG_RECORD_TYPES.EMBED_LANDED]: 6, - [LOG_RECORD_TYPES.BLOB_MANIFEST]: 8, - [LOG_RECORD_TYPES.PROJECTION_NOTE]: 5, - [LOG_RECORD_TYPES.BOOTSTRAP_BASELINE]: 8, - [LOG_RECORD_TYPES.LOG_GENESIS]: 7 -} - -/** - * Decode one wire record. Returns `null` for pads (skipped by definition). - * Unknown type / newer version throw {@link UnknownLogRecordError} — never - * skip-and-continue. The reserved crypto envelope is verified BEFORE the - * arity check (an encrypted record's field layout is a newer reader's - * business, not a malformed-record error): any nonzero cipherFlag or non-nil - * keyId refuses with the same typed error class. - */ -function decodeRecord(raw: unknown): LogRecord | null { - if (!Array.isArray(raw) || raw.length < 2) { - throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])') - } - const recordType = wireToU8(raw[0], 'recordType') - const recordVersion = wireToU8(raw[1], 'recordVersion') - - if (recordType === LOG_RECORD_TYPES.PAD) { - // Length-only filler: skipped wholesale, filler fields never inspected - // (pads therefore carry no crypto envelope — by definition, not omission). - return null - } - const arity = RECORD_ARITY[recordType] - if (arity === undefined) { - throw new UnknownLogRecordError( - recordType, - recordVersion, - `fact log v2: unknown record type ${recordType} (record version ${recordVersion}) — ` + - `a newer reader is required to decode this log` - ) - } - if (recordVersion > LOG_RECORD_VERSION) { - throw new UnknownLogRecordError( - recordType, - recordVersion, - `fact log v2: record type ${recordType} carries record version ${recordVersion}; ` + - `this reader knows version ${LOG_RECORD_VERSION} — a newer reader is required to decode this log` - ) - } - if (recordVersion !== LOG_RECORD_VERSION) { - throw new Error(`fact log v2: record type ${recordType} has invalid record version ${recordVersion}`) - } - if (raw.length < 4) { - throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])') - } - const cipherFlag = wireToU8(raw[2], 'cipherFlag') - const keyId = raw[3] - if (cipherFlag !== LOG_RECORD_CIPHER_PLAINTEXT || (keyId !== null && keyId !== undefined)) { - throw new UnknownLogRecordError( - recordType, - recordVersion, - `fact log v2: record type ${recordType} carries cipherFlag ${cipherFlag}` + - `${keyId !== null && keyId !== undefined ? ' and a keyId' : ''} — ` + - `encrypted records need a newer reader` - ) - } - if (raw.length !== arity) { - throw new Error( - `fact log v2: record type ${recordType} expects ${arity} wire fields; got ${raw.length}` - ) - } - - switch (recordType) { - case LOG_RECORD_TYPES.NOUN_AFTER_IMAGE: - return { - type: 'noun.afterImage', - id: bytesToUuid(raw[4], 'noun.afterImage id'), - entityInt: wireToBigint(raw[5], 'entityInt'), - metadata: raw[6] ?? null, - vectorLeg: decodeVectorLeg(raw[7], 'noun.afterImage') - } - case LOG_RECORD_TYPES.NOUN_TOMBSTONE: - return { type: 'noun.tombstone', id: bytesToUuid(raw[4], 'noun.tombstone id') } - case LOG_RECORD_TYPES.VERB_AFTER_IMAGE: { - if (typeof raw[8] !== 'string') { - throw new Error('fact log v2: verb.afterImage verb name is not a string') - } - return { - type: 'verb.afterImage', - id: bytesToUuid(raw[4], 'verb.afterImage id'), - verbInt: wireToBigint(raw[5], 'verbInt'), - metadata: raw[6] ?? null, - vectorLeg: decodeVectorLeg(raw[7], 'verb.afterImage'), - verb: raw[8], - sourceId: bytesToUuid(raw[9], 'verb.afterImage sourceId'), - sourceInt: wireToBigint(raw[10], 'sourceInt'), - targetId: bytesToUuid(raw[11], 'verb.afterImage targetId'), - targetInt: wireToBigint(raw[12], 'targetInt') - } - } - case LOG_RECORD_TYPES.VERB_TOMBSTONE: - return { type: 'verb.tombstone', id: bytesToUuid(raw[4], 'verb.tombstone id') } - case LOG_RECORD_TYPES.BATCH_META: { - if (!isPlainMap(raw[4])) throw new Error('fact log v2: batch.meta payload is not a map') - return { type: 'batch.meta', meta: raw[4] } - } - case LOG_RECORD_TYPES.EMBED_PENDING: - return { - type: 'embed.pending', - id: bytesToUuid(raw[4], 'embed.pending id'), - enqueuedAt: wireToNumber(raw[5], 'enqueuedAt') - } - case LOG_RECORD_TYPES.EMBED_LANDED: { - const leg = decodeVectorLeg(raw[5], 'embed.landed') - if (!Array.isArray(leg)) { - throw new Error( - 'fact log v2: embed.landed must carry an INLINE float vector — refs and nil are not allowed here' - ) - } - return { type: 'embed.landed', id: bytesToUuid(raw[4], 'embed.landed id'), vector: leg } - } - case LOG_RECORD_TYPES.BLOB_MANIFEST: { - if (typeof raw[6] !== 'string') { - throw new Error('fact log v2: blob.manifest mimeType is not a string') - } - const refOp = wireToU8(raw[7], 'refOp') - if (refOp !== 0 && refOp !== 1) { - throw new Error(`fact log v2: blob.manifest refOp must be 0 (add) or 1 (release); got ${refOp}`) - } - return { - type: 'blob.manifest', - hash: bytesToHash(raw[4]), - size: wireToNumber(raw[5], 'blob size'), - mimeType: raw[6], - refOp: refOp === 0 ? 'add' : 'release' - } - } - case LOG_RECORD_TYPES.PROJECTION_NOTE: { - if (!isPlainMap(raw[4])) throw new Error('fact log v2: projection.note payload is not a map') - return { type: 'projection.note', note: raw[4] } - } - case LOG_RECORD_TYPES.BOOTSTRAP_BASELINE: { - const kind = wireToU8(raw[5], 'bootstrap.baseline kind') - if (kind !== 0 && kind !== 1) { - throw new Error(`fact log v2: bootstrap.baseline kind must be 0 (noun) or 1 (verb); got ${kind}`) - } - return { - type: 'bootstrap.baseline', - id: bytesToUuid(raw[4], 'bootstrap.baseline id'), - kind: kind === 0 ? 'noun' : 'verb', - metadata: raw[6] ?? null, - vectorLeg: decodeVectorLeg(raw[7], 'bootstrap.baseline') - } - } - case LOG_RECORD_TYPES.LOG_GENESIS: { - const width = wireToU8(raw[4], 'idSpaceWidth') - if (width !== 32 && width !== 64) { - throw new Error(`fact log v2: log.genesis idSpaceWidth must be 32 or 64; got ${width}`) - } - return { - type: 'log.genesis', - idSpaceWidth: width, - brainId: bytesToUuid(raw[5], 'log.genesis brainId'), - createdAt: wireToNumber(raw[6], 'createdAt') - } - } - default: - // Unreachable: every arity-table type is handled above. - throw new Error(`fact log v2: unhandled record type ${recordType}`) - } -} - -// --------------------------------------------------------------------------- -// Fact encode/decode -// --------------------------------------------------------------------------- - -/** - * Encode one committed generation as a complete v2 FRAME (length + crc32c + - * msgpack payload) ready for appending or sealing. - * - * Writer-enforced invariants (refusals, never silent fixes): at least one - * record; no pad records (pads belong to {@link sealGroup}); at most one - * batch.meta; log.genesis only as the first record; vector refs only with a - * passing single-hop validator; embed.landed vectors inline only. - * - * @param fact - The fact to encode (generation ≥ 1; generation 0 marks filler). - * @param options - Single-hop validation for vector refs. - * @returns The complete frame bytes. - */ -export function encodeFactV2(fact: CommitFactV2, options?: EncodeFactV2Options): Uint8Array { - if (!Number.isSafeInteger(fact.generation) || fact.generation < 1) { - throw new Error(`fact log v2: generation must be a positive integer; got ${fact.generation}`) - } - if (!Number.isSafeInteger(fact.timestamp) || fact.timestamp < 0) { - throw new Error(`fact log v2: timestamp must be a non-negative integer; got ${fact.timestamp}`) - } - // records MAY be empty: a committed generation whose ops all collapsed - // (e.g. a batch whose relates deduped to no-ops) is still a real - // generation — v1 encoded empty ops the same way; refusing here would - // fork the two formats' commit semantics. - if (!Array.isArray(fact.records)) { - throw new Error('fact log v2: records must be an array') - } - if (fact.meta !== undefined && !isPlainMap(fact.meta)) { - throw new Error('fact log v2: fact meta must be a map when present') - } - if ( - fact.blobHashes !== undefined && - (!Array.isArray(fact.blobHashes) || fact.blobHashes.some((h) => typeof h !== 'string')) - ) { - throw new Error('fact log v2: blobHashes must be an array of strings when present') - } - - let batchMetaCount = 0 - const wireRecords = fact.records.map((record, index) => { - if (record.type === 'batch.meta' && ++batchMetaCount > 1) { - throw new Error('fact log v2: at most one batch.meta record per fact') - } - if (record.type === 'log.genesis' && index !== 0) { - throw new Error('fact log v2: log.genesis must be the first record of its fact') - } - return encodeRecord(record, options) - }) - - const payload = enc([ - toWireU64(fact.generation, 'generation'), - toWireU64(fact.timestamp, 'timestamp'), - wireRecords, - fact.meta ?? null, - fact.blobHashes && fact.blobHashes.length > 0 ? fact.blobHashes : null - ]) - return buildFrame(payload) -} - -/** - * Decode one fact PAYLOAD (the msgpack bytes inside a frame — see - * {@link framePayload}). The segment's formatVersion, read from its header, - * selects the schema: version 1 decodes the v1 ops shape into a - * {@link CommitFact}; version 2 decodes the record envelope into a - * {@link CommitFactV2}. Any other version is refused. - */ -export function decodeFact(payload: Uint8Array, segmentFormatVersion: 1): CommitFact -export function decodeFact( - payload: Uint8Array, - segmentFormatVersion: 2, - options?: DecodeFactV2Options -): CommitFactV2 -export function decodeFact( - payload: Uint8Array, - segmentFormatVersion: number, - options?: DecodeFactV2Options -): CommitFact | CommitFactV2 -export function decodeFact( - payload: Uint8Array, - segmentFormatVersion: number, - options?: DecodeFactV2Options -): CommitFact | CommitFactV2 { - if (segmentFormatVersion === FACT_LOG_FORMAT_V1) return decodeFactV1(payload) - if (segmentFormatVersion === FACT_LOG_FORMAT_V2) return decodeFactV2(payload, options) - throw new Error( - `fact log: no decoder for segment formatVersion ${segmentFormatVersion} — this build reads 1 and 2` - ) -} - -/** - * The v1 decode path — byte-identical in behavior to the v1 log's own - * decoder (positional ops, bin16 ids, body-less tombstones). Kept here so v1 - * segments stay readable through the same entry point forever. - */ -function decodeFactV1(payload: Uint8Array): CommitFact { - const raw = msgpackDecode(payload) as unknown[] - const [generation, timestamp, ops, meta, blobHashes] = raw as [ - number, - number, - Array<[number, Uint8Array, [unknown, unknown] | null]>, - Record | null, - string[] | null - ] - return { - generation: Number(generation), - timestamp: Number(timestamp), - ops: ops.map(([kind, idBytes, record]) => ({ - kind: kind === 0 ? ('noun' as const) : ('verb' as const), - id: bytesToUuid(idBytes, 'op id'), - record: record === null ? null : { metadata: record[0] ?? null, vector: record[1] ?? null } - })), - ...(meta ? { meta } : {}), - ...(blobHashes && blobHashes.length > 0 ? { blobHashes } : {}) - } -} - -/** The v2 decode path: record envelope, decoder-law enforcement, pad skip. */ -function decodeFactV2(payload: Uint8Array, options?: DecodeFactV2Options): CommitFactV2 { - const raw = dec(payload) - if (!Array.isArray(raw) || raw.length !== 5) { - throw new Error('fact log v2: fact payload must be a positional array of 5') - } - const [genWire, tsWire, recordsWire, metaWire, blobsWire] = raw - if (!Array.isArray(recordsWire)) { - throw new Error('fact log v2: fact records position is not an array') - } - - const records: LogRecord[] = [] - let batchMetaCount = 0 - recordsWire.forEach((rawRecord, index) => { - const record = decodeRecord(rawRecord) - if (record === null) return // pad: length-only filler, skipped by definition - if (record.type === 'log.genesis') { - if (index !== 0) { - throw new Error('fact log v2: log.genesis must be the first record of its fact') - } - const expected = options?.expectedIdSpaceWidth - if (expected !== undefined && record.idSpaceWidth !== expected) { - throw new GenesisWidthMismatchError(expected, record.idSpaceWidth) - } - } - if (record.type === 'batch.meta' && ++batchMetaCount > 1) { - throw new Error('fact log v2: at most one batch.meta record per fact') - } - records.push(record) - }) - - let meta: Record | undefined - if (metaWire !== null && metaWire !== undefined) { - if (!isPlainMap(metaWire)) throw new Error('fact log v2: fact meta position is not a map') - meta = metaWire - } - let blobHashes: string[] | undefined - if (blobsWire !== null && blobsWire !== undefined) { - if (!Array.isArray(blobsWire) || blobsWire.some((h) => typeof h !== 'string')) { - throw new Error('fact log v2: fact blobHashes position is not a string array') - } - blobHashes = blobsWire - } - - return { - generation: wireToNumber(genWire, 'generation'), - timestamp: wireToNumber(tsWire, 'timestamp'), - records, - ...(meta ? { meta } : {}), - ...(blobHashes && blobHashes.length > 0 ? { blobHashes } : {}) - } -} - -// --------------------------------------------------------------------------- -// Sector seals -// --------------------------------------------------------------------------- - -/** - * Smallest constructible pad frame in bytes (frame prefix + the bare pad - * record fact), memoized. Exported for streaming writers that pad an - * append-only tail to a seal boundary: a gap smaller than this cannot hold - * any frame, so the writer pads through one extra sector (the same rule - * {@link sealGroup} applies). - */ -let minPadFrameBytesMemo: number | null = null -export function minPadFrameBytes(): number { - if (minPadFrameBytesMemo === null) { - minPadFrameBytesMemo = - FRAME_PREFIX_BYTES + - enc([0n, 0n, [[LOG_RECORD_TYPES.PAD, LOG_RECORD_VERSION]], null, null]).length - } - return minPadFrameBytesMemo -} - -/** - * Build a pad frame of EXACTLY `totalBytes`: a filler fact - * `[0, 0, [[0, 1, filler?]], nil, nil]` sized via a binary filler field. - * Readers skip pad records by definition, so filler fields are never - * inspected — only their length matters. - */ -function buildPadFrame(totalBytes: number): Uint8Array { - const targetPayload = totalBytes - FRAME_PREFIX_BYTES - const attempt = (record: unknown[]): Uint8Array => enc([0n, 0n, [record], null, null]) - - let payload = attempt([LOG_RECORD_TYPES.PAD, LOG_RECORD_VERSION]) - if (payload.length !== targetPayload) { - // One byte short: a fixint filler adds exactly one byte. - payload = attempt([LOG_RECORD_TYPES.PAD, LOG_RECORD_VERSION, 0]) - } - if (payload.length !== targetPayload) { - // Binary filler: msgpack bin grows byte-for-byte within a size class; - // iterate to absorb the class-header steps (bin8 → bin16 → bin32). - let fillerLength = Math.max(0, targetPayload - payload.length - 1) - let converged = false - for (let i = 0; i < 8; i++) { - const candidate = attempt([ - LOG_RECORD_TYPES.PAD, - LOG_RECORD_VERSION, - new Uint8Array(fillerLength) - ]) - const diff = targetPayload - candidate.length - if (diff === 0) { - payload = candidate - converged = true - break - } - fillerLength += diff - if (fillerLength < 0) break - } - if (!converged) { - throw new Error(`fact log v2: a pad frame of ${totalBytes} bytes is not constructible`) - } - } - return buildFrame(payload) -} - -/** - * Build a pad frame of EXACTLY `totalBytes` — the streaming-append counterpart - * of {@link sealGroup} for writers that append pads directly to a live tail - * instead of sealing an in-memory group. Refuses sizes smaller than the - * smallest constructible pad frame ({@link minPadFrameBytes}); readers skip - * the result by definition (a type-0 record is length-only filler). - * - * @param totalBytes - The exact frame size to construct (prefix included). - * @returns The complete pad frame bytes. - */ -export function encodePadFrame(totalBytes: number): Uint8Array { - if (!Number.isInteger(totalBytes) || totalBytes < minPadFrameBytes()) { - throw new Error( - `fact log v2: a pad frame must be at least ${minPadFrameBytes()} bytes; got ${totalBytes}` - ) - } - return buildPadFrame(totalBytes) -} - -/** - * Seal a group of frames to a sector boundary: concatenate the frames and pad - * to the next `sealSize` multiple with ONE pad frame. An already-aligned - * group gets no pad. When the gap is smaller than the smallest constructible - * pad frame, the group is padded through to the boundary AFTER next (one - * extra sealSize) — input frames are never rewritten. - * - * @param frames - Complete, well-formed frames (verified; garbage is refused). - * @param sealSize - The sector-seal size (device probing is the caller's - * business; default {@link DEFAULT_SEAL_SIZE}). - * @returns The sector-aligned group (`length % sealSize === 0`). - */ -export function sealGroup(frames: Uint8Array[], sealSize: number = DEFAULT_SEAL_SIZE): Uint8Array { - assertValidSealSize(sealSize) - if (!Array.isArray(frames) || frames.length === 0) { - throw new Error('fact log v2: sealGroup needs at least one frame') - } - frames.forEach((frame, i) => { - try { - framePayload(frame) - } catch (error) { - throw new Error( - `fact log v2: sealGroup frame ${i} is not a well-formed frame: ${(error as Error).message}` - ) - } - }) - - const total = frames.reduce((n, f) => n + f.length, 0) - const remainder = total % sealSize - let padBytes = remainder === 0 ? 0 : sealSize - remainder - if (padBytes !== 0 && padBytes < minPadFrameBytes()) { - padBytes += sealSize // gap too small for any frame — pad through one more sector - } - - const sealed = new Uint8Array(total + padBytes) - let offset = 0 - for (const frame of frames) { - sealed.set(frame, offset) - offset += frame.length - } - if (padBytes > 0) { - sealed.set(buildPadFrame(padBytes), offset) - } - return sealed -} - -/** - * Decode a sequence of v2 frames (a sealed group, or a segment body after its - * 32-byte header) with the torn-tail discipline: a frame whose length overruns - * the buffer or whose CRC fails TERMINATES the walk — everything before it is - * intact and returned; nothing after it is guessed at. Pad frames are dropped - * (invisible). CRC-valid frames with unknown record types still throw - * {@link UnknownLogRecordError} — physical damage truncates, format novelty - * refuses. - */ -export function decodeGroupV2(bytes: Uint8Array, options?: DecodeFactV2Options): DecodedFrameGroup { - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - const facts: CommitFactV2[] = [] - let offset = 0 - while (offset + FRAME_PREFIX_BYTES <= bytes.length) { - const length = view.getUint32(offset, true) - const expectedCrc = view.getUint32(offset + 4, true) - const start = offset + FRAME_PREFIX_BYTES - const end = start + length - if (end > bytes.length) break // torn tail: frame length overruns the buffer - const payload = bytes.subarray(start, end) - if (crc32c(payload) !== expectedCrc) break // torn tail: payload CRC mismatch - const fact = decodeFactV2(payload, options) - // Pad filler carries generation 0 (writers can never mint it — encode - // refuses generation < 1). A zero-record fact at a REAL generation is a - // legitimate commit (an all-deduped batch) and must stay visible — - // discriminating on record count would silently swallow generations. - if (fact.generation > 0) facts.push(fact) - offset = end - } - return { facts, validBytes: offset } -} diff --git a/src/db/faultInjectionStorage.ts b/src/db/faultInjectionStorage.ts deleted file mode 100644 index cafd4198..00000000 --- a/src/db/faultInjectionStorage.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * @module db/faultInjectionStorage - * @description Deterministic fault injection at the fact log's raw-byte - * storage surface — the test harness half of the durability protocol. Wraps - * any adapter exposing the {@link FactLogStorage} primitives (the exact - * surface the fact log appends and syncs through) and injects the three - * crash shapes durability tests must prove against: - * - * - **torn write** ({@link FaultInjectionStorage.tearWriteAtByte}): the next - * append persists only its first N bytes, then reports success — the shape - * of power loss after a partially-flushed page. The caller-side "crash" is - * simulated by abandoning in-memory state and reopening from storage. - * - **dropped sync** ({@link FaultInjectionStorage.dropNextSync}): the next - * sync becomes a silent no-op — an fsync the device acknowledged into a - * volatile cache and lost. - * - **failed append** ({@link FaultInjectionStorage.failNextAppend}): the next - * append throws {@link FaultInjectedError} without writing a byte — EIO or - * a full disk, surfaced to the writer. - * - * Every injected fault is journaled on {@link FaultInjectionStorage.injectedFaults} - * so tests can assert not just the outcome but that the fault actually fired. - * Knobs are one-shot (they disarm on firing) and re-arming overwrites the - * pending shot. All other operations pass through untouched. - */ -import type { FactLogStorage } from './factLog.js' - -/** The error a {@link FaultInjectionStorage.failNextAppend} shot throws. */ -export class FaultInjectedError extends Error { - /** The operation the fault fired on. */ - public readonly operation: 'append' - /** The storage path the operation targeted. */ - public readonly path: string - - constructor(operation: 'append', path: string) { - super(`fault injection: ${operation} to ${path} failed by test design`) - this.name = 'FaultInjectedError' - this.operation = operation - this.path = path - } -} - -/** One journaled fault event — proof the injected fault actually fired. */ -export interface InjectedFault { - kind: 'torn-write' | 'dropped-sync' | 'failed-append' - /** The target path (torn-write / failed-append). */ - path?: string - /** The paths a dropped sync was asked to make durable. */ - paths?: string[] - /** Bytes the caller asked to append (torn-write). */ - requestedBytes?: number - /** Bytes actually persisted (torn-write). */ - writtenBytes?: number -} - -/** - * A {@link FactLogStorage} wrapper that injects deterministic storage faults. - * Construct it around any conforming adapter and hand it wherever a - * FactLogStorage is accepted — unarmed, it is a transparent passthrough. - */ -export class FaultInjectionStorage implements FactLogStorage { - private readonly inner: FactLogStorage - /** Pending torn-write byte count, or null when unarmed. */ - private tearAtByte: number | null = null - /** Pending dropped-sync shot. */ - private dropSyncArmed = false - /** Pending failed-append shot. */ - private failAppendArmed = false - /** Journal of every fault that fired, in firing order. */ - public readonly injectedFaults: InjectedFault[] = [] - - constructor(inner: FactLogStorage) { - this.inner = inner - } - - /** - * Arm a torn write: the NEXT {@link appendRawBytes} persists only the first - * `n` bytes of its buffer (all of it when `n` exceeds the buffer) and then - * reports success. One-shot. - */ - tearWriteAtByte(n: number): void { - if (!Number.isInteger(n) || n < 0) { - throw new Error(`fault injection: tearWriteAtByte needs a non-negative integer; got ${n}`) - } - this.tearAtByte = n - } - - /** Arm a dropped sync: the NEXT {@link syncRawObjects} silently does nothing. One-shot. */ - dropNextSync(): void { - this.dropSyncArmed = true - } - - /** - * Arm a failed append: the NEXT {@link appendRawBytes} throws - * {@link FaultInjectedError} without writing. One-shot; wins over a - * simultaneously-armed torn write (nothing is written at all). - */ - failNextAppend(): void { - this.failAppendArmed = true - } - - /** Append bytes — the injection point for torn writes and failed appends. */ - async appendRawBytes(path: string, bytes: Uint8Array): Promise { - if (this.failAppendArmed) { - this.failAppendArmed = false - this.injectedFaults.push({ kind: 'failed-append', path }) - throw new FaultInjectedError('append', path) - } - if (this.tearAtByte !== null) { - const writtenBytes = Math.min(this.tearAtByte, bytes.length) - this.tearAtByte = null - this.injectedFaults.push({ - kind: 'torn-write', - path, - requestedBytes: bytes.length, - writtenBytes - }) - if (writtenBytes > 0) { - await this.inner.appendRawBytes(path, bytes.subarray(0, writtenBytes)) - } - return - } - return this.inner.appendRawBytes(path, bytes) - } - - /** Make paths durable — the injection point for dropped syncs. */ - async syncRawObjects(paths: string[]): Promise { - if (this.dropSyncArmed) { - this.dropSyncArmed = false - this.injectedFaults.push({ kind: 'dropped-sync', paths: [...paths] }) - return - } - return this.inner.syncRawObjects(paths) - } - - /** Passthrough. */ - async readRawBytes(path: string): Promise { - return this.inner.readRawBytes(path) - } - - /** Passthrough. */ - async writeRawBytes(path: string, bytes: Uint8Array): Promise { - return this.inner.writeRawBytes(path, bytes) - } - - /** Passthrough. */ - async rawByteSize(path: string): Promise { - return this.inner.rawByteSize(path) - } - - /** Passthrough. */ - async readRawObject(path: string): Promise { - return this.inner.readRawObject(path) - } - - /** Passthrough. */ - async writeRawObject(path: string, data: any): Promise { - return this.inner.writeRawObject(path, data) - } - - /** Passthrough. */ - async deleteRawObject(path: string): Promise { - return this.inner.deleteRawObject(path) - } -} diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 6922da6d..aede17a4 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -45,15 +45,7 @@ import type { GenerationStorage, TxLogEntry } from './types.js' -import { readLogAuthority } from './logAuthority.js' -import { - FactLog, - storageSupportsFactLog, - type CommitFact, - type FactOp, - type FactIntMinter, - type FactMarkerRecord -} from './factLog.js' +import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js' import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js' import { crc32c } from '../utils/crc32c.js' @@ -75,13 +67,6 @@ export interface CommitBeforeImages { export const GENERATION_COUNTER_PATH = '_system/generation.json' /** Storage-root-relative path of the commit manifest. */ export const MANIFEST_PATH = '_system/manifest.json' -/** - * The clean-shutdown marker (log-authority recovery gate): written+fsynced at - * a clean close carrying the committed generation; CONSUMED at every open. - * Absent or generation-mismatched at open = unclean shutdown = the whole-log - * replay fold. Its absence is always safe (costs one replay, loses nothing). - */ -export const CLEAN_SHUTDOWN_PATH = '_system/clean-shutdown.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' @@ -103,43 +88,12 @@ export const GENERATIONS_PREFIX = '_generations' * IS committed); the tx-log append has NOT happened yet. A crash here must * keep the transaction (the tx-log is advisory metadata, not the source of * commit truth). - * - `'transact-after-fact-sync'` — the batch's fact is appended AND fsynced, - * but neither the counter nor the manifest advanced. A crash here must cost - * the whole batch: recovery restores the before-images and open() truncates - * the synced fact back to the manifest watermark. - * - * Single-op (Model-B group-commit) phases — `commitSingleOp`: - * - * - `'singleop-after-execute'` — the live canonical write has applied (tmp+ - * rename, not individually fsynced); no history, fact, or generation record - * exists yet. A crash here must cost only the never-returned ack — the - * baseline stays intact and the log stays at the committed watermark. - * - `'singleop-after-fact-append'` — the fact is appended (and, in at-ack - * mode, fsynced); the manifest never saw the generation. A crash here must - * cost the buffered history + the fact (open() truncates it back), never - * the baseline. - * - * Pending-tier flush phases — `flushPendingSingleOps`: - * - * - `'flush-after-staging'` — the window's record-set dirs are written but not - * fsynced and the manifest never advanced. A crash here must cost only the - * window's HISTORY (drop-without-restore) — the acked live writes stay. - * - `'flush-before-manifest'` — staging is fsynced and the facts are fsynced, - * but the manifest never advanced. A crash here must cost only the window's - * history and its facts (truncated at open) — the acked live writes stay. - * - `'before-manifest-rename'` is ALSO fired by the flush path just before its - * commit point (see `flushPendingSingleOpsUnlocked`). */ export type CommitFaultPhase = | 'after-staging' | 'after-execute' | 'before-manifest-rename' | 'after-manifest-rename' - | 'transact-after-fact-sync' - | 'singleop-after-execute' - | 'singleop-after-fact-append' - | 'flush-after-staging' - | 'flush-before-manifest' /** * @description Identifies which ids a transaction touches, split by kind. @@ -180,38 +134,6 @@ export class GenerationStore { */ private factLog: FactLog | null = null - /** - * Fact-log durability mode. 'deferred' (default) = the fact becomes - * durable at the group-commit flush, together with the buffered history — - * the pre-log-authority contract, zero added ack latency. 'at-ack' = - * every single-op ack awaits a covering log fsync (shared via the log's - * group commit) — the log-authority contract: an acked write's fact - * survives power loss. Set by the owner from the stored authority switch - * at open; transact() is durable-at-return in BOTH modes (unchanged). - */ - private logDurability: 'deferred' | 'at-ack' = 'deferred' - - /** Switch the fact-log durability mode (see {@link logDurability}). */ - setLogDurability(mode: 'deferred' | 'at-ack'): void { - this.logDurability = mode - } - - /** - * The fact log's v2 int minter — injected by the OWNER (brainy wires the - * metadata index's id mapper here right after the index is ready), because - * this store cannot know the mapper. With the minter installed, new fact - * segments write the v2 format and after-image records carry minted dense - * ints reproducible by an id-mapper rebuild. Survives reopen: `open()` - * re-installs it on the fresh {@link FactLog} instance. - */ - private intMinter: FactIntMinter | null = null - - /** Install the fact log's v2 int minter (see {@link intMinter}). */ - setIntMinter(mint: FactIntMinter): void { - this.intMinter = mint - this.factLog?.setIntMinter(mint) - } - /** Latest reserved/observed generation (≥ {@link committed}). */ private counter = 0 /** Committed-transaction watermark (manifest generation). */ @@ -468,26 +390,9 @@ export class GenerationStore { | null const manifest = (await this.storage.readRawObject(MANIFEST_PATH)) as GenerationManifest | null - // TORN-ARTIFACT VALIDATION (power-loss survivors): a torn manifest or - // counter can carry NaN/garbage where a generation belongs — unguarded, - // that NaN reaches BigInt() conversions at init and kills the open with - // a RangeError. A non-finite-integer generation is DISCARDED with - // narration (the conservative floor: 0 = re-derive from the record - // directories / fact log below, exactly the recovery machinery's job). - const finiteGen = (v: unknown, source: string): number => { - if (typeof v === 'number' && Number.isSafeInteger(v) && v >= 0) return v - if (v !== undefined && v !== null) { - prodLog.warn( - `[GenerationStore] ${source} carries a non-integer generation ` + - `(${String(v)}) — torn write survivor; discarding and re-deriving ` + - `from recovery (never a RangeError at open)` - ) - } - return 0 - } - this.committed = finiteGen(manifest?.generation, 'manifest') - this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') - this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) + this.committed = manifest?.generation ?? 0 + this.horizonGen = manifest?.horizon ?? 0 + this.counter = Math.max(counterFile?.generation ?? 0, this.committed) // Discover existing generation record directories. const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) @@ -540,86 +445,6 @@ export class GenerationStore { // hosts no fact log (readers fall back to canonical enumeration). if (storageSupportsFactLog(this.storage)) { this.factLog = new FactLog(this.storage) - if (this.intMinter) this.factLog.setIntMinter(this.intMinter) - // LOG-AUTHORITY REPLAY (durable-at-ack's recovery half): when this - // brain's stored authority is the log, an intact fact ABOVE the - // manifest is an ACKED write whose canonical bytes may not have - // survived the crash — its fsynced fact is the ONLY durable copy. - // Truncating it would lose an acked write; instead REPLAY it into - // canonical and advance the manifest to cover it. Tree-authority - // brains keep the truncate contract (their acks never promised the - // fact was durable). Derived indexes reconcile through the normal - // drift machinery at open — same as group-commit recovery. - const authority = await readLogAuthority(this.storage) - if (authority.authority === 'log') { - // TWO REPLAY TIERS, gated by the clean-shutdown marker: - // - // (1) ABOVE-MANIFEST (always): an intact fact above the manifest is - // an acked write whose canonical bytes may not have survived — - // replay it in and advance the manifest. - // (2) WHOLE-LOG (unclean shutdown only): power loss can ALSO vaporize - // canonical bytes BELOW the manifest — live entity writes are - // tmp+rename without per-file fsync; the group-commit flush syncs - // the staging copies and the manifest, never the live tree. The - // manifest therefore over-states canonical durability across a - // power cut, and facts ≤ manifest can be the ONLY durable copy - // of acked state (measured: 299 of 301 acks lost while the log - // held every fact scan-clean). Under log authority, recovery is - // REPLAY: an unclean open folds the ENTIRE log into canonical — - // whole-entity after-images are idempotent, so re-applying - // already-intact records is byte-safe. A clean close writes the - // marker and skips all of this (zero open cost on the happy - // path); crash recovery pays one narrated log fold — LC1 and - // LC5 are the same code, a crash is just bigger lag. - const cleanShutdown = await this.readCleanShutdownMarker() - const orphans = await this.factLog.peekFactsAbove(this.committed) - const uncleanOpen = cleanShutdown === null || cleanShutdown !== this.committed - const factsToReplay = uncleanOpen - ? await this.factLog.peekFactsAbove(0) - : orphans - if (factsToReplay.length > 0) { - let replayed = 0 - for (const fact of factsToReplay) { - for (const op of fact.ops) { - const image = - op.record === null - ? { metadata: null, vector: null } - : { metadata: op.record.metadata, vector: op.record.vector } - if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) - else await this.storage.writeNounRaw(op.id, image) - } - replayed++ - if (fact.generation > this.committed) { - this.committed = fact.generation - this.appendCommittedGen(fact.generation) - this.setDelta(fact.generation, { - nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)), - verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)), - timestamp: fact.timestamp, - bytes: 0 - }) - } - } - if (this.counter < this.committed) this.counter = this.committed - await this.persistCounterUnlocked() - const manifest: GenerationManifest = { - version: 1, - generation: this.committed, - committedAt: new Date().toISOString(), - horizon: this.horizonGen - } - await this.storage.writeRawObject(MANIFEST_PATH, manifest) - await this.storage.syncRawObjects([MANIFEST_PATH]) - prodLog.warn( - `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical (${uncleanOpen ? 'WHOLE-LOG fold — unclean shutdown' : 'above-manifest'}; ` + - `committed at ${this.committed}) — an acked write is never lost` - ) - } - // The marker is consumed: any session that can write invalidates it - // at first commit (see the commit paths); a clean close re-writes it. - await this.clearCleanShutdownMarker() - } await this.factLog.open(this.committed) } else { this.factLog = null @@ -672,37 +497,6 @@ export class GenerationStore { await this.flushPendingSingleOps() this.storage.setGenerationBumpHook(undefined) await this.persistCounterNow() - // Clean-shutdown marker (log-authority recovery gate): everything above - // is durable; stamp the committed generation so the next open can adopt - // instead of folding the log. Written LAST — a crash before this line is - // exactly the unclean case the marker's absence reports. - try { - await this.storage.writeRawObject(CLEAN_SHUTDOWN_PATH, { generation: this.committed }) - await this.storage.syncRawObjects([CLEAN_SHUTDOWN_PATH]) - } catch { - // A failed marker write only costs the next open a replay fold — safe. - } - } - - /** Read the clean-shutdown marker's generation, or null (absent/unreadable). */ - private async readCleanShutdownMarker(): Promise { - try { - const raw = (await this.storage.readRawObject(CLEAN_SHUTDOWN_PATH)) as { - generation?: number - } | null - return raw && Number.isSafeInteger(raw.generation) ? (raw.generation as number) : null - } catch { - return null - } - } - - /** Consume the clean-shutdown marker (every open; a clean close re-writes it). */ - private async clearCleanShutdownMarker(): Promise { - try { - await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH) - } catch { - // Absent or undeletable: the conservative outcome is a future replay. - } } /** @@ -918,37 +712,6 @@ export class GenerationStore { else this.pins.set(gen, count - 1) } - - /** - * Torn-tolerant raw read for BEFORE-IMAGE contexts: a write landing on a - * TORN record (power-loss survivor) is a HEAL — the new after-image - * replaces the unreadable bytes. The before-image is unknowable, so it - * reads as the CREATE SENTINEL ({metadata:null, vector:null}) with - * narration: history for this id restarts at this generation (an asOf - * below it resolves absent for the id — the honest statement of what the - * crash destroyed). The adapter's loud floor (error + gauge) fired at - * throw time; real storage faults still propagate. - */ - private async readRawForBeforeImage( - kind: 'noun' | 'verb', - id: string - ): Promise<{ metadata: unknown | null; vector: unknown | null }> { - try { - return kind === 'noun' - ? await this.storage.readNounRaw(id) - : await this.storage.readVerbRaw(id) - } catch (err) { - if ((err as { code?: string }).code === 'TORN_RECORD') { - prodLog.warn( - `[GenerationStore] before-image of ${kind} ${id} is TORN — the incoming ` + - `write HEALS the record; its history restarts at this generation` - ) - return { metadata: null, vector: null } - } - throw err - } - } - /** @returns Total number of live pins across all generations. */ activePinCount(): number { let total = 0 @@ -1021,8 +784,6 @@ export class GenerationStore { nouns: string[] verbs: string[] meta?: Record - /** V2 marker records riding this fact (same generation, same append). */ - records?: FactMarkerRecord[] }): Promise { const ops: FactOp[] = [] const afterRecords: GenerationRecord[] = [] @@ -1046,8 +807,7 @@ export class GenerationStore { timestamp: args.timestamp, ops, ...(args.meta ? { meta: args.meta } : {}), - ...(blobHashes.length > 0 ? { blobHashes } : {}), - ...(args.records && args.records.length > 0 ? { records: args.records } : {}) + ...(blobHashes.length > 0 ? { blobHashes } : {}) } } @@ -1060,17 +820,11 @@ export class GenerationStore { * per-record analogue of `ifAtGeneration`. A throw aborts the whole batch: * the generation reservation is returned and no staging I/O has happened. */ precommit?: (before: CommitBeforeImages) => void - /** Optional v2 marker records riding this batch's ONE commit fact (e.g. - * the deferred-embedding lifecycle markers) — same generation, same - * atomic append, same durability barrier as the batch itself, so a - * marker can never be orphaned from its write nor the write from its - * marker. Additive: omitted on every markerless path. */ - records?: FactMarkerRecord[] execute: () => Promise }): Promise<{ generation: number; timestamp: number }> { return this.withMutex(async () => { // A latched history-durability failure compromises the whole generation - // chain — refuse a transact too (advancing the manifest past stuck, + // spine — refuse a transact too (advancing the manifest past stuck, // un-durable single-op generations would be inconsistent). Same loud // error; self-clears when the pending tier drains. this.assertHistoryDurable() @@ -1114,11 +868,11 @@ export class GenerationStore { // conflicting batch aborts with zero staging I/O. The maps hold the // byte-identical records the staged files are written from. for (const id of nouns) { - const prev = await this.readRawForBeforeImage('noun', id) + const prev = await this.storage.readNounRaw(id) nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) } for (const id of verbs) { - const prev = await this.readRawForBeforeImage('verb', id) + const prev = await this.storage.readVerbRaw(id) verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) } @@ -1202,15 +956,11 @@ export class GenerationStore { timestamp, nouns, verbs, - ...(args.meta ? { meta: args.meta } : {}), - ...(args.records && args.records.length > 0 ? { records: args.records } : {}) + ...(args.meta ? { meta: args.meta } : {}) }) await this.factLog.append(fact) await this.factLog.sync() } - // A crash here must cost the whole batch: the synced fact is truncated - // back at open() and the before-images are restored byte-identically. - faultPoint('transact-after-fact-sync') // -- 5. Counter + manifest rename (COMMIT POINT) ---------------------- await this.persistCounterUnlocked() @@ -1416,18 +1166,6 @@ export class GenerationStore { touched: { nouns?: string[]; verbs?: string[] } execute: () => Promise precommit?: (before: CommitBeforeImages) => void - /** - * Optional v2 marker records riding this write's commit fact (e.g. the - * deferred-embedding lifecycle markers) — same generation, same atomic - * append, and in 'at-ack' log durability the SAME covering fsync as the - * write itself (zero extra sync). A marker can never be orphaned from - * its write nor the write from its marker. Additive: omitted on every - * markerless path. When the storage hosts no fact log the markers have - * no durable home — matching that storage's overall durability posture - * (it cannot host the log's crash guarantees either); callers own - * surfacing that honestly. - */ - records?: FactMarkerRecord[] }): Promise<{ generation: number; timestamp: number; degraded?: string[] }> { return this.withMutex(async () => { // Refuse to accept a write whose history we cannot make durable: if the @@ -1446,12 +1184,12 @@ export class GenerationStore { // {metadata:null, vector:null} = the create sentinel. const nounBefore = new Map() for (const id of nouns) { - const prev = await this.readRawForBeforeImage('noun', id) + const prev = await this.storage.readNounRaw(id) nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) } const verbBefore = new Map() for (const id of verbs) { - const prev = await this.readRawForBeforeImage('verb', id) + const prev = await this.storage.readVerbRaw(id) verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) } @@ -1497,13 +1235,7 @@ export class GenerationStore { // buffered history). if (this.factLog) { await this.factLog.append( - await this.buildCommitFact({ - generation: gen, - timestamp, - nouns, - verbs, - ...(args.records && args.records.length > 0 ? { records: args.records } : {}) - }) + await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) ) } prodLog.warn( @@ -1530,12 +1262,6 @@ export class GenerationStore { throw err } this.inTransact = false - // Test-only crash simulation (direct call — a throw propagates with no - // cleanup, exactly like a process death; recovery-on-open restores the - // contract). A crash here must cost only the never-returned ack: the - // live canonical write applied, but no history, fact, or generation - // record exists for it yet. - if (this.commitFaultInjector) this.commitFaultInjector('singleop-after-execute') // Buffer the pending generation + make it instantly visible to reads. this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) @@ -1544,52 +1270,14 @@ export class GenerationStore { // Fact log (dual-write): the acked write's AFTER-IMAGE fact, appended // now (read back warm, under the mutex — group-commit means flush-time // canonical only holds the LATEST state, so each generation's after-image - // exists only here). - // - // Durability is MODE-GOVERNED: - // - 'deferred' (default, the pre-log-authority behavior): durability - // rides the group-commit flush like the buffered history — a crash - // before the flush loses the fact AND the generation together, never - // a torn state. - // - 'at-ack' (log-authority mode): the ack awaits a covering fsync via - // the log's group-commit (many concurrent writers share ONE sync) — - // an acked write's fact survives power loss, by contract. + // exists only here). Durability rides the group-commit flush, exactly + // like the buffered before-image history: a crash before the flush loses + // the fact AND the generation together — never a torn state. if (this.factLog) { - try { - await this.factLog.append( - await this.buildCommitFact({ - generation: gen, - timestamp, - nouns, - verbs, - ...(args.records && args.records.length > 0 ? { records: args.records } : {}) - }) - ) - if (this.logDurability === 'at-ack') { - await this.factLog.ensureSynced() - } - } catch (err) { - // A rejected write must NOT commit: the generation was buffered - // before the append, so un-buffer it and return the counter - // reservation — otherwise the next flush would durably commit a - // generation with NO fact, a silent log gap a later replay would - // turn into loss. Canonical bytes from execute() remain as an - // uncommitted orphan — identical to a crash at this point; never - // a torn committed state. - this.pendingBuffer.delete(gen) - const idx = this.pendingGens.lastIndexOf(gen) - if (idx !== -1) this.pendingGens.splice(idx, 1) - this.invalidateChains() - if (this.counter === gen) this.counter = gen - 1 - throw err - } + await this.factLog.append( + await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs }) + ) } - // Test-only crash simulation. A crash here must cost the buffered - // history + the appended fact in 'deferred' mode (open() truncates it - // back to the manifest watermark) — while under 'log' authority the - // intact fact is REPLAYED at open, never the baseline or the applied - // live write. - if (this.commitFaultInjector) this.commitFaultInjector('singleop-after-fact-append') this.schedulePendingFlush() return { generation: gen, timestamp } }) @@ -1708,11 +1396,6 @@ export class GenerationStore { logEntries.push({ generation: gen, timestamp: buf.timestamp }) } - // Test-only crash simulation. A crash here must cost only the window's - // HISTORY: un-fsynced record-set dirs may sit above the manifest, and - // recovery drops them WITHOUT restore — the acked live writes stay. - if (this.commitFaultInjector) this.commitFaultInjector('flush-after-staging') - // ONE fsync for the whole window — the durability-batching win. await this.storage.syncRawObjects(stagedPaths) @@ -1722,12 +1405,6 @@ export class GenerationStore { // generation without its durable fact. await this.factLog?.sync() - // Test-only crash simulation. A crash here must cost only the window's - // history and its (already fsynced) facts — open() truncates the facts - // back to the manifest watermark and drops the staged group-commit dirs - // without restore; the acked live writes stay. - if (this.commitFaultInjector) this.commitFaultInjector('flush-before-manifest') - // Test-only crash simulation: a throwing injector here leaves the staged // group-commit generation dirs on disk with NO manifest advance — the // exact "crashed mid-flush" state recovery must DROP-WITHOUT-RESTORE diff --git a/src/db/logAuthority.ts b/src/db/logAuthority.ts deleted file mode 100644 index 0703d11f..00000000 --- a/src/db/logAuthority.ts +++ /dev/null @@ -1,332 +0,0 @@ -/** - * @module db/logAuthority - * @description The per-brain LOG-AUTHORITY SWITCH and its verification - * oracle — the guarded adoption path for log-canonical storage. - * - * Two storage authorities exist during the adoption window: - * - `'tree'` (the default, today's behavior): the canonical record tree is - * authoritative; the generation log is a complete dual-written journal. - * - `'log'`: the generation log is authoritative for this brain; single-op - * write acks await a covering log fsync (durable-at-ack), and derived - * state treats the log as ground truth. - * - * THE SWITCH IS PER BRAIN, STORED, CHECKED AT OPEN ONLY, and ONE-DIRECTIONAL - * unless explicitly reverted by an operator. A brain flips ONLY when its - * verification oracle is green: a full replay-and-diff of the log against - * the still-authoritative tree (the read-only witness). The oracle failing - * NAMES every divergence — a brain with pre-log history (records the log - * never saw) reports them as `pre-log-record` mismatches and needs a - * baseline backfill before it can ever flip. - * - * Nothing in this module mutates data: the oracle is read-only; the flip - * writes ONE artifact. Reverting = rewriting the artifact to 'tree' (the - * tree remained authoritative-quality throughout the window by dual-write). - */ - -import type { FactScanHandle } from './factLog.js' -import { prodLog } from '../utils/logger.js' -import { createHash } from 'crypto' - -/** Storage-root-relative path of the authority switch artifact. */ -export const LOG_AUTHORITY_PATH = '_system/log-authority.json' - -/** The persisted shape of the authority switch. */ -export interface LogAuthorityRecord { - /** Which store is authoritative for this brain. */ - authority: 'tree' | 'log' - /** When the flip happened (ms epoch). Absent while authority = 'tree'. */ - flippedAt?: number - /** The oracle verdict that justified the flip (summary, not the full report). */ - oracle?: { - verifiedAt: number - generationsScanned: number - nounsChecked: number - verbsChecked: number - } - /** - * Recorded when an OPEN-TIME adoption attempt (the 10.0.0 fleet default) - * was refused — the oracle could not go green. Keeps subsequent opens - * cheap; an operator re-runs adoptLogAuthority() after resolving it. - */ - adoptRefusal?: { at: number; reason: string } -} - -/** The narrow storage surface this module needs. */ -export interface LogAuthorityStorage { - readRawObject(path: string): Promise - writeRawObject(path: string, data: unknown): Promise - syncRawObjects(paths: string[]): Promise - getNouns(opts: { - pagination: { limit: number; offset?: number; cursor?: string } - }): Promise<{ items: unknown[]; hasMore?: boolean; nextCursor?: string }> - getNounMetadata(id: string): Promise -} - -/** One divergence found by the oracle. */ -export interface OracleMismatch { - id: string - kind: 'noun' | 'verb' - reason: - | 'pre-log-record' // canonical row the log never saw — needs baseline backfill - | 'state-differs' // latest log after-image ≠ canonical bytes - | 'log-live-canonical-absent' // log says live, canonical has no record - | 'log-tombstone-canonical-present' // log says deleted, canonical still has it -} - -/** The oracle's full report. */ -export interface OracleReport { - verdict: 'green' | 'red' - generationsScanned: number - nounsChecked: number - verbsChecked: number - matched: number - mismatches: OracleMismatch[] - /** Mismatch listing is capped; the counts above are always complete. */ - mismatchListTruncated: boolean -} - -const MISMATCH_LIST_CAP = 200 - -/** Read the stored authority (absent artifact = 'tree', the safe default). */ -export async function readLogAuthority( - storage: Pick -): Promise { - const raw = (await storage - .readRawObject(LOG_AUTHORITY_PATH) - .catch(() => null)) as LogAuthorityRecord | null - if (raw && (raw.authority === 'log' || raw.authority === 'tree')) return raw - return { authority: 'tree' } -} - -/** - * Normalize a canonical noun record to its ENTITY TRUTH before diffing: - * the canonical vector-file wrapper denormalizes derived index residue - * (`connections` — HNSW graph edges; `level` — the node's random skip-list - * level) that the generation log deliberately does NOT carry (projections - * own their own rebuild paths). Digesting the residue would report false - * `state-differs` on ~any brain whose HNSW assigned a nonzero level. Both - * sides of every oracle comparison pass through this normalizer. - */ -export function nounEntityTruth(record: { - metadata: unknown - vector: unknown -}): { metadata: unknown; vector: unknown } { - const v = record.vector - if (v && typeof v === 'object' && !Array.isArray(v)) { - const { connections: _c, level: _l, ...entity } = v as Record - return { metadata: record.metadata, vector: entity } - } - return { metadata: record.metadata, vector: v } -} - -/** - * Stable content hash of a stored record for diffing — key-sorted JSON so - * property order can never fake a divergence. - */ -export function recordDigest(record: unknown): string { - const stable = (v: unknown): unknown => { - if (Array.isArray(v)) return v.map(stable) - if (v && typeof v === 'object') { - const out: Record = {} - for (const k of Object.keys(v as Record).sort()) { - out[k] = stable((v as Record)[k]) - } - return out - } - return v - } - return createHash('sha256').update(JSON.stringify(stable(record))).digest('hex') -} - -/** - * THE VERIFICATION ORACLE: replay the fact log's noun records and diff the - * final state per id against the canonical tree (the witness). Read-only; - * bounded memory (id → {tombstoned, digest} — digests, never bodies). - * - * Verdict law: 'green' iff EVERY canonical row's latest state is exactly - * reproduced by the log AND the log claims nothing canonical denies. A - * brain older than its log reports its unlogged rows as `pre-log-record` - * mismatches — the named cure is a baseline backfill, never a silent pass. - */ -export async function runLogCompletenessOracle(args: { - storage: LogAuthorityStorage - scanFacts: () => FactScanHandle | null - /** Digest the canonical record the same way the log's after-image is digested. */ - canonicalNounDigest: (id: string) => Promise - /** Digest a log after-image record's payload. */ - factRecordDigest: (record: unknown) => string - /** - * Verb legs (optional until every owner wires them): the canonical verb - * digest + the paged verb enumeration. When ABSENT, the oracle counts NO - * verbs and says so via verbsChecked = 0 — an honest partial verdict, - * never a silent full-pass claim. - */ - canonicalVerbDigest?: (id: string) => Promise - getVerbs?: (opts: { - pagination: { limit: number; offset?: number; cursor?: string } - }) => Promise<{ items: unknown[]; hasMore?: boolean; nextCursor?: string }> -}): Promise { - const report: OracleReport = { - verdict: 'red', - generationsScanned: 0, - nounsChecked: 0, - verbsChecked: 0, - matched: 0, - mismatches: [], - mismatchListTruncated: false - } - const addMismatch = (m: OracleMismatch): void => { - if (report.mismatches.length < MISMATCH_LIST_CAP) report.mismatches.push(m) - else report.mismatchListTruncated = true - } - - // Pass 1: fold the log — latest state per noun id (digest or tombstone). - const scan = args.scanFacts() - if (!scan) { - // No fact log on this store: nothing can be verified — red, loudly. - prodLog.warn('[logAuthority] oracle: this store has no fact log — cannot verify, verdict red') - return report - } - const logState = new Map() - const verbLogState = new Map() - for await (const batch of scan.batches()) { - for (const fact of batch.facts) { - report.generationsScanned++ - for (const op of fact.ops) { - const state = - op.record === null - ? { tombstoned: true, digest: null } - : { tombstoned: false, digest: args.factRecordDigest(op.record) } - if (op.kind === 'noun') logState.set(op.id, state) - else verbLogState.set(op.id, state) - } - } - } - - // Pass 2: walk canonical (paged) and diff. - const seenCanonical = new Set() - const PAGE = 500 - let offset = 0 - let cursor: string | undefined - for (;;) { - const page = await args.storage.getNouns({ - pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } - }) - for (const item of page.items) { - const id = (item as { id: string }).id - seenCanonical.add(id) - report.nounsChecked++ - const inLog = logState.get(id) - if (!inLog) { - addMismatch({ id, kind: 'noun', reason: 'pre-log-record' }) - continue - } - if (inLog.tombstoned) { - addMismatch({ id, kind: 'noun', reason: 'log-tombstone-canonical-present' }) - continue - } - const canonicalDigest = await args.canonicalNounDigest(id) - if (canonicalDigest === null) { - addMismatch({ id, kind: 'noun', reason: 'pre-log-record' }) - continue - } - if (canonicalDigest === inLog.digest) report.matched++ - else addMismatch({ id, kind: 'noun', reason: 'state-differs' }) - } - if (!page.hasMore || page.items.length === 0) break - if (page.nextCursor) cursor = page.nextCursor - else offset += page.items.length - } - - // Pass 3: log-live ids canonical never showed us. - for (const [id, state] of logState) { - if (!state.tombstoned && !seenCanonical.has(id)) { - addMismatch({ id, kind: 'noun', reason: 'log-live-canonical-absent' }) - } - } - - // Verb passes — only when the owner wired the verb legs; otherwise the - // report says verbsChecked: 0, an honest partial scope, never a claim. - if (args.canonicalVerbDigest && args.getVerbs) { - const seenVerbs = new Set() - let vOffset = 0 - let vCursor: string | undefined - for (;;) { - const page = await args.getVerbs({ - pagination: vCursor ? { limit: PAGE, cursor: vCursor } : { limit: PAGE, offset: vOffset } - }) - for (const item of page.items) { - const id = (item as { id: string }).id - seenVerbs.add(id) - report.verbsChecked++ - const inLog = verbLogState.get(id) - if (!inLog) { - addMismatch({ id, kind: 'verb', reason: 'pre-log-record' }) - continue - } - if (inLog.tombstoned) { - addMismatch({ id, kind: 'verb', reason: 'log-tombstone-canonical-present' }) - continue - } - const canonical = await args.canonicalVerbDigest(id) - if (canonical === null) { - addMismatch({ id, kind: 'verb', reason: 'pre-log-record' }) - continue - } - if (canonical === inLog.digest) report.matched++ - else addMismatch({ id, kind: 'verb', reason: 'state-differs' }) - } - if (!page.hasMore || page.items.length === 0) break - if (page.nextCursor) vCursor = page.nextCursor - else vOffset += page.items.length - } - for (const [id, state] of verbLogState) { - if (!state.tombstoned && !seenVerbs.has(id)) { - addMismatch({ id, kind: 'verb', reason: 'log-live-canonical-absent' }) - } - } - } - - const totalMismatches = - report.mismatches.length + (report.mismatchListTruncated ? 1 : 0) - report.verdict = totalMismatches === 0 ? 'green' : 'red' - return report -} - -/** - * Flip this brain's authority to the log — REFUSES unless the supplied - * oracle report is green (the caller runs the oracle; the flip records its - * summary). Writes + fsyncs the switch artifact; the mode takes full effect - * at the NEXT open (checked-at-open-only law), except durable-at-ack which - * the owner may enable immediately. - */ -export async function flipToLogAuthority( - storage: Pick, - oracle: OracleReport -): Promise { - if (oracle.verdict !== 'green') { - throw new Error( - `log-authority flip refused: the verification oracle is RED ` + - `(${oracle.mismatches.length}${oracle.mismatchListTruncated ? '+' : ''} mismatches; ` + - `first: ${oracle.mismatches[0] ? `${oracle.mismatches[0].reason} on ${oracle.mismatches[0].id}` : 'n/a'}). ` + - `A brain flips only on green — fix the divergences (pre-log records need a baseline backfill) and re-run.` - ) - } - const record: LogAuthorityRecord = { - authority: 'log', - flippedAt: Date.now(), - oracle: { - verifiedAt: Date.now(), - generationsScanned: oracle.generationsScanned, - nounsChecked: oracle.nounsChecked, - verbsChecked: oracle.verbsChecked - } - } - await storage.writeRawObject(LOG_AUTHORITY_PATH, record) - await storage.syncRawObjects([LOG_AUTHORITY_PATH]) - prodLog.info( - `[logAuthority] this brain's storage authority is now the generation log ` + - `(oracle green over ${oracle.nounsChecked} nouns / ${oracle.generationsScanned} generations)` - ) - return record -} diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index d002164e..b37391aa 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -27,22 +27,6 @@ import { UnifiedCache, getGlobalCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' import { LSMTree } from './lsm/LSMTree.js' import type { GraphIndexProvider } from '../plugin.js' -import { - computeWatermarkVerdict, - makeProjectionStamp, - readStampedWatermark, - type WatermarkVerdict, - type WatermarkVerdictResult -} from '../utils/projectionWatermark.js' - -/** - * Storage key for the graph-adjacency projection's watermark stamp — a - * sidecar record beside the artifact (the two verb-id LSM trees' persisted - * SSTables + manifests). Written LAST in - * {@link GraphAdjacencyIndex.flush} / {@link GraphAdjacencyIndex.close} so - * stamp-after-data ordering holds for every byte the stamp certifies. - */ -export const GRAPH_ADJACENCY_STAMP_KEY = '__index_graph_adjacency_watermark__' export interface GraphIndexConfig { maxIndexSize?: number // Default: 100000 @@ -128,14 +112,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { // Initialization flag private initialized = false - // --- Watermark stamp state (see utils/projectionWatermark for the law) --- - /** Generation handed in via {@link stampWatermark}, awaiting the next flush. */ - private pendingWatermark: number | null = null - /** Last watermark durably stamped by this instance or loaded at init. */ - private stampedWatermark: number | null = null - /** The three-way verdict computed at init; null until init runs. */ - private loadVerdict: WatermarkVerdictResult | null = null - /** * Check if index is initialized and ready for use */ @@ -265,135 +241,12 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { await this.populateVerbIdSetFromStorage() } - // Watermark verdict for the persisted adjacency artifact (the LSM - // SSTables just loaded) — computed and exposed only: today's rebuild / - // recovery triggers are unchanged (acting on 'catchup' — the incremental - // fold — lands with the coordinator's wiring). - await this.loadWatermarkVerdict(lsmTreeSize > 0) - // Start auto-flush timer after initialization this.startAutoFlush() this.initialized = true } - /** - * @description Record the committed generation this projection reflects. - * The stamp is NOT written here — it is written as the final storage write - * of the next {@link flush} (or {@link close}), so stamp-after-data - * ordering is a module guarantee, not a caller obligation. The coordinator - * calls this with the store's committed generation right before flushing. - * @param generation - The committed generation every flushed byte reflects. - */ - stampWatermark(generation: number): void { - this.pendingWatermark = generation - } - - /** - * @description The projection's current watermark: the stamp loaded at - * init (or the last stamp durably written by this instance). Null = - * unstamped (legacy artifact, first boot, or stamping never wired). - */ - watermark(): number | null { - return this.stampedWatermark - } - - /** - * @description The three-way adoption verdict computed at init — - * `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped < - * committed; the gap from {@link watermarkGap} awaits an incremental - * fold), `'rescan'` (unstamped or stamped above committed — never - * trusted). Null until init() has run. Computed and exposed only; no - * load behavior changes ride on it yet. - */ - watermarkVerdict(): WatermarkVerdict | null { - return this.loadVerdict?.verdict ?? null - } - - /** - * @description The catch-up window `(from, to]` when the init verdict was - * `'catchup'`; null otherwise. - */ - watermarkGap(): { from: number; to: number } | null { - return this.loadVerdict?.gap ?? null - } - - /** - * @description Write the pending watermark stamp as a sidecar record — - * always called AFTER the LSM flushes it certifies completed. A stamp-write - * failure is fail-safe (unstamped/behind → rescan/catchup on next open, - * never a wrong adopt) but is said out loud and the pending stamp is - * retained for the next flush. - */ - private async writePendingStamp(): Promise { - if (this.pendingWatermark === null) return - const watermark = this.pendingWatermark - try { - await this.storage.saveMetadata(GRAPH_ADJACENCY_STAMP_KEY, { - noun: 'IndexWatermark', - ...makeProjectionStamp(watermark) - }) - this.stampedWatermark = watermark - this.pendingWatermark = null - } catch (error) { - prodLog.error( - `[GraphAdjacencyIndex] failed to write watermark stamp (generation ${watermark}) — ` + - `artifact stays behind-stamped (safe: verdicts catchup/rescan, never wrong-adopt); ` + - `retrying on next flush:`, - error - ) - } - } - - /** - * @description Read the artifact's stamp and compute the three-way verdict - * against the store's committed generation. Unstamped state on a stamped - * store verdicts `'rescan'` LOUDLY — never a silent adopt. - * - * MIGRATION COST: existing pre-stamp brains verdict `'rescan'` exactly - * once (that open re-derives via the recovery walk it already runs); the - * next flush stamps them, and every later open adopts. - * - * @param artifactPresent - Whether persisted SSTables exist at all; gates - * loud-vs-quiet on the rescan verdict so first boots don't scream. - */ - private async loadWatermarkVerdict(artifactPresent: boolean): Promise { - const committed = this.storage.committedGeneration?.() ?? null - let stamped: number | null = null - try { - const record = await this.storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY) - stamped = readStampedWatermark(record) - } catch { - // An unreadable stamp is unstamped — the fail-safe direction. - stamped = null - } - const result = computeWatermarkVerdict(stamped, committed) - this.loadVerdict = result - this.stampedWatermark = stamped - - if (result.verdict === 'rescan') { - if (artifactPresent || stamped !== null) { - prodLog.warn( - `[GraphAdjacencyIndex] watermark verdict: RESCAN — persisted adjacency is ` + - (stamped === null - ? 'unstamped (legacy pre-stamp artifact, or a crash between data and stamp)' - : `stamped at generation ${stamped}, ABOVE the store's committed generation ${committed}`) + - ` — never adopting unverifiable state` - ) - } else { - prodLog.debug( - '[GraphAdjacencyIndex] watermark verdict: rescan (no persisted artifact — first boot)' - ) - } - } else if (result.verdict === 'catchup') { - prodLog.info( - `[GraphAdjacencyIndex] watermark verdict: catchup — adjacency stamped at generation ` + - `${stamped}, store committed at ${committed}; the (${stamped}, ${committed}] window ` + - `awaits an incremental fold (verdict exposed; the fold lands with the coordinator's wiring)` - ) - } - } - /** * Populate verbIdSet from storage without full rebuild * Lighter weight than full rebuild - only loads verb IDs, not all verb data @@ -1082,12 +935,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { }), ]) - // STAMP-AFTER-DATA: the watermark stamp is the LAST write of the flush — - // both trees' SSTables are durable before the stamp lands. A crash - // anywhere above leaves the artifact behind-stamped or unstamped, which - // verdicts as catchup/rescan on the next open — never a wrong adopt. - await this.writePendingStamp() - const elapsed = Date.now() - startTime prodLog.debug(`GraphAdjacencyIndex: Flush completed in ${elapsed}ms`) @@ -1108,10 +955,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { this.lsmTreeVerbsBySource.close(), this.lsmTreeVerbsByTarget.close(), ]) - - // Stamp-after-data on the shutdown path too: the trees' final flushes - // completed above, so a pending watermark may land now. - await this.writePendingStamp() } prodLog.info('GraphAdjacencyIndex: Shutdown complete') diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 77e4f84d..eb2acd71 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -16,22 +16,6 @@ import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' import type { VectorIndexProvider, OpaqueIdSet, AtGenerationVectors } from '../plugin.js' import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' -import { - computeWatermarkVerdict, - makeProjectionStamp, - readStampedWatermark, - type WatermarkVerdict, - type WatermarkVerdictResult -} from '../utils/projectionWatermark.js' - -/** - * Storage key for the JS HNSW projection's watermark stamp — a sidecar - * record beside the artifact (per-node vector-index records + connection - * blobs + the entryPoint/maxLevel system record). Written LAST in - * {@link JsHnswVectorIndex.flush} so stamp-after-data ordering holds for - * every byte the stamp certifies. - */ -export const HNSW_INDEX_STAMP_KEY = '__index_hnsw_watermark__' // Default HNSW parameters const DEFAULT_CONFIG: HNSWConfig = { @@ -115,14 +99,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { private dirtyNodes: Set = new Set() // Nodes with unpersisted HNSW data private dirtySystem: boolean = false // Whether system data (entryPoint, maxLevel) needs persist - // --- Watermark stamp state (see utils/projectionWatermark for the law) --- - /** Generation handed in via {@link stampWatermark}, awaiting the next flush. */ - private pendingWatermark: number | null = null - /** Last watermark durably stamped by this instance or loaded on rebuild. */ - private stampedWatermark: number | null = null - /** The three-way verdict computed at load; null until rebuild() runs. */ - private loadVerdict: WatermarkVerdictResult | null = null - // Lazy vector storage (B2 optimization): evict the float32 vector to // storage after insert; reload on demand via getVectorSafe() + UnifiedCache. private vectorStorageMode: 'memory' | 'lazy' = 'memory' @@ -194,9 +170,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } if (this.dirtyNodes.size === 0 && !this.dirtySystem) { - // Nothing dirty — but a pending watermark still stamps: every byte it - // certifies is already durable, so stamp-after-data holds trivially. - await this.writePendingStamp() return 0 } @@ -266,13 +239,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { throw new HnswFlushError(failedNodes.size, systemFailed, firstError ?? undefined) } - // STAMP-AFTER-DATA: the watermark stamp is the LAST write of the flush — - // it lands only after every dirty node and the system record persisted - // (the throw above guarantees it). A crash anywhere earlier leaves the - // artifact behind-stamped or unstamped, which verdicts as catchup/rescan - // on the next open — never a wrong adopt. - await this.writePendingStamp() - if (nodeCount > 0) { prodLog.info(`[HNSW] Flushed ${nodeCount} dirty nodes in ${duration}ms`) } @@ -280,126 +246,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return nodeCount } - /** - * @description Record the committed generation this projection reflects. - * The stamp is NOT written here — it is written as the final storage write - * of the next {@link flush} (stamp-after-data ordering is a module - * guarantee, not a caller obligation). The coordinator calls this with the - * store's committed generation right before flushing. - * @param generation - The committed generation every flushed byte reflects. - */ - public stampWatermark(generation: number): void { - this.pendingWatermark = generation - } - - /** - * @description The projection's current watermark: the stamp loaded at - * rebuild (or the last stamp durably written by this instance). Null = - * unstamped (legacy artifact, first boot, or stamping never wired). - */ - public watermark(): number | null { - return this.stampedWatermark - } - - /** - * @description The three-way adoption verdict computed at load — - * `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped < - * committed; the gap from {@link watermarkGap} awaits an incremental - * fold), `'rescan'` (unstamped or stamped above committed — never - * trusted). Null until rebuild() has run. Computed and exposed only; no - * load behavior changes ride on it yet — today's rebuild triggers are - * unchanged. - */ - public watermarkVerdict(): WatermarkVerdict | null { - return this.loadVerdict?.verdict ?? null - } - - /** - * @description The catch-up window `(from, to]` when the load verdict was - * `'catchup'`; null otherwise. - */ - public watermarkGap(): { from: number; to: number } | null { - return this.loadVerdict?.gap ?? null - } - - /** - * @description Write the pending watermark stamp as a sidecar record — - * always called AFTER the data it certifies is durable. The stamp carries - * the vector-space identity this module can honestly assert: dimensions - * only (no embedding-model id is reachable from the index — it never sees - * the embedder). A stamp-write failure is fail-safe (unstamped/behind → - * rescan/catchup on next open, never a wrong adopt) but is said out loud - * and the pending stamp is retained for the next flush. - */ - private async writePendingStamp(): Promise { - if (this.pendingWatermark === null || !this.storage) return - const watermark = this.pendingWatermark - try { - await this.storage.saveMetadata(HNSW_INDEX_STAMP_KEY, { - noun: 'IndexWatermark', - ...makeProjectionStamp(watermark, { dimensions: this.dimension }) - }) - this.stampedWatermark = watermark - this.pendingWatermark = null - } catch (error) { - prodLog.error( - `[HNSW] failed to write watermark stamp (generation ${watermark}) — ` + - `artifact stays behind-stamped (safe: verdicts catchup/rescan, never wrong-adopt); ` + - `retrying on next flush:`, - error - ) - } - } - - /** - * @description Read the artifact's stamp and compute the three-way verdict - * against the store's committed generation. Unstamped state on a stamped - * store verdicts `'rescan'` LOUDLY — never a silent adopt. - * - * MIGRATION COST: existing pre-stamp brains verdict `'rescan'` exactly - * once (that open re-derives via the rebuild it is already running); the - * next flush stamps them, and every later open adopts. - * - * @param artifactPresent - Whether a persisted artifact exists at all (a - * system record was found); gates loud-vs-quiet on the rescan verdict so - * first boots don't scream. - */ - private async loadWatermarkVerdict(artifactPresent: boolean): Promise { - if (!this.storage) return - const committed = this.storage.committedGeneration?.() ?? null - let stamped: number | null = null - try { - const record = await this.storage.getMetadata(HNSW_INDEX_STAMP_KEY) - stamped = readStampedWatermark(record) - } catch { - // An unreadable stamp is unstamped — the fail-safe direction. - stamped = null - } - const result = computeWatermarkVerdict(stamped, committed) - this.loadVerdict = result - this.stampedWatermark = stamped - - if (result.verdict === 'rescan') { - if (artifactPresent || stamped !== null) { - prodLog.warn( - `[HNSW] watermark verdict: RESCAN — persisted index is ` + - (stamped === null - ? 'unstamped (legacy pre-stamp artifact, or a crash between data and stamp)' - : `stamped at generation ${stamped}, ABOVE the store's committed generation ${committed}`) + - ` — never adopting unverifiable state` - ) - } else { - prodLog.debug('[HNSW] watermark verdict: rescan (no persisted artifact — first boot)') - } - } else if (result.verdict === 'catchup') { - prodLog.info( - `[HNSW] watermark verdict: catchup — index stamped at generation ${stamped}, ` + - `store committed at ${committed}; the (${stamped}, ${committed}] window awaits ` + - `an incremental fold (verdict exposed; the fold lands with the coordinator's wiring)` - ) - } - } - /** * @description Persist one node's connections. When the connections codec is * wired AND the storage adapter exposes `saveBinaryBlob`, the per-level @@ -559,15 +405,8 @@ export class JsHnswVectorIndex implements VectorIndexProvider { /** * Add a vector to the index - * - * @param generation - Brainy's commit generation for this write (contract - * parity with `VectorIndexProvider.addItem`). This JS index serves "now" - * only — no per-record delta log, no natural slot — so the value is - * accepted and ignored; a native provider stamps its durable records - * with it. The JS twin adopts stamping with the watermark train. */ - public async addItem(item: VectorDocument, generation?: bigint): Promise { - void generation // Contract parity — the JS index keeps no per-write log. + public async addItem(item: VectorDocument): Promise { // Check if item is defined if (!item) { throw new Error('Item is undefined or null') @@ -647,90 +486,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return id } - // Wire the node into the graph: greedy descent + per-level linking. - // Extracted to linkNode so updateItem's in-place relink runs the SAME - // insertion linking (one implementation, never a diverging copy). - await this.linkNode(noun, entryPoint) - - // Update max level and entry point if needed - if (nounLevel > this.maxLevel) { - this.maxLevel = nounLevel - this.entryPointId = id - } - - // Add noun to the index - this.nouns.set(id, noun) - - // Track high-level nodes for O(1) entry point selection - if (nounLevel >= 2 && nounLevel <= this.MAX_TRACKED_LEVELS) { - if (!this.highLevelNodes.has(nounLevel)) { - this.highLevelNodes.set(nounLevel, new Set()) - } - this.highLevelNodes.get(nounLevel)!.add(id) - } - - // Lazy vector eviction (B2: graph-only memory after insert) - // After graph construction completes, evict the full vector from memory. - // Future searches will load vectors on-demand via getVectorSafe() + UnifiedCache. - if (this.vectorStorageMode === 'lazy' && this.storage) { - noun.vector = [] // Release float32 vector from memory - } - - // Persist HNSW graph data to storage - // Respect persistMode setting - if (this.storage && this.persistMode === 'immediate') { - // IMMEDIATE MODE: Original behavior - persist new entity and system data. - // Goes through the per-node helper so the compressed-blob branch fires - // identically here vs. the deferred-flush + neighbor-update paths. - await this.persistNodeConnections(id, noun).catch((error) => { - console.error(`Failed to persist HNSW data for ${id}:`, error) - }) - - // Persist system data (entry point and max level) - await this.storage.saveHNSWSystem({ - entryPointId: this.entryPointId, - maxLevel: this.maxLevel - }).catch((error) => { - console.error('Failed to persist HNSW system data:', error) - }) - } else if (this.persistMode === 'deferred') { - // DEFERRED MODE: Track dirty nodes for later batch persistence - this.dirtyNodes.add(id) - this.dirtySystem = true - } - - return id - } - - /** - * @description The insertion LINKING phase shared by {@link addItem} and - * {@link updateItem}: greedy-descend from `entryPoint` through the levels - * above `noun.level`, then at each level from `min(noun.level, maxLevel)` - * down to 0 find `efConstruction` candidates, select the M nearest, and - * create bidirectional edges — maintaining the reverse-adjacency index via - * {@link addIncoming} and re-pruning any neighbor pushed over M. - * - * Persistence follows the caller's mode exactly as the historical inline - * addItem code did: `'immediate'` persists each touched neighbor's - * connections concurrently (batched by `maxConcurrentNeighborWrites`); - * `'deferred'` marks each touched neighbor dirty for the next flush. - * - * Does NOT touch index membership (`this.nouns`), the entry point, or - * `maxLevel` — the caller owns that bookkeeping: addItem inserts a NEW node - * afterwards and may raise maxLevel; updateItem relinks an EXISTING node in - * place whose level was already counted, so nothing may change. `noun.vector` - * must be the live in-memory vector at call time; both callers guarantee it - * (lazy-mode eviction happens only after linking completes). - * - * A `neighborId === noun.id` candidate is skipped defensively: during - * updateItem the node is already IN `this.nouns` (visibility-atomicity — - * unlike addItem, which links before inserting), and a self-edge must never - * be creatable no matter what the traversal surfaces. - */ - private async linkNode(noun: HNSWNoun, entryPoint: HNSWNoun): Promise { - const { id, vector } = noun - const nounLevel = noun.level - let currObj = entryPoint // Calculate distance to entry point (handles lazy loading + sync fast path) @@ -792,10 +547,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { }> = [] for (const [neighborId, _] of neighbors) { - if (neighborId === id) { - // Never self-link (see method JSDoc — reachable only via updateItem) - continue - } const neighbor = this.nouns.get(neighborId) if (!neighbor) { // Skip neighbors that don't exist (expected during rapid additions/deletions) @@ -879,7 +630,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider { const nearestNoun = this.nouns.get(nearestId) if (!nearestNoun) { console.error( - `Nearest noun with ID ${nearestId} not found in linkNode` + `Nearest noun with ID ${nearestId} not found in addItem` ) // Keep the current object as is } else { @@ -888,178 +639,55 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } } } - } - /** - * @description Atomically replace an item's vector IN PLACE — the row is - * NEVER absent from the index during an update. The historical shape staged - * a remove followed by an add as two separately-awaited transaction - * operations; between them the row was in NEITHER index — dark to semantic - * recall while perfectly visible to metadata reads (observed as seconds-long - * production flicker in a downstream deployment). Mandate: a row that - * exists must never be invisible to a read path, even transiently. - * - * Behavior: - * - id not in the index → delegates to {@link addItem} (plain insert). - * - SAME vector (element-wise equal) → pure no-op. This is the production - * flicker shape: a type-only update re-indexes an UNCHANGED vector, so the - * old remove+add did pure damage. (In lazy vector-storage mode the - * comparison baseline is whatever {@link getVectorSafe} serves — the - * cache, or the persisted record; if the caller already rewrote the - * record with the new vector before calling in, equality may report "no - * change" and skip the relink. Query correctness is unaffected either - * way — distances always use the live vector — the graph edges just keep - * their pre-update geometry, which HNSW tolerates by construction.) - * - DIFFERENT vector → the node never leaves `this.nouns`: - * 1. `node.vector` is swapped SYNCHRONOUSLY first (and the shared vector - * cache updated in the same tick), so from that point every query sees - * the node with correct distances; - * 2. its old edges are unlinked via the same reverse-adjacency walk - * removeItem uses ({@link unlinkNodeEdges}) — the node stays in the - * map and KEEPS its level; - * 3. the insertion linking re-runs at the node's EXISTING level - * ({@link linkNode}). Entry-point cases: if the node IS the entry - * point it REMAINS the entry point (still valid — same id, same - * level); the relink traversal then starts from another node via - * {@link resolveRelinkStart}, because the node's own edges were just - * cleared and a traversal starting AT it would find nothing and link - * nothing — stranding the whole graph behind an edgeless entry point. - * maxLevel never regresses: the node keeps its level and its - * membership, so the remove-side relevel bookkeeping never runs. - * - * Persistence mirrors {@link addItem}'s tail for the node itself plus the - * in-neighbors whose connection sets changed during the unlink: - * `'immediate'` persists their connections now; `'deferred'` marks them - * dirty for the next flush. The system record (entry point + maxLevel) is - * NOT rewritten — an in-place update changes neither. - * - * @param generation - Brainy's commit generation for this write (contract - * parity with the feature-detected `updateItem` provider capability). - * Accepted and ignored — the JS index keeps no per-write log. - */ - public async updateItem(item: VectorDocument, generation?: bigint): Promise { - void generation // Contract parity — the JS index keeps no per-write log. - if (!item) { - throw new Error('Item is undefined or null') - } - const { id, vector } = item - if (!vector) { - throw new Error('Vector is undefined or null') + // Update max level and entry point if needed + if (nounLevel > this.maxLevel) { + this.maxLevel = nounLevel + this.entryPointId = id } - const node = this.nouns.get(id) - if (!node) { - // Absent → plain insert. - await this.addItem(item) - return - } + // Add noun to the index + this.nouns.set(id, noun) - if (this.dimension === null) { - this.dimension = vector.length - } else if (vector.length !== this.dimension) { - throw new Error( - `Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}` - ) - } - - // Fast path: element-wise-equal vector → NOTHING to do (the production - // flicker shape — a type-only update re-indexing an unchanged vector). - // getVectorSafe handles the lazy-evicted case (loads from cache/storage). - const current = await this.getVectorSafe(node) - if (current.length === vector.length) { - let same = true - for (let i = 0; i < vector.length; i++) { - if (current[i] !== vector[i]) { - same = false - break - } + // Track high-level nodes for O(1) entry point selection + if (nounLevel >= 2 && nounLevel <= this.MAX_TRACKED_LEVELS) { + if (!this.highLevelNodes.has(nounLevel)) { + this.highLevelNodes.set(nounLevel, new Set()) } - if (same) return + this.highLevelNodes.get(nounLevel)!.add(id) } - // (1) Visibility-atomic swap: from this synchronous assignment on, every - // query sees the node with correct distances. The shared vector cache is - // updated in the same tick so the lazy-mode read path can never serve the - // stale vector either. - node.vector = vector - this.unifiedCache.set(`hnsw:vector:${id}`, vector, 'vectors', vector.length * 4, 50) - - // (2) Unlink the old edges — the node stays in the map, keeps its level. - const touchedReferrers = await this.unlinkNodeEdges(node) - node.connections = new Map() - for (let level = 0; level <= node.level; level++) { - node.connections.set(level, new Set()) - } - // The node's own reverse entry is rebuilt by the relink below. - this.incoming?.delete(id) - - // (3) Relink at the node's EXISTING level (see JSDoc for the entry-point - // reasoning). A single-node index has nothing to link to — trivially done. - const start = this.resolveRelinkStart(id) - if (start) { - await this.linkNode(node, start) + // Lazy vector eviction (B2: graph-only memory after insert) + // After graph construction completes, evict the full vector from memory. + // Future searches will load vectors on-demand via getVectorSafe() + UnifiedCache. + if (this.vectorStorageMode === 'lazy' && this.storage) { + noun.vector = [] // Release float32 vector from memory } - // Persistence — addItem's tail, minus the system record (entry point and - // maxLevel are untouched by an in-place update). Unlink-touched referrers - // are included so the persisted graph converges on the live one instead of - // keeping their pre-update edge sets forever. + // Persist HNSW graph data to storage + // Respect persistMode setting if (this.storage && this.persistMode === 'immediate') { - await this.persistNodeConnections(id, node).catch((error) => { + // IMMEDIATE MODE: Original behavior - persist new entity and system data. + // Goes through the per-node helper so the compressed-blob branch fires + // identically here vs. the deferred-flush + neighbor-update paths. + await this.persistNodeConnections(id, noun).catch((error) => { console.error(`Failed to persist HNSW data for ${id}:`, error) }) - for (const refId of touchedReferrers) { - const ref = this.nouns.get(refId) - if (!ref) continue - await this.persistNodeConnections(refId, ref).catch((error) => { - console.error(`Failed to persist HNSW data for ${refId}:`, error) - }) - } + + // Persist system data (entry point and max level) + await this.storage.saveHNSWSystem({ + entryPointId: this.entryPointId, + maxLevel: this.maxLevel + }).catch((error) => { + console.error('Failed to persist HNSW system data:', error) + }) } else if (this.persistMode === 'deferred') { + // DEFERRED MODE: Track dirty nodes for later batch persistence this.dirtyNodes.add(id) - for (const refId of touchedReferrers) { - this.dirtyNodes.add(refId) - } + this.dirtySystem = true } - // Lazy vector eviction — same contract as addItem: after graph work - // completes the float32 vector leaves memory; reads serve from the - // (just-updated) cache or the persisted record. - if (this.vectorStorageMode === 'lazy' && this.storage) { - node.vector = [] - } - } - - /** - * @description Pick the traversal start for an in-place relink - * ({@link updateItem} step 3): the current entry point — unless that IS the - * node being relinked. Its edges were just unlinked, so a traversal - * starting there would see an empty neighborhood and produce zero links, - * stranding the graph behind an edgeless entry point. In that case (or when - * the entry point is missing/stale) fall back to the best OTHER node: - * highest tracked level first (the same O(1) heuristic as - * {@link recoverEntryPointO1}), then any other node. Returns null when the - * node is the only one in the index — nothing to link to, trivially valid. - */ - private resolveRelinkStart(excludeId: string): HNSWNoun | null { - if (this.entryPointId && this.entryPointId !== excludeId) { - const entry = this.nouns.get(this.entryPointId) - if (entry) return entry - } - for (let level = this.MAX_TRACKED_LEVELS; level >= 2; level--) { - const nodesAtLevel = this.highLevelNodes.get(level) - if (!nodesAtLevel) continue - for (const nodeId of nodesAtLevel) { - if (nodeId !== excludeId) { - const candidate = this.nouns.get(nodeId) - if (candidate) return candidate - } - } - } - for (const [nodeId, candidate] of this.nouns) { - if (nodeId !== excludeId) return candidate - } - return null + return id } /** @@ -1320,34 +948,20 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } /** - * @description Unlink every graph edge touching `noun`, in BOTH directions, - * WITHOUT removing the node from `this.nouns` — the unlink walk shared by - * {@link removeItem} (which then drops the node) and {@link updateItem} - * (which relinks the node in place, so it must never leave the map and - * KEEPS its level). - * - * Reverse-adjacency lets us touch ONLY the nodes that actually reference - * `noun.id` (its in-neighbors) rather than scanning the whole corpus — - * turning a delete from O(N) into O(in-degree) and a bulk delete from O(N²) - * into O(N·degree). Each referrer set is snapshotted because - * pruneConnections mutates the index. Outgoing edges are unhooked from each - * target's reverse set so no stale referrer survives. - * - * `incoming[noun.id]` itself is intentionally NOT maintained edge-by-edge - * inside the walk — both callers dispose of it wholesale afterwards - * (removeItem deletes it with the node; updateItem clears it and lets the - * relink rebuild it). - * - * @returns The ids of in-neighbors whose connection sets were modified - * (they dropped their edge to `noun` and may have been re-pruned), so a - * caller that persists per-node connections (updateItem) can mark them - * dirty / persist them. removeItem ignores the return — its persistence - * story lives in the caller's delete path, unchanged. + * Remove an item from the index */ - private async unlinkNodeEdges(noun: HNSWNoun): Promise> { - const id = noun.id - const touchedReferrers = new Set() + public async removeItem(id: string): Promise { + if (!this.nouns.has(id)) { + return false + } + + const noun = this.nouns.get(id)! + + // Reverse-adjacency lets us touch ONLY the nodes that actually reference `id` + // (its in-neighbors) rather than scanning the whole corpus — turning a delete + // from O(N) into O(in-degree) and a bulk delete from O(N²) into O(N·degree). + // Snapshot each referrer set because pruneConnections mutates the index. const incoming = this.ensureIncoming() const referrers = incoming.get(id) if (referrers) { @@ -1355,11 +969,11 @@ export class JsHnswVectorIndex implements VectorIndexProvider { for (const refId of Array.from(refSet)) { const ref = this.nouns.get(refId) if (ref && ref.connections.has(level)) { - // Drop the forward edge ref → id, then re-prune ref so the graph - // stays navigable. + // Drop the forward edge ref → id, then re-prune ref so the graph stays + // navigable. (id's own reverse entry is dropped wholesale below, so we + // intentionally do not maintain incoming[id] inside this loop.) ref.connections.get(level)!.delete(id) await this.pruneConnections(ref, level) - touchedReferrers.add(refId) } } } @@ -1373,32 +987,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } } - return touchedReferrers - } - - /** - * Remove an item from the index - * - * @param generation - Brainy's commit generation for this removal (contract - * parity with `VectorIndexProvider.removeItem`). Accepted and ignored — - * this JS index removes immediately; a native provider records the - * tombstone at this generation. - */ - public async removeItem(id: string, generation?: bigint): Promise { - void generation // Contract parity — the JS index keeps no per-write log. - if (!this.nouns.has(id)) { - return false - } - - - const noun = this.nouns.get(id)! - - // Unlink every edge touching the node (shared with updateItem's in-place - // relink — see unlinkNodeEdges). The returned touched-referrer set is - // ignored here: removeItem's persistence story lives in the caller's - // delete path, unchanged. - await this.unlinkNodeEdges(noun) - // Remove the noun + its reverse-index entry. this.nouns.delete(id) this.incoming?.delete(id) @@ -1717,11 +1305,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { this.maxLevel = systemData.maxLevel } - // Step 2b: Watermark verdict for the persisted artifact — computed and - // exposed only (today's rebuild flow is unchanged; this rebuild IS the - // re-derive a 'rescan' verdict asks for). - await this.loadWatermarkVerdict(systemData !== null) - // Step 3: Determine preloading strategy (adaptive caching) // Check if vectors should be preloaded at init or loaded on-demand const stats = await this.storage.getStatistics() diff --git a/src/index.ts b/src/index.ts index 2dfc8352..3186a6a7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -83,14 +83,6 @@ export type { AggregationProvider } from './types/brainy.types.js' -// Read-barrier contract (waitForIndexed): the leg names, the options, and -// the typed timeout error (a value export — consumers catch it by instanceof) -export type { - IndexedProjectionPath, - WaitForIndexedOptions -} from './types/brainy.types.js' -export { WaitForIndexedTimeoutError } from './types/brainy.types.js' - // Reserved-field contract — the canonical list of Brainy-owned field names // that may never appear inside a `metadata` bag (see docs/concepts/consistency-model.md) export { @@ -362,15 +354,6 @@ export { MemoryStorage, createStorage } // FileSystemStorage is exported separately to avoid browser build issues. export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js' -// Torn-record surface: a stored file that EXISTS but cannot be decoded throws -// a typed, catchable error on entity reads (never a silent "not found"), and -// every encounter is counted on a per-process gauge. -export { - TornRecordError, - isTornRecordError, - getTornRecordGauge -} from './storage/tornRecordError.js' - // Export types import type { Vector, diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 4fe45bff..d33c05c4 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -31,7 +31,6 @@ import { ColumnSegmentCursor, TailBufferCursor, type CursorEntry } from './Colum import { writeSegmentToBuffer, readSegmentFromBuffer } from './ColumnSegmentFormat.js' import { RoaringBitmap32 } from '../../utils/roaring/index.js' import { compareCodePoints } from '../../utils/collation.js' -import { prodLog } from '../../utils/logger.js' /** * Configuration for the ColumnStore. @@ -613,24 +612,6 @@ export class ColumnStore implements ColumnStoreProvider { /** * Get all segment cursors for a field, loading from storage if needed. */ - /** - * Per-field quarantine ledger for torn segments (power-loss survivors: - * manifest-listed but unloadable). A quarantined segment is skipped with - * per-doubling narration and the field serves its REMAINING segments as a - * DEGRADED-ANNOUNCED result — never a raw throw killing the query, never - * a silent drop. Cleared when a heal/rebuild rewrites the field. - */ - private readonly segmentQuarantine = new Map() - - /** Torn-segment quarantine entries for a field (observability + heal input). */ - quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> { - const out: Array<{ segment: string; error: string; hits: number }> = [] - for (const [key, q] of this.segmentQuarantine) { - if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits }) - } - return out - } - private async getSegmentCursors(field: string): Promise { const manifest = this.manifests.get(field) if (!manifest) return [] @@ -641,38 +622,11 @@ export class ColumnStore implements ColumnStoreProvider { let cursor = this.segmentCache.get(cacheKey) if (!cursor) { - const quarantined = this.segmentQuarantine.get(cacheKey) - if (quarantined) { - // Already-quarantined torn segment: skip, count, narrate per doubling. - quarantined.hits++ - if ((quarantined.hits & (quarantined.hits - 1)) === 0) { - prodLog.warn( - `[ColumnStore] field '${field}' serving DEGRADED: torn segment ${seg.id} ` + - `quarantined (${quarantined.error}) — ${quarantined.hits} queries served ` + - `without it; heal/rebuild the metadata index to restore` - ) - } - continue - } - try { - cursor = await this.loadSegmentCursor(field, seg) - } catch (err) { - if (err instanceof ColumnSegmentLoadError) { - // POWER-LOSS SURVIVOR: a manifest-listed segment whose bytes are - // torn/absent. Quarantine at DISCOVERY and serve the remaining - // segments degraded-announced — a raw throw here killed every - // query on the field forever; a silent skip hid the loss. The - // quarantine is the middle: loud once, counted always, healable. - this.segmentQuarantine.set(cacheKey, { error: (err as Error).message, hits: 1 }) - prodLog.error( - `[ColumnStore] torn segment QUARANTINED at discovery: field '${field}' ` + - `segment ${seg.id} — ${(err as Error).message}. The field serves its ` + - `remaining segments DEGRADED until a heal/rebuild rewrites it.` - ) - continue - } - throw err // real storage faults propagate — never absorbed - } + // loadSegmentCursor either returns a cursor or THROWS — a corrupt / + // missing manifest-listed segment raises ColumnSegmentLoadError and a + // real storage fault propagates, so a listed segment is never silently + // dropped from the result set. + cursor = await this.loadSegmentCursor(field, seg) this.segmentCache.set(cacheKey, cursor) } diff --git a/src/plugin.ts b/src/plugin.ts index 947c86a5..ce973386 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -277,35 +277,8 @@ export interface MetadataIndexProvider { */ isMigrating?(): boolean - /** - * @description Index one entity's metadata. - * @param id - The entity's UUID. - * @param entityOrMetadata - Entity structure or plain metadata bag. - * @param skipFlush - Transactional atomicity: defer the flush to the commit seam. - * @param deferWrites - Batch mode: buffer postings for a later flush. - * @param generation - OPTIONAL (additive) — Brainy's commit generation for - * this write: the SAME u64 counter {@link GraphIndexProvider.addVerb} - * carries, resolved at operation-execute time. A provider with per-record - * delta logs stamps it onto the durable record so its watermark - * ("this projection reflects generation N") is derivable from real data — - * never a literal 0. `undefined` means the caller genuinely has no commit - * generation for this write (rebuild-from-canonical scans, bootstrap - * writes before generation stamping activates); a provider must treat - * that as "unstamped", not as generation 0. The built-in JS manager - * accepts and ignores it (single live view, no per-record log). - */ - addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean, deferWrites?: boolean, generation?: bigint): Promise - /** - * @description Remove one entity from the index. - * @param id - The entity's UUID. - * @param metadata - The entity's metadata (targets exact postings; absent → full scan). - * @param generation - OPTIONAL (additive) — Brainy's commit generation for - * this removal, same contract as {@link MetadataIndexProvider.addToIndex}: - * a provider with per-record delta logs records the tombstone at this - * generation (so as-of reads before it still see the entity); the JS - * manager removes immediately and ignores it. - */ - removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise + addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean, deferWrites?: boolean): Promise + removeFromIndex(id: string, metadata?: any): Promise getIds(field: string, value: any): Promise /** @@ -395,14 +368,7 @@ export interface MetadataIndexProvider { * the ceiling on the JS path), so `Number(bigint)` narrowing is lossless. */ getIdMapper(): { - /** - * Resolve-or-mint the entity's int. `generation` is OPTIONAL (additive): - * Brainy's commit generation current at mint time, so a mapper with - * per-record delta logs stamps the assignment record with a real - * watermark instead of a literal 0. Ignored when the uuid is already - * assigned (assignments are append-only) and by the JS mapper. - */ - getOrAssign(uuid: string, generation?: bigint): number + getOrAssign(uuid: string): number getInt(uuid: string): number | undefined getUuid(intId: number): string | undefined } @@ -1086,33 +1052,8 @@ export interface VectorIndexProvider { */ readonly name: string - /** - * @description Insert one vector. - * @param item - The vector document (`id` + `vector`). - * @param generation - OPTIONAL (additive) — Brainy's commit generation for - * this write: the SAME u64 counter the graph provider's - * `addVerb(..., generation)` carries (and that `search`'s as-of - * `options.generation` reads back), resolved at operation-execute time. - * A provider with per-record delta logs / segment stamps records it so - * its watermark reflects real data — never a literal 0. `undefined` = - * the caller has no commit generation (rebuild-from-canonical, the - * at-generation materializer's ephemeral reader); treat as "unstamped", - * not generation 0. The built-in JS index accepts and ignores it (it - * serves "now" only). The feature-detected `updateItem` capability (see - * `src/transaction/operations/IndexOperations.ts`) carries the same - * optional trailing generation. - */ - addItem(item: VectorDocument, generation?: bigint): Promise - /** - * @description Remove one vector by id. - * @param id - The entity's UUID. - * @param generation - OPTIONAL (additive) — Brainy's commit generation for - * this removal, same contract as {@link VectorIndexProvider.addItem}: a - * provider with durable delete records stamps the tombstone at this - * generation (as-of reads before it still see the vector); the JS index - * removes immediately and ignores it. - */ - removeItem(id: string, generation?: bigint): Promise + addItem(item: VectorDocument): Promise + removeItem(id: string): Promise search( queryVector: Vector, k?: number, @@ -1258,29 +1199,10 @@ export interface EntityIdMapperProvider { * stays compatible — `restore()` falls back to `init()` when this is absent. */ rebuild?(): Promise - /** - * @description Resolve-or-mint the entity's interned int (append-only: - * once assigned, a uuid's int never changes and is never recycled). - * @param uuid - The entity's UUID. - * @param generation - OPTIONAL (additive) — Brainy's commit generation - * current at mint time (the same u64 counter the graph/metadata write - * surfaces carry). A mapper with per-record delta logs stamps the - * assignment record with this real watermark instead of a literal 0. - * Ignored when the uuid is already assigned, and by the JS mapper - * (which keeps no per-record log). - */ - getOrAssign(uuid: string, generation?: bigint): number + getOrAssign(uuid: string): number getUuid(intId: number): string | undefined getInt(uuid: string): number | undefined - /** - * @description Remove the uuid's mapping (the int stays reserved). - * @param uuid - The entity's UUID. - * @param generation - OPTIONAL (additive) — Brainy's commit generation for - * this removal: a mapper with a per-key version chain tombstones the - * mapping at this generation (as-of reads before it still resolve); - * the JS mapper removes immediately and ignores it. - */ - remove(uuid: string, generation?: bigint): boolean + remove(uuid: string): boolean flush(): Promise clear(): Promise getAllIntIds(): number[] diff --git a/src/reprojection/factLogSource.ts b/src/reprojection/factLogSource.ts deleted file mode 100644 index 796fb343..00000000 --- a/src/reprojection/factLogSource.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * @module reprojection/factLogSource - * @description The production {@link FactSource}: adapts the database's - * committed-fact scan to the reprojection engine's `scan(from, limit)` - * window contract. - * - * DEPENDENCY-CLEAN BY DESIGN: this module never imports the database class. - * It wraps a host-owned scan callback `(from, limit) => Promise` - * injected at construction, so the host wires itself in one line — either by - * handing {@link FactLogSource} a callback built on its own scan API, or via - * {@link factSourceFromHost}, which builds that callback from any object - * structurally exposing `scanFacts` (the batch-handle shape the fact log - * serves). - * - * CONTRACT ENFORCEMENT — loud, never quiet: every `scan` return is checked - * (≤ limit facts, strictly ascending generations, all strictly above `from`); - * a violating callback throws instead of silently corrupting a fold. A host - * with NO fact log throws too — reporting "caught up" against an unscannable - * store would be a silent lie. - */ - -import type { CommitFact } from '../db/factLog.js' -import type { FactSource } from './reprojectionEngine.js' - -/** - * The host-owned scan callback: return up to `limit` committed facts with - * generation strictly greater than `from`, in ascending generation order; - * empty means caught up to the head as of the call. - */ -export type FactScanCallback = (from: number, limit: number) => Promise - -/** - * The minimal structural surface of a fact-scanning host — matches the - * database's `scanFacts` shape without importing it. `scanFacts` returns a - * handle whose `batches()` yields ordered, non-empty fact batches, or `null` - * when the store hosts no fact log. - */ -export interface FactScanHost { - scanFacts(options?: { fromGeneration?: number; batchSize?: number }): { - batches: () => AsyncGenerator<{ facts: CommitFact[] }> - } | null -} - -/** - * The production {@link FactSource}: wraps an injected scan callback and - * enforces the window contract on every return. - * - * COST NOTE: each `scan` call is stateless (a fresh window above the caller's - * watermark), which is exactly what resumable, crash-tolerant folds need — - * at the price of the host re-opening its scan per call. Fine for - * budget-capped maintenance; not a hot-path read primitive. - */ -export class FactLogSource implements FactSource { - private readonly scanCallback: FactScanCallback - - /** @param scanCallback - The host-owned scan (see {@link FactScanCallback}). */ - constructor(scanCallback: FactScanCallback) { - if (typeof scanCallback !== 'function') { - throw new Error('FactLogSource: a scan callback (from, limit) => Promise is required') - } - this.scanCallback = scanCallback - } - - /** - * Fetch up to `limit` committed facts strictly above generation `from`, - * verifying the callback honored the window contract. - * @param from - Exclusive lower bound generation (≥ 0 integer). - * @param limit - Maximum facts to return (≥ 1 integer). - */ - async scan(from: number, limit: number): Promise { - if (!Number.isInteger(from) || from < 0) { - throw new Error(`FactLogSource.scan: 'from' must be a non-negative integer (got ${from})`) - } - if (!Number.isInteger(limit) || limit < 1) { - throw new Error(`FactLogSource.scan: 'limit' must be a positive integer (got ${limit})`) - } - const facts = await this.scanCallback(from, limit) - if (!Array.isArray(facts)) { - throw new Error('FactLogSource.scan: the scan callback must resolve to an array of facts') - } - if (facts.length > limit) { - throw new Error( - `FactLogSource.scan: the scan callback returned ${facts.length} facts for limit ${limit} — ` + - `contract violation; refusing to fold an oversized window` - ) - } - let prev = from - for (const fact of facts) { - const g = fact?.generation - if (typeof g !== 'number' || !Number.isFinite(g) || g <= prev) { - throw new Error( - `FactLogSource.scan: the scan callback violated the window contract — generation ` + - `${String(g)} is not strictly ascending above ${prev} (from=${from}); refusing to fold` - ) - } - prev = g - } - return facts - } -} - -/** - * Build the production source from any host structurally exposing - * `scanFacts` — the one-line wiring for the database side: - * - * ```ts - * const source = factSourceFromHost(brain) - * ``` - * - * Each `scan(from, limit)` opens `scanFacts({ fromGeneration: from + 1, - * batchSize: limit })` (the engine's `from` is exclusive; `scanFacts` bounds - * are inclusive) and returns the FIRST batch, closing the handle — short - * batches at segment boundaries are legal under the source contract (only - * EMPTY means caught up). A host with no fact log throws loudly. - * - * @param host - Any object with the `scanFacts` batch-handle shape. - */ -export function factSourceFromHost(host: FactScanHost): FactLogSource { - if (!host || typeof host.scanFacts !== 'function') { - throw new Error('factSourceFromHost: the host must expose scanFacts(options)') - } - return new FactLogSource(async (from, limit) => { - const scan = host.scanFacts({ fromGeneration: from + 1, batchSize: limit }) - if (scan === null) { - throw new Error( - 'reprojection: this store hosts no fact log — reprojection folds committed facts, ' + - 'and reporting a caught-up fold against an unscannable store would be a silent lie' - ) - } - const iterator = scan.batches() - try { - const first = await iterator.next() - return first.done ? [] : first.value.facts - } finally { - // Close the abandoned generator so its cleanup (timers) runs. - if (typeof iterator.return === 'function') { - await iterator.return(undefined) - } - } - }) -} diff --git a/src/reprojection/reprojectionEngine.ts b/src/reprojection/reprojectionEngine.ts deleted file mode 100644 index 1465b1f6..00000000 --- a/src/reprojection/reprojectionEngine.ts +++ /dev/null @@ -1,648 +0,0 @@ -/** - * @module reprojection/reprojectionEngine - * @description The pure-TS reprojection engine — the ONE machinery for - * rebuilding, healing, and migrating persisted projections from the committed - * fact log on the JS side. It is the TypeScript twin of the native engine's - * reprojection core: the same frozen contract (names AND semantics), so a - * single shared conformance suite runs against both implementations and - * TS-only deployments green the same rows without native code. - * - * THE AVAILABILITY LAW — maintenance never holds the doors: - * - * - Work proceeds in INSTALLMENTS of at most {@link MAX_INSTALLMENT_MS} (50ms) - * of wall time each. Between installments the loop awaits a REAL macrotask - * boundary (never a busy loop, never a bare microtask), so foreground I/O - * and timers always interleave with a running fold. - * - Foreground door traffic announces itself via {@link DoorSignal.bump}. An - * in-flight {@link ReprojectionEngine.advance} yields at the next - * installment boundary and returns `{ status: 'preempted' }` — the doors - * never wait for maintenance to finish. - * - Budgets are honored: `advance` stops once `budgetMs` is spent and reports - * exactly how far it got; a later call RESUMES from the adapter's own - * watermark. Nothing ever refolds from zero because a budget ran out. - * - * WATERMARK DISCIPLINE — the engine NEVER writes stamps. Each adapter's - * `applyBatch` owns its own durability and its own stamp (stamp-after-data, - * the law stated in src/utils/projectionWatermark.ts); the engine only READS - * `watermark()` to decide the next scan window. Delivery is therefore - * at-least-once: an adapter that crashed between data and stamp is re-served - * the same facts on resume and MUST apply idempotently. - * - * THE FOUR ANSWER CLASSES of an advance: `'caught-up'` (folded to the head of - * the requested window, ledger clean), `'preempted'` (a door bumped), - * `'budget-exhausted'` (time ran out mid-stream), and `'quarantined'` (folded - * to the head, but this family's quarantine ledger is non-empty — one or more - * poison facts are being skipped and reads touching them are suspect). - */ - -import type { CommitFact } from '../db/factLog.js' -import { prodLog } from '../utils/logger.js' - -/** - * The hard ceiling on one installment of fold work, in wall-clock ms. An - * advance loop that has run this long without yielding closes the installment - * and awaits a macrotask boundary so foreground traffic interleaves. Frozen by - * the shared contract — both engines install the same ceiling. - */ -export const MAX_INSTALLMENT_MS = 50 - -/** Default facts-per-batch pulled from the {@link FactSource} per step. */ -export const DEFAULT_REPROJECTION_BATCH_SIZE = 256 - -/** - * One registered projection family: a named consumer that folds committed - * facts into its own persisted artifact and stamps its own watermark. - * - * OWNERSHIP: the adapter owns durability AND the stamp. `applyBatch` must - * persist its data first and stamp `upTo` after (stamp-after-data), and must - * tolerate at-least-once delivery — on resume after a crash between data and - * stamp, the same facts arrive again. - */ -export interface ProjectionAdapter { - /** Unique family name — the registry key; one adapter serves a family at a time. */ - family: string - /** - * The highest generation this projection's persisted state reflects, or - * `null` when the projection is unbuilt/unstamped. The engine reads this to - * open the next scan window; it never writes it. - */ - watermark(): number | null - /** - * Fold `facts` (ascending generations, all strictly above the current - * watermark) into the projection, then stamp `watermark = upTo`. - * - * `facts` MAY be empty while `upTo` is above the current watermark: that is - * a pure watermark advance past quarantined generations — the adapter must - * still stamp, or the fold cannot make progress past the poison. - * - * FAILURE CONTRACT: throw a {@link ProjectionApplyError} to name exactly one - * poison fact (the engine quarantines it and continues). ANY other throw - * aborts the advance loudly — an unknown failure is never treated as a - * poison record. - */ - applyBatch(facts: CommitFact[], upTo: number): Promise - /** - * Destroy this adapter's persisted artifact(s). The engine calls this on - * the LOSING adapter after a successful {@link ReprojectionEngine.swap}, - * and on a partially-built replacement whose build aborted. - */ - discard(): Promise -} - -/** - * The committed-fact scan the engine folds from. `from` is an EXCLUSIVE lower - * bound generation; the source returns at most `limit` facts in ascending - * generation order, and an empty array means caught up to the head as of this - * call. Short non-empty returns are legal (e.g. a segment boundary) — only - * empty means done. - */ -export interface FactSource { - scan(from: number, limit: number): Promise -} - -/** - * The foreground-preemption signal. Door traffic (foreground reads/writes) - * calls {@link DoorSignal.bump}; an in-flight `advance` observes the bump at - * its next installment boundary, yields a macrotask, and returns - * `{ status: 'preempted' }`. Bumps are edge-triggered per advance: only bumps - * that arrive AFTER an advance began preempt it. - */ -export class DoorSignal { - private count = 0 - - /** Announce foreground door traffic — an in-flight advance will yield. */ - bump(): void { - this.count++ - } - - /** - * The current bump epoch — the engine snapshots this at advance entry and - * compares at installment boundaries. - * @internal - */ - epoch(): number { - return this.count - } -} - -/** - * The TYPED poison-record failure an adapter throws from `applyBatch` to name - * exactly one unfoldable fact. The engine quarantines that generation for - * that family (skips it, ledgers it, narrates per-doubling) and keeps - * folding. Any OTHER throw from `applyBatch` aborts the advance loudly. - */ -export class ProjectionApplyError extends Error { - /** The generation of the fact that cannot be applied. */ - readonly generation: number - /** Optional index of the offending record within the fact's ops. */ - readonly recordIndex?: number - /** The underlying failure. */ - override readonly cause: unknown - - /** - * @param args - `generation` names the poison fact; `recordIndex` - * optionally narrows to one record inside it; `cause` carries the - * underlying failure. - */ - constructor(args: { generation: number; recordIndex?: number; cause: unknown }) { - super( - `projection apply failed at generation ${args.generation}` + - (args.recordIndex !== undefined ? ` (record ${args.recordIndex})` : '') - ) - this.name = 'ProjectionApplyError' - this.generation = args.generation - if (args.recordIndex !== undefined) this.recordIndex = args.recordIndex - this.cause = args.cause - } -} - -/** - * The TYPED single-flight refusal: a second concurrent - * {@link ReprojectionEngine.swap} on a family whose replacement is still - * building. The caller retries after the in-flight swap settles. - */ -export class SwapInFlightError extends Error { - /** The family whose swap is already in flight. */ - readonly family: string - - /** @param family - The family whose swap is already in flight. */ - constructor(family: string) { - super( - `reprojection: a swap is already in flight for family '${family}' — ` + - `swaps are single-flight per family; retry after the current build settles` - ) - this.name = 'SwapInFlightError' - this.family = family - } -} - -/** One quarantined fact in a family's ledger. */ -export interface QuarantineEntry { - /** The generation being skipped for this family. */ - generation: number - /** The typed apply failure that condemned it. */ - error: ProjectionApplyError - /** Wall-clock ms when it was quarantined (diagnostic). */ - at: number -} - -/** How an advance ended — the four answer classes (see the module header). */ -export type AdvanceStatus = 'caught-up' | 'preempted' | 'budget-exhausted' | 'quarantined' - -/** The result of one advance over one family. */ -export interface AdvanceResult { - /** The answer class. */ - status: AdvanceStatus - /** The family's watermark as stamped by its own adapter, after this advance. */ - watermark: number | null - /** - * Facts delivered in SUCCESSFUL `applyBatch` calls during this advance. - * At-least-once delivery means retried facts (after a quarantine or a - * resume) count again; this is delivered work, not distinct generations. - */ - applied: number -} - -/** The result of a completed {@link ReprojectionEngine.swap}. */ -export interface SwapResult { - /** The NEW adapter's watermark at the flip (parity with the head). */ - watermark: number | null - /** Facts delivered to the replacement during its beside-build. */ - applied: number -} - -/** Constructor options for {@link ReprojectionEngine}. */ -export interface ReprojectionEngineOptions { - /** The committed-fact scan every family folds from. */ - source: FactSource - /** The preemption signal; a fresh one is created when omitted. */ - doorSignal?: DoorSignal - /** - * Installment ceiling in ms, `(0, MAX_INSTALLMENT_MS]`. Out-of-range values - * throw — the 50ms law is a ceiling, never a suggestion. - */ - installmentMs?: number - /** Facts per {@link FactSource.scan} pull (default {@link DEFAULT_REPROJECTION_BATCH_SIZE}). */ - batchSize?: number -} - -/** The fold-side state shared by a serving family and a swap's beside-build. */ -interface FoldState { - adapter: ProjectionAdapter - /** The quarantine ledger, in condemnation order. */ - quarantine: QuarantineEntry[] - /** Generations filtered out of every batch served to this adapter. */ - skip: Set - /** Next ledger size that triggers a narration (1, 2, 4, 8, …). */ - nextWarnAt: number -} - -/** A registered family: fold state plus the single-flight swap latch. */ -interface FamilyState extends FoldState { - swapInFlight: boolean -} - -/** One real macrotask boundary — foreground I/O and timers run before resume. */ -function yieldToDoors(): Promise { - return new Promise((resolve) => { - if (typeof setImmediate === 'function') { - setImmediate(resolve) - } else { - setTimeout(resolve, 0) - } - }) -} - -/** - * The reprojection engine: registry of projection families, budget-capped - * yielding advances, round-robin `advanceAll`, atomic build-beside `swap`, - * and the per-family quarantine ledger. Pure TS, no storage dependencies — - * everything durable lives behind the injected {@link FactSource} and the - * registered {@link ProjectionAdapter}s. - */ -export class ReprojectionEngine { - /** The preemption signal foreground door traffic bumps. */ - readonly doorSignal: DoorSignal - - private readonly source: FactSource - private readonly installmentMs: number - private readonly batchSize: number - private readonly registry = new Map() - /** Rotates the family that leads each `advanceAll`, so repeated tiny-budget calls stay fair. */ - private roundRobinCursor = 0 - - /** @param options - See {@link ReprojectionEngineOptions}. */ - constructor(options: ReprojectionEngineOptions) { - if (!options || typeof options.source?.scan !== 'function') { - throw new Error('reprojection: a FactSource with scan(from, limit) is required') - } - const installmentMs = options.installmentMs ?? MAX_INSTALLMENT_MS - if (!(installmentMs > 0) || installmentMs > MAX_INSTALLMENT_MS) { - throw new Error( - `reprojection: installmentMs must be in (0, ${MAX_INSTALLMENT_MS}] — ` + - `${installmentMs} would let maintenance hold the doors` - ) - } - const batchSize = options.batchSize ?? DEFAULT_REPROJECTION_BATCH_SIZE - if (!Number.isInteger(batchSize) || batchSize < 1) { - throw new Error(`reprojection: batchSize must be a positive integer (got ${batchSize})`) - } - this.source = options.source - this.doorSignal = options.doorSignal ?? new DoorSignal() - this.installmentMs = installmentMs - this.batchSize = batchSize - } - - /** - * Register a projection family. Refuses a duplicate family loudly — the - * sanctioned way to replace a serving adapter is {@link swap}, never - * re-registration. - * @param adapter - The adapter that will serve this family. - */ - register(adapter: ProjectionAdapter): void { - if (!adapter || typeof adapter.family !== 'string' || adapter.family.length === 0) { - throw new Error('reprojection: adapter.family must be a non-empty string') - } - if (this.registry.has(adapter.family)) { - throw new Error( - `reprojection: family '${adapter.family}' is already registered — ` + - `replace a serving adapter via swap(), never by re-registering` - ) - } - this.registry.set(adapter.family, { - adapter, - quarantine: [], - skip: new Set(), - nextWarnAt: 1, - swapInFlight: false - }) - } - - /** - * The adapter currently serving `family` (observability — e.g. asserting - * the old adapter still serves during a swap's beside-build), or undefined - * when the family is not registered. - * @param family - The family name. - */ - getAdapter(family: string): ProjectionAdapter | undefined { - return this.registry.get(family)?.adapter - } - - /** - * This family's quarantine ledger (a defensive copy, condemnation order). - * Non-empty means one or more generations are being skipped for this - * family — the projection owner should refuse reads the skipped facts - * would have affected. - * @param family - The family name (must be registered). - */ - quarantined(family: string): QuarantineEntry[] { - return [...this.mustGet(family).quarantine] - } - - /** - * Advance one family toward the head of the fact log (or toward `upTo`), - * in installments, under a wall-clock budget, preemptible by the door - * signal. Always makes at least ONE step of progress before any budget - * check, so a zero budget still advances. - * - * @param family - The registered family to advance. - * @param options - `budgetMs` caps this call's wall time (≥ 0); `upTo` - * optionally caps the fold at a generation (inclusive). - * @returns The answer class with the adapter-stamped watermark and the - * count of facts delivered in successful applyBatch calls. - */ - async advance(family: string, options: { budgetMs: number; upTo?: number }): Promise { - const state = this.mustGet(family) - const budgetMs = options?.budgetMs - if (typeof budgetMs !== 'number' || !(budgetMs >= 0)) { - throw new Error(`reprojection: advance('${family}') requires budgetMs >= 0 (got ${budgetMs})`) - } - const start = Date.now() - const entryEpoch = this.doorSignal.epoch() - let installmentStart = start - let applied = 0 - - for (;;) { - const stepResult = await this.step(state, options.upTo) - applied += stepResult.applied - if (stepResult.done) { - return this.completed(state, applied) - } - // A bump ends the current installment immediately: yield a macrotask so - // the foreground work runs, then answer 'preempted'. - if (this.doorSignal.epoch() !== entryEpoch) { - await yieldToDoors() - return { status: 'preempted', watermark: state.adapter.watermark(), applied } - } - const t = Date.now() - if (t - start >= budgetMs) { - return { status: 'budget-exhausted', watermark: state.adapter.watermark(), applied } - } - if (t - installmentStart >= this.installmentMs) { - await yieldToDoors() - installmentStart = Date.now() - } - } - } - - /** - * Advance EVERY registered family toward the head under one shared budget, - * round-robin at batch granularity — one batch per family per turn — so no - * family starves behind another's backlog. The leading family rotates - * across calls, keeping repeated tiny-budget calls fair too. - * - * @param options - `budgetMs` caps this call's total wall time (≥ 0). - * @returns Per-family results. Families still mid-stream when the budget - * ran out (or a door bumped) report `'budget-exhausted'` (or - * `'preempted'`) at their current watermark. - */ - async advanceAll(options: { budgetMs: number }): Promise> { - const budgetMs = options?.budgetMs - if (typeof budgetMs !== 'number' || !(budgetMs >= 0)) { - throw new Error(`reprojection: advanceAll requires budgetMs >= 0 (got ${budgetMs})`) - } - const start = Date.now() - const entryEpoch = this.doorSignal.epoch() - let installmentStart = start - - const all = [...this.registry.values()] - const results: Record = {} - const appliedBy = new Map() - if (all.length === 0) return results - - // Rotate the leader across calls (fairness across repeated small budgets). - const offset = this.roundRobinCursor % all.length - this.roundRobinCursor = (this.roundRobinCursor + 1) % all.length - let queue = [...all.slice(offset), ...all.slice(0, offset)] - for (const s of queue) appliedBy.set(s.adapter.family, 0) - - const finish = ( - status: 'preempted' | 'budget-exhausted', - remaining: FamilyState[] - ): Record => { - for (const s of remaining) { - results[s.adapter.family] = { - status, - watermark: s.adapter.watermark(), - applied: appliedBy.get(s.adapter.family) ?? 0 - } - } - return results - } - - while (queue.length > 0) { - const survivors: FamilyState[] = [] - for (let i = 0; i < queue.length; i++) { - const s = queue[i] - const fam = s.adapter.family - const stepResult = await this.step(s, undefined) - appliedBy.set(fam, (appliedBy.get(fam) ?? 0) + stepResult.applied) - if (stepResult.done) { - results[fam] = this.completed(s, appliedBy.get(fam) ?? 0) - } else { - survivors.push(s) - } - const remaining = [...survivors, ...queue.slice(i + 1)] - if (this.doorSignal.epoch() !== entryEpoch) { - await yieldToDoors() - return finish('preempted', remaining) - } - const t = Date.now() - if (t - start >= budgetMs && remaining.length > 0) { - return finish('budget-exhausted', remaining) - } - if (t - installmentStart >= this.installmentMs) { - await yieldToDoors() - installmentStart = Date.now() - } - } - queue = survivors - } - return results - } - - /** - * Replace a family's adapter by BUILD-BESIDE: the old adapter keeps serving - * (stays registered, its watermark untouched) while the replacement folds - * from its own watermark (null/0 for a fresh build) to parity with the head - * of the fact log. The flip is ATOMIC — a single registry pointer swap with - * no await between the parity check and the assignment — and the losing - * adapter's `discard()` is called after the flip. - * - * SINGLE-FLIGHT: a second concurrent swap on the same family throws a - * typed {@link SwapInFlightError}. The build yields at installment - * boundaries like any fold (doors interleave), but it is never - * preemption-aborted — a swap under steady foreground traffic still - * completes. - * - * On a build failure the partially-built replacement is discarded - * (best-effort, narrated if that also fails) and the error propagates; the - * old adapter keeps serving untouched. - * - * @param family - The registered family to replace. - * @param buildAdapter - Factory for the replacement adapter (same family). - * @returns The new adapter's watermark at the flip and the facts delivered - * during the build. - */ - async swap(family: string, buildAdapter: () => Promise): Promise { - const state = this.mustGet(family) - if (state.swapInFlight) throw new SwapInFlightError(family) - state.swapInFlight = true - try { - const next = await buildAdapter() - if (!next || next.family !== family) { - throw new Error( - `reprojection: swap('${family}') built an adapter for family ` + - `'${next?.family}' — the replacement must serve the same family` - ) - } - const build: FoldState = { adapter: next, quarantine: [], skip: new Set(), nextWarnAt: 1 } - let applied = 0 - let installmentStart = Date.now() - let stalledDoneAt: number | null = null - - try { - for (;;) { - const stepResult = await this.step(build, undefined) - applied += stepResult.applied - if (stepResult.applied > 0) stalledDoneAt = null - if (stepResult.done) { - // Parity: the build just saw an empty scan (caught up to the head - // as of that call). The serving adapter can never be beyond the - // head, so newWm >= oldWm holds — verified loudly, never assumed. - const oldWm = state.adapter.watermark() ?? 0 - const newWm = next.watermark() ?? 0 - if (newWm >= oldWm) break - if (stalledDoneAt === newWm) { - throw new Error( - `reprojection: swap('${family}') build is caught up to the head at ` + - `generation ${newWm} but the serving adapter claims watermark ${oldWm} — ` + - `the serving stamp is beyond the fact log; refusing to flip` - ) - } - // The head moved past our scan (a concurrent fold advanced the - // serving adapter) — keep folding to the new head. - stalledDoneAt = newWm - } - if (Date.now() - installmentStart >= this.installmentMs) { - await yieldToDoors() - installmentStart = Date.now() - } - } - } catch (err) { - await next.discard().catch((cleanupErr) => { - prodLog.warn( - `reprojection: swap('${family}') build failed AND the failed build's discard() ` + - `also failed — its artifact may be orphaned`, - cleanupErr - ) - }) - throw err - } - - // THE FLIP — atomic by construction: no await between the parity check - // above and this pointer swap; readers see the old adapter until this - // line and the new one from it. - const losing = state.adapter - state.adapter = next - state.quarantine = build.quarantine - state.skip = build.skip - state.nextWarnAt = build.nextWarnAt - - try { - await losing.discard() - } catch (discardErr) { - // The flip already happened and the new adapter serves; the only loss - // is the loser's orphaned artifact — said out loud, never rethrown as - // a false swap failure. - prodLog.warn( - `reprojection: swap('${family}') completed but the losing adapter's discard() ` + - `failed — its artifact may be orphaned`, - discardErr - ) - } - return { watermark: next.watermark(), applied } - } finally { - state.swapInFlight = false - } - } - - /** One fold step: scan a batch above the watermark, filter quarantined generations, apply. */ - private async step(state: FoldState, upTo: number | undefined): Promise<{ done: boolean; applied: number }> { - const from = state.adapter.watermark() ?? 0 - if (upTo !== undefined && from >= upTo) return { done: true, applied: 0 } - let facts = await this.source.scan(from, this.batchSize) - if (facts.length === 0) return { done: true, applied: 0 } - if (upTo !== undefined) { - facts = facts.filter((f) => f.generation <= upTo) - if (facts.length === 0) return { done: true, applied: 0 } - } - const batchUpTo = facts[facts.length - 1].generation - const toApply = state.skip.size > 0 ? facts.filter((f) => !state.skip.has(f.generation)) : facts - try { - await state.adapter.applyBatch(toApply, batchUpTo) - } catch (err) { - if (err instanceof ProjectionApplyError) { - this.recordQuarantine(state, err) - return { done: false, applied: 0 } - } - throw err // unknown failure ≠ poison record — abort the advance loudly - } - // Anti-spin guard: a successful applyBatch that never advances the stamp - // would re-serve the same window forever. Refuse loudly instead. - const after = state.adapter.watermark() ?? 0 - if (after <= from) { - throw new Error( - `reprojection: family '${state.adapter.family}' applyBatch succeeded up to ` + - `generation ${batchUpTo} but the watermark did not advance past ${from} — ` + - `the adapter is not stamping; refusing to spin` - ) - } - return { done: false, applied: toApply.length } - } - - /** Ledger a typed apply failure, skip its generation, narrate per-doubling. */ - private recordQuarantine(state: FoldState, err: ProjectionApplyError): void { - if (!Number.isFinite(err.generation)) { - throw new Error( - `reprojection: family '${state.adapter.family}' threw ProjectionApplyError with a ` + - `non-finite generation (${err.generation}) — cannot quarantine; aborting the advance` - ) - } - if (state.skip.has(err.generation)) { - throw new Error( - `reprojection: family '${state.adapter.family}' threw ProjectionApplyError for ` + - `generation ${err.generation}, which is ALREADY quarantined and was not in the ` + - `batch — the adapter is misreporting; aborting the advance` - ) - } - state.skip.add(err.generation) - state.quarantine.push({ generation: err.generation, error: err, at: Date.now() }) - const n = state.quarantine.length - if (n === state.nextWarnAt) { - state.nextWarnAt *= 2 - prodLog.warn( - `reprojection: family '${state.adapter.family}' quarantined generation ` + - `${err.generation} (${n} quarantined total) — the fact is skipped for this family ` + - `and ledgered; reads it would have affected should be refused by the owner`, - err.cause - ) - } - } - - /** A window completed: 'caught-up' with a clean ledger, 'quarantined' otherwise. */ - private completed(state: FoldState, applied: number): AdvanceResult { - return { - status: state.quarantine.length > 0 ? 'quarantined' : 'caught-up', - watermark: state.adapter.watermark(), - applied - } - } - - /** The registered family state, or a loud refusal. */ - private mustGet(family: string): FamilyState { - const state = this.registry.get(family) - if (!state) throw new Error(`reprojection: family '${family}' is not registered`) - return state - } -} diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fea817d5..5eb4785a 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -18,11 +18,6 @@ import { } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' -import { - TornRecordError, - isUnparseablePayloadError, - registerTornRecordEncounter -} from '../tornRecordError.js' // Node.js modules - dynamically imported to avoid issues in browser environments let fs: any @@ -415,22 +410,8 @@ export class FileSystemStorage extends BaseStorage { /** * Primitive operation: Read object from path * All metadata operations use this internally via base class routing + * Enhanced error handling for corrupted metadata files (Bug #3 mitigation) * Supports reading both compressed (.gz) and uncompressed files for backward compatibility - * - * Read contract (loud errors, never quiet losses): - * - Genuine absence (ENOENT on every variant) → `null`. Only a missing file - * is "not found". - * - TORN record (a file EXISTS but its bytes cannot be decoded — invalid - * JSON, truncated/garbled gzip) → the encounter is registered (production - * ERROR log + per-process gauge) and a typed {@link TornRecordError} is - * thrown. Corruption must NEVER read as absence: callers that can degrade - * (manifest recovery, rebuildable statistics) catch the typed error at - * their sites; entity reads surface it. - * Legacy dual-format exception: when the `.gz` variant is torn but the - * uncompressed fallback decodes, the recovered object is returned — AFTER - * the torn `.gz` was logged and counted (loud recovery, not a silent skip). - * - Real storage fault (EIO/EACCES/EMFILE/…) → propagates as itself; a - * fault is neither absence nor corruption and must not be reshaped. */ protected async readObjectFromPath(pathStr: string): Promise { await this.ensureInitialized() @@ -438,10 +419,7 @@ export class FileSystemStorage extends BaseStorage { const fullPath = path.join(this.rootDir, pathStr) const compressedPath = `${fullPath}.gz` - // Try reading compressed file first (if compression is enabled or file exists). - // A torn .gz is remembered so the uncompressed fallback can either recover - // (legacy dual-format installs) or surface the corruption typed. - let tornCompressed: TornRecordError | null = null + // Try reading compressed file first (if compression is enabled or file exists) try { const compressedData = await fs.promises.readFile(compressedPath) const decompressed = await new Promise((resolve, reject) => { @@ -452,16 +430,9 @@ export class FileSystemStorage extends BaseStorage { }) return JSON.parse(decompressed.toString('utf-8')) } catch (error: any) { - if (error.code === 'ENOENT') { - // No compressed variant — fall through to the uncompressed path. - } else if (isUnparseablePayloadError(error)) { - // The .gz EXISTS but cannot be decoded (zlib Z_* error or JSON - // SyntaxError after gunzip): torn record. Register NOW (log + gauge), - // then attempt the uncompressed fallback as a recovery read. - tornCompressed = registerTornRecordEncounter(`${pathStr}.gz`, error) - } else { - // Real storage fault on an existing .gz (EIO/EACCES/…): propagate. - throw error + // If compressed file doesn't exist, fall back to uncompressed + if (error.code !== 'ENOENT') { + console.warn(`Failed to read compressed file ${compressedPath}:`, error) } } @@ -471,26 +442,24 @@ export class FileSystemStorage extends BaseStorage { return JSON.parse(data) } catch (error: any) { if (error.code === 'ENOENT') { - // No uncompressed file. If the .gz variant existed but was torn, the - // object EXISTS and is unreadable — that must surface typed, never as - // "absent". Otherwise this is genuine absence. - if (tornCompressed !== null) { - throw tornCompressed - } return null } - // The file EXISTS but its content cannot be parsed: torn record. - // Register (production ERROR + gauge) and throw typed — a corrupt row - // must be distinguishable from a missing row, or nothing ever heals it. - if (isUnparseablePayloadError(error)) { - throw registerTornRecordEncounter(pathStr, error) + // Enhanced error handling for corrupted JSON files (race condition from Bug #3) + if (error instanceof SyntaxError || error.name === 'SyntaxError') { + console.warn( + `⚠️ Corrupted metadata file detected: ${pathStr}\n` + + ` This may be caused by concurrent writes during import.\n` + + ` Gracefully skipping this entry. File may be repaired on next write.` + ) + return null } // A real storage fault (EIO/EACCES/EMFILE/…) is NOT "object absent". The - // ENOENT branch (above) already returns null; a genuine fault reaching - // here must propagate loudly rather than masquerade as a missing object - // — which would corrupt reads and drive needless rebuilds. + // ENOENT branch (above) already returns null, and the corrupted-JSON + // branch (above) is a deliberate concurrent-write tolerance; a genuine + // fault reaching here must propagate loudly rather than masquerade as a + // missing object — which would corrupt reads and drive needless rebuilds. throw error } } @@ -1816,32 +1785,6 @@ export class FileSystemStorage extends BaseStorage { const now = new Date().toISOString() const existing = await this.readWriterLock() - // TORN-LOCK RECOVERY: power loss can legally leave the lock file - // present but EMPTY/unparseable (the claim's non-atomic write died - // mid-flight). readWriterLock() reports it as null — but the O_EXCL - // claim below would EEXIST forever, a PERMANENT lockout no staleness - // check can clear (staleness needs a parsed PID). A torn lock is - // stale BY DEFINITION: no live holder has one (a holder either - // completed its write or is dead). Unlink loudly and re-loop; a - // racer that rewrites a VALID lock first simply wins the next read. - if (existing === null) { - try { - await fs.promises.access(lockFile) - console.warn( - `[brainy] Writer lock at ${lockFile} exists but is unreadable/unparseable ` + - `(torn write from a previous power loss) — treating as stale and removing.` - ) - try { - await fs.promises.unlink(lockFile) - } catch (unlinkErr: any) { - if (unlinkErr.code !== 'ENOENT') throw unlinkErr - } - } catch (accessErr: any) { - if (accessErr.code !== 'ENOENT') throw accessErr - // Absent: the normal fresh-claim path below. - } - } - if (existing) { // Same-process re-open: a second Brainy instance in this Node process // (e.g. test "simulate server restart" patterns, or a consumer that diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 23003ede..b78b4a49 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -32,7 +32,6 @@ import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js' import { unwrapBinaryData } from './binaryDataCodec.js' import { prodLog } from '../utils/logger.js' import { isAbsentError } from '../utils/errorClassification.js' -import { isTornRecordError } from './tornRecordError.js' import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from '../errors/brainyError.js' import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' import { @@ -675,11 +674,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { // — hash verification must run on the original content bytes. return unwrapBinaryData(data) } catch (error) { - // A TORN blob object (exists but undecodable) must not read as - // "blob absent" — that would misdiagnose disk corruption as a - // missing blob. This is an IDENTITY read (a caller asked for THIS - // key): propagate the typed error to the blob layer. - if (isTornRecordError(error)) throw error return undefined } }, @@ -774,20 +768,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (m) hashes.add(m[1]) } - // Recovery-path read: a TORN object here maps to "not usable" (null) BY - // DESIGN — the adapter has already logged + counted the encounter, and - // treating a torn `_cas/` copy as absent lets the re-copy from `_cow/` - // OVERWRITE the corrupt file with the good original (the heal), while a - // torn `_cow/` original is reported via `incomplete`. Real faults propagate. - const readOrNullIfTorn = async (p: string): Promise => { - try { - return await this.readObjectFromPath(p) - } catch (error) { - if (isTornRecordError(error)) return null - throw error - } - } - let adopted = 0 let alreadyPresent = 0 let incomplete = 0 @@ -795,15 +775,15 @@ export abstract class BaseStorage extends BaseStorageAdapter { // A blob counts as present only when BOTH its bytes and its metadata // already live in `_cas/`. A half-adopted blob (bytes without meta — the // exact "Blob metadata not found" state) is re-adopted. - const casBlob = await readOrNullIfTorn(`_cas/blob:${hash}`) - const casMeta = await readOrNullIfTorn(`_cas/blob-meta:${hash}`) + const casBlob = await this.readObjectFromPath(`_cas/blob:${hash}`) + const casMeta = await this.readObjectFromPath(`_cas/blob-meta:${hash}`) if (casBlob !== null && casMeta !== null) { alreadyPresent++ continue } - const cowBlob = await readOrNullIfTorn(`_cow/blob:${hash}`) - const cowMeta = await readOrNullIfTorn(`_cow/blob-meta:${hash}`) + const cowBlob = await this.readObjectFromPath(`_cow/blob:${hash}`) + const cowMeta = await this.readObjectFromPath(`_cow/blob-meta:${hash}`) if (cowBlob === null || cowMeta === null) { // Can't register a blob the store can't fully describe — report it so an // operator investigates rather than silently half-adopting. @@ -1154,28 +1134,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { * cache (record-layer files are written through * {@link BaseStorage.writeRawObject} only). * - * TORN-record contract (deliberate, loud-by-design): this surface serves - * SYSTEM ARTIFACTS — manifests with recovery paths, markers whose verdict - * machinery treats "unreadable" as rescan, generation/transaction records - * whose recovery is built for absent artifacts. For these readers a torn - * file maps to their existing absent-artifact degrade, so a typed - * torn-record error from the adapter is caught here and returned as `null` - * — AFTER the adapter has already logged a production ERROR and counted - * the per-process torn-record gauge (never silent). Entity reads do NOT go - * through this surface; they use the canonical read paths, which propagate - * the typed error. Real storage faults (EIO/EACCES/…) still propagate. - * * @param path - Storage-root-relative object path (e.g. `_system/manifest.json`). - * @returns The parsed object, or `null` if absent (or torn — logged + counted). + * @returns The parsed object, or `null` if absent. */ public async readRawObject(path: string): Promise { await this.ensureInitialized() - try { - return await this.readObjectFromPath(path) - } catch (error) { - if (isTornRecordError(error)) return null - throw error - } + return this.readObjectFromPath(path) } /** @@ -2182,13 +2146,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (!metadata) return null return { deserialized, metadata } } catch (error) { - // A TORN record must surface typed — a paginated read that - // silently skips a corrupt row hides data loss from the caller. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip nouns that fail to load return null } @@ -2218,12 +2175,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } } catch (error) { - // A TORN record propagates (typed) — only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -2332,13 +2283,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { batch.map(async (id) => { try { return { id, metadata: await this.getNounMetadata(id) } - } catch (error) { - // A TORN record must surface typed, never as a skipped id. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } + } catch { return null } }) @@ -2360,12 +2305,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } } catch (error) { - // A TORN record propagates (typed) — only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards with no data } } @@ -2576,23 +2515,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { // reserved fields top-level, ONLY custom fields in `metadata`. collected.push({ verb: this.hydrateVerbWithMetadata(verb, metadata), shard }) } catch (error) { - // A TORN record must surface typed — a paginated read that - // silently skips a corrupt row hides data loss from the caller. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load } } } catch (error) { - // A TORN record propagates (typed) — only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -3377,14 +3303,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getNounMetadataPath(id) // Determine if this is a new entity by checking if metadata already exists - // Torn-tolerant: a WRITE landing on a torn record HEALS it — the read - // here only classifies new-vs-update and captures the prior subtype; - // a torn prior reads as "no previous" (fresh write) with the adapter's - // loud floor already fired. Never let corruption block its own cure. - const existingMetadata = await this.readCanonicalObject(path).catch((err) => { - if ((err as { code?: string }).code === 'TORN_RECORD') return null - throw err - }) + const existingMetadata = await this.readCanonicalObject(path) const isNew = !existingMetadata // Save the metadata (write-cache coherent canonical write) @@ -3750,22 +3669,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { ) for (const result of chunkResults) { - if (result.status === 'fulfilled') { - if (result.value.data !== null) { - results.set(result.value.path, result.value.data) - } - } else if (isTornRecordError(result.reason)) { - // A torn record inside a SET-SHAPED read (batch hydration behind - // find/sort pages and recovery walks): the adapter narrated + - // counted at throw time; the batch HEALS PAST the victim and - // serves the remaining rows — one crash casualty must not kill - // every query that pages over its shard (and init-time recovery - // walks ride this exact path). Identity point-reads still throw. - continue - } else { - // A REAL storage fault (EIO-class) is not a torn victim — - // propagate loudly, never absorb. - throw result.reason + if (result.status === 'fulfilled' && result.value.data !== null) { + results.set(result.value.path, result.value.data) } } } @@ -3901,14 +3806,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getVerbMetadataPath(id) // Determine if this is a new verb by checking if metadata already exists - // Torn-tolerant: a WRITE landing on a torn record HEALS it — the read - // here only classifies new-vs-update and captures the prior subtype; - // a torn prior reads as "no previous" (fresh write) with the adapter's - // loud floor already fired. Never let corruption block its own cure. - const existingMetadata = await this.readCanonicalObject(path).catch((err) => { - if ((err as { code?: string }).code === 'TORN_RECORD') return null - throw err - }) + const existingMetadata = await this.readCanonicalObject(path) const isNew = !existingMetadata // Save the metadata (write-cache coherent canonical write) @@ -4738,23 +4636,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } } catch (error) { - // A TORN record must surface typed — an enumeration that silently - // skips a corrupt row hides data loss from the caller. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip nouns that fail to load } } } catch (error) { - // A TORN record propagates (typed) — only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -4940,24 +4825,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { results.push(this.hydrateVerbWithMetadata(verb, metadata)) } } catch (error) { - // A TORN record must surface typed — an enumeration that silently - // skips a corrupt row hides data loss from the caller. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load prodLog.debug(`[BaseStorage] Failed to load verb from ${verbPath}:`, error) } } } catch (error) { - // A TORN record propagates (typed) — only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -5073,13 +4945,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { sourceVerbs.push(hydratedVerb) } } catch (error) { - // A TORN record propagates (typed) — batch hydration must not - // silently drop a corrupt row. Only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -5165,23 +5030,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { results.push(this.hydrateVerbWithMetadata(verb, metadata)) } } catch (error) { - // A TORN record must surface typed — an enumeration that silently - // skips a corrupt row hides data loss from the caller. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load } } } catch (error) { - // A TORN record propagates (typed) — only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } @@ -5226,23 +5078,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { ) ) } catch (error) { - // A TORN record must surface typed — an enumeration that silently - // skips a corrupt row hides data loss from the caller. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip verbs that fail to load } } } catch (error) { - // A TORN record propagates (typed) — only shard-listing absence is skippable. - // Torn record inside an ENUMERATION/RECOVERY walk: the adapter already - // narrated + counted it (TornRecordError registers at creation); the - // walk's job is to HEAL PAST it — skip the victim, serve the rest. - // Identity point-reads (get-by-id) still throw typed upstream. - if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } // Skip shards that have no data } } diff --git a/src/storage/tornRecordError.ts b/src/storage/tornRecordError.ts deleted file mode 100644 index e3248f80..00000000 --- a/src/storage/tornRecordError.ts +++ /dev/null @@ -1,132 +0,0 @@ -/** - * @module storage/tornRecordError - * @description Typed surface for TORN records — files that EXIST in storage but - * cannot be decoded (invalid JSON, truncated/garbled gzip). A torn record is - * disk corruption, not absence: reading it as `null` ("not found") makes the - * consumer unable to distinguish "never existed" from "exists but unreadable", - * so nothing ever heals it. Mandate: loud errors, never quiet losses. - * - * Contract implemented across the storage layer: - * - Genuine absence (ENOENT) still reads as clean `null` — no error, no noise. - * - A torn record ALWAYS registers here (error log + per-process gauge), then: - * - entity read paths (get/getBatch/pagination/enumeration hydration) throw - * {@link TornRecordError} to the caller — a row is never silently dropped; - * - system-artifact read paths whose machinery is designed for - * absent-artifact degradation (manifests with recovery paths, markers - * whose verdict is "rescan", rebuildable statistics) map torn → their - * existing degrade AFTER the encounter is logged and counted. - */ - -import { prodLog } from '../utils/logger.js' - -/** - * @description Thrown when a stored object EXISTS but cannot be decoded — - * corrupt/torn bytes on disk (invalid JSON, undecodable gzip). Deliberately - * distinct from absence: `readObjectFromPath` returns `null` only for ENOENT. - * Catchable by type (`instanceof`), by `name === 'TornRecordError'`, or by - * `code === 'TORN_RECORD'` (cross-realm safe; never matches `isAbsentError`). - */ -export class TornRecordError extends Error { - /** Stable machine-checkable discriminator (errno-style). */ - public readonly code = 'TORN_RECORD' - /** Storage-root-relative path of the torn object. */ - public readonly path: string - /** The underlying decode failure (SyntaxError, zlib error, …). */ - public override readonly cause: unknown - - /** - * @param path - Storage-root-relative path of the torn object. - * @param cause - The underlying decode failure. - */ - constructor(path: string, cause: unknown) { - const causeMessage = - cause instanceof Error ? cause.message : String(cause) - super( - `Torn record at '${path}': file exists but cannot be decoded (${causeMessage}). ` + - `This is storage corruption, not absence — the record was not silently skipped.` - ) - this.name = 'TornRecordError' - this.path = path - this.cause = cause - } -} - -/** - * @description True IFF `e` is a torn-record error — matches by `instanceof` - * first, then by `name`/`code` so errors crossing module-duplication or realm - * boundaries are still recognized. - * @param e - The caught value. - * @returns Whether `e` denotes an existing-but-undecodable stored object. - */ -export function isTornRecordError(e: unknown): e is TornRecordError { - if (e instanceof TornRecordError) return true - if (e === null || typeof e !== 'object') return false - const { name, code } = e as { name?: unknown; code?: unknown } - return name === 'TornRecordError' || code === 'TORN_RECORD' -} - -/** - * @description True IFF `e` is a payload-decode failure — the file's BYTES were - * read fine but could not be turned back into an object: `SyntaxError` from - * `JSON.parse`, or a zlib error (`Z_DATA_ERROR`, `Z_BUF_ERROR`, …) from gunzip. - * Distinguishes "torn record" from real I/O faults (EIO/EACCES/…), which must - * propagate as themselves. - * @param e - The caught value. - * @returns Whether the error means "bytes present, content undecodable". - */ -export function isUnparseablePayloadError(e: unknown): boolean { - if (e === null || typeof e !== 'object') return false - if (e instanceof SyntaxError) return true - const { name, code } = e as { name?: unknown; code?: unknown } - if (name === 'SyntaxError') return true - return typeof code === 'string' && code.startsWith('Z_') -} - -/** Per-process torn-record gauge state (module-scoped; see the accessors). */ -let tornRecordCount = 0 -let lastTornRecordPath: string | null = null - -/** - * @description Register a torn-record encounter: logs a production ERROR - * naming the path, increments the per-process gauge, and returns the typed - * error for the caller to throw (or to map into a documented loud degrade). - * EVERY torn encounter goes through here, whatever the caller decides — - * the floor is: never silent. - * @param path - Storage-root-relative path of the torn object. - * @param cause - The underlying decode failure. - * @returns The constructed {@link TornRecordError}. - */ -export function registerTornRecordEncounter( - path: string, - cause: unknown -): TornRecordError { - tornRecordCount++ - lastTornRecordPath = path - const error = new TornRecordError(path, cause) - prodLog.error( - `[Storage] TORN RECORD #${tornRecordCount}: '${path}' exists but cannot be decoded — ` + - `corrupt or partially written bytes. Cause: ${ - cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause) - }` - ) - return error -} - -/** - * @description Read the per-process torn-record gauge: how many torn records - * this process has encountered and the most recent path. Observability seam — - * lets operators and tests confirm that corruption was seen, not swallowed. - * @returns The current gauge snapshot. - */ -export function getTornRecordGauge(): { count: number; lastPath: string | null } { - return { count: tornRecordCount, lastPath: lastTornRecordPath } -} - -/** - * @description Reset the per-process torn-record gauge to zero. Test seam only - * (the gauge is process-lifetime state); production code never resets it. - */ -export function resetTornRecordGauge(): void { - tornRecordCount = 0 - lastTornRecordPath = null -} diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 139c67fe..d130bb3f 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -56,33 +56,16 @@ function resolveVectorProviderId(index: VectorIndexProvider): string { * or timing trace see which engine actually ran, never a fossil name from * whichever engine happened to be active when this op class was written. * - * Generation: `generationFn` is resolved at execute time (not construction) so - * the write is stamped at the transaction's in-flight commit generation — - * which the generation store only assigns once the batch begins executing. - * The same generation is reused for the rollback removal, so an add and its - * undo reference one watermark in a provider's per-record delta log (the - * exact pattern the graph operations established). - * * Rollback strategy: * - Remove item from index */ export class AddToVectorIndexOperation implements Operation { readonly name: string - /** - * @param index - The vector-index provider (JS HNSW or native). - * @param id - The entity's UUID. - * @param vector - The vector to index. - * @param generationFn - OPTIONAL: resolves the commit generation to stamp - * this write at, evaluated when the operation executes (see class note). - * Absent -> the provider receives no generation (undefined), never a - * fabricated 0. - */ constructor( private readonly index: VectorIndexProvider, private readonly id: string, - private readonly vector: number[], - private readonly generationFn?: () => bigint | undefined + private readonly vector: number[] ) { this.name = `AddToVectorIndex(${resolveVectorProviderId(index)})` } @@ -91,18 +74,14 @@ export class AddToVectorIndexOperation implements Operation { // Check if item already exists (for rollback decision) const existed = await this.itemExists(this.id) - // Stamp this write at the in-flight commit generation; reuse it for the - // rollback so add + undo reference the same watermark. - const generation = this.generationFn?.() - // Add to index - await this.index.addItem({ id: this.id, vector: this.vector }, generation) + await this.index.addItem({ id: this.id, vector: this.vector }) // Return rollback action return async () => { if (!existed) { // Remove newly added item - await this.index.removeItem(this.id, generation) + await this.index.removeItem(this.id) } // If item existed before, we don't rollback (update is OK) // This prevents index corruption from removing pre-existing items @@ -152,138 +131,22 @@ export class AddToVectorIndexOperation implements Operation { export class RemoveFromVectorIndexOperation implements Operation { readonly name: string - /** - * @param index - The vector-index provider (JS HNSW or native). - * @param id - The entity's UUID. - * @param vector - The removed vector (required for rollback re-add). - * @param generationFn - Resolves the commit generation for this removal, - * evaluated when the operation executes; reused for the rollback re-add - * so the round trip references one watermark. - */ constructor( private readonly index: VectorIndexProvider, private readonly id: string, - private readonly vector: number[], // Required for rollback - private readonly generationFn?: () => bigint | undefined + private readonly vector: number[] // Required for rollback ) { this.name = `RemoveFromVectorIndex(${resolveVectorProviderId(index)})` } async execute(): Promise { - // Resolve the removal generation once; reuse it for the rollback re-add. - const generation = this.generationFn?.() - // Remove from index - await this.index.removeItem(this.id, generation) + await this.index.removeItem(this.id) // Return rollback action return async () => { // Re-add item with original vector - await this.index.addItem({ id: this.id, vector: this.vector }, generation) - } - } -} - -/** - * Replace an item's vector in the vector index as ONE atomic transaction leg — - * the row is never absent from vector search during an update. - * - * Backend-neutral: see {@link AddToVectorIndexOperation} — `index` may be the - * JS HNSW fallback or a native acceleration provider; the emitted `name` - * stamps the active backend. - * - * Why this op exists: update flows historically staged a - * {@link RemoveFromVectorIndexOperation} followed by an - * {@link AddToVectorIndexOperation} as two separately-awaited operations. - * Between them the row was in NEITHER index — dark to semantic recall while - * perfectly visible to metadata reads (a transient-invisibility window that - * stretched to seconds in a production deployment). The structural cure is a - * single leg that never removes without simultaneously re-inserting. - * - * Execution strategy (feature-detected, in preference order): - * 1. Provider exposes `updateItem` → ONE in-place call. The provider swaps - * the vector without the row ever leaving its index, and an element-wise - * UNCHANGED vector (the type-only-update production shape) is a pure - * no-op on its side. - * 2. Provider without `updateItem` (a native provider that has not shipped - * it yet) → `removeItem` + `addItem` executed ADJACENT within this single - * op. Still strictly better than the historical pair: no other transaction - * operation can interleave between the two calls. This is a temporary - * seam — the native side of the pair is expected to ship its own - * `updateItem` so path 1 applies everywhere; when it does, this fallback - * becomes dead code that costs nothing. - * - * Rollback strategy (mirrors the execute branch that ran): - * - `updateItem` path → `updateItem` back to `oldVector`. - * - Fallback path → `removeItem` + `addItem` back to `oldVector`. - * - * Rollback semantics when the item did not exist at execute time: this op's - * contract is that the caller read the entity and its CURRENT vector - * (`oldVector`) before staging — update flows only stage it for existing - * rows. If the item was somehow absent, execute() inserts it (`updateItem` - * delegates to add; the fallback's remove is a no-op before its add), and - * rollback restores `oldVector` rather than removing — the same posture as - * {@link RemoveFromVectorIndexOperation}'s unconditional re-add: by - * constructing the op with `oldVector` the caller DECLARED the before-state, - * and rollback reconstructs that declared state instead of silently deciding - * the row should vanish. - */ -export class ReplaceInVectorIndexOperation implements Operation { - readonly name: string - - /** - * @param index - The vector-index provider (JS HNSW or native). - * @param id - The entity's UUID. - * @param oldVector - The pre-update vector (required for rollback). - * @param newVector - The replacement vector. - * @param generationFn - Resolves the commit generation to stamp this write - * at, evaluated when the operation executes and reused across both - * execute branches AND the rollback — one watermark for the whole - * replace round trip. - */ - constructor( - private readonly index: VectorIndexProvider, - private readonly id: string, - private readonly oldVector: number[], // Required for rollback - private readonly newVector: number[], - private readonly generationFn?: () => bigint | undefined - ) { - this.name = `ReplaceInVectorIndex(${resolveVectorProviderId(index)})` - } - - async execute(): Promise { - // Feature-detect the in-place capability — optional on the provider - // contract, like `getItem`/`setPersistMode` (Brainy's JS HNSW index - // ships it; a native provider may not have yet). The capability carries - // the same optional trailing generation as the required write surface. - const index = this.index as VectorIndexProvider & { - updateItem?: (item: { id: string; vector: number[] }, generation?: bigint) => Promise - } - - // One commit generation for the whole replace (both branches + rollback). - const generation = this.generationFn?.() - - if (typeof index.updateItem === 'function') { - // Atomic path: one in-place call, the row never leaves the index. - await index.updateItem({ id: this.id, vector: this.newVector }, generation) - - return async () => { - // Restore the declared before-state in place (see class JSDoc for - // the item-did-not-exist posture). - await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) - } - } - - // Fallback seam: remove+add ADJACENT within this single op — no other - // transaction operation can interleave between them (see class JSDoc). - await this.index.removeItem(this.id, generation) - await this.index.addItem({ id: this.id, vector: this.newVector }, generation) - - return async () => { - // updateItem-style restore via the same adjacent pair, back to the - // declared before-state. - await this.index.removeItem(this.id, generation) - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + await this.index.addItem({ id: this.id, vector: this.vector }) } } } @@ -291,43 +154,26 @@ export class ReplaceInVectorIndexOperation implements Operation { /** * Add to metadata index with rollback support * - * Generation: `generationFn` is resolved at execute time (not construction) — - * see {@link AddToVectorIndexOperation}'s class note; the same generation is - * reused for the rollback removal so add + undo reference one watermark in a - * provider's per-record delta log. - * * Rollback strategy: * - Remove item from index */ export class AddToMetadataIndexOperation implements Operation { readonly name = 'AddToMetadataIndex' - /** - * @param index - The metadata-index manager (JS baseline or a registered provider). - * @param id - The entity's UUID. - * @param entity - Entity or metadata structure to index. - * @param generationFn - Resolves the commit generation to stamp this write - * at, evaluated when the operation executes. - */ constructor( private readonly index: MetadataIndexManager, private readonly id: string, - private readonly entity: any, // Entity or metadata structure - private readonly generationFn?: () => bigint | undefined + private readonly entity: any // Entity or metadata structure ) {} async execute(): Promise { - // Stamp this write at the in-flight commit generation; reuse it for the - // rollback so add + undo reference the same watermark. - const generation = this.generationFn?.() - // Add to metadata index (skipFlush=true for transaction atomicity) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + await this.index.addToIndex(this.id, this.entity, true) // Return rollback action return async () => { // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + await this.index.removeFromIndex(this.id, this.entity) } } } @@ -335,41 +181,26 @@ export class AddToMetadataIndexOperation implements Operation { /** * Remove from metadata index with rollback support * - * Generation: resolved at execute time and reused for the rollback re-add — - * one watermark for the removal round trip (see - * {@link AddToMetadataIndexOperation}). - * * Rollback strategy: * - Re-add item to index with original metadata */ export class RemoveFromMetadataIndexOperation implements Operation { readonly name = 'RemoveFromMetadataIndex' - /** - * @param index - The metadata-index manager (JS baseline or a registered provider). - * @param id - The entity's UUID. - * @param entity - The entity/metadata being removed (required for rollback). - * @param generationFn - Resolves the commit generation for this removal, - * evaluated when the operation executes. - */ constructor( private readonly index: MetadataIndexManager, private readonly id: string, - private readonly entity: any, // Required for rollback - private readonly generationFn?: () => bigint | undefined + private readonly entity: any // Required for rollback ) {} async execute(): Promise { - // Resolve the removal generation once; reuse it for the rollback re-add. - const generation = this.generationFn?.() - // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + await this.index.removeFromIndex(this.id, this.entity) // Return rollback action return async () => { // Re-add with original metadata (skipFlush=true) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + await this.index.addToIndex(this.id, this.entity, true) } } } @@ -438,7 +269,7 @@ export class AddToGraphIndexOperation implements Operation { // Stamp this edge at the in-flight commit generation; reuse it for the // rollback so add + undo reference the same watermark. Endpoint ints // resolve HERE — after any same-batch adds have applied. - const generation = this.generationFn?.() + const generation = this.generationFn() const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts) const verbInt = await this.index.addVerb(this.verb, sourceInt, targetInt, generation) this.onVerbInt?.(verbInt) @@ -487,7 +318,7 @@ export class RemoveFromGraphIndexOperation implements Operation { // Resolve the removal generation once; reuse it for the rollback re-add. // Endpoint ints resolve HERE (after any same-batch adds applied) and are // captured for the rollback, whose re-add must use the same mappings. - const generation = this.generationFn?.() + const generation = this.generationFn() const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts) await this.index.removeVerb(this.verb.id, generation) @@ -511,20 +342,13 @@ export class BatchAddToVectorIndexOperation implements Operation { private operations: AddToVectorIndexOperation[] - /** - * @param index - The vector-index provider (JS HNSW or native). - * @param items - The vectors to index. - * @param generationFn - Resolves the commit generation shared by every item - * in the batch, evaluated when the operations execute. - */ constructor( index: VectorIndexProvider, - items: Array<{ id: string; vector: number[] }>, - generationFn?: () => bigint | undefined + items: Array<{ id: string; vector: number[] }> ) { this.name = `BatchAddToVectorIndex(${resolveVectorProviderId(index)})` this.operations = items.map( - item => new AddToVectorIndexOperation(index, item.id, item.vector, generationFn) + item => new AddToVectorIndexOperation(index, item.id, item.vector) ) } @@ -559,19 +383,12 @@ export class BatchAddToMetadataIndexOperation implements Operation { private operations: AddToMetadataIndexOperation[] - /** - * @param index - The metadata-index manager (JS baseline or a registered provider). - * @param items - The entities to index. - * @param generationFn - Resolves the commit generation shared by every item - * in the batch, evaluated when the operations execute. - */ constructor( index: MetadataIndexManager, - items: Array<{ id: string; entity: any }>, - generationFn?: () => bigint | undefined + items: Array<{ id: string; entity: any }> ) { this.operations = items.map( - item => new AddToMetadataIndexOperation(index, item.id, item.entity, generationFn) + item => new AddToMetadataIndexOperation(index, item.id, item.entity) ) } diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index c1e9f1c1..9858219b 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -12,7 +12,6 @@ import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' -import { prodLog } from '../../utils/logger.js' /** * Save noun metadata with rollback support @@ -21,30 +20,6 @@ import { prodLog } from '../../utils/logger.js' * - If metadata existed: Restore previous metadata * - If metadata was new: Delete metadata */ - -/** - * Torn-tolerant previous-state read for ROLLBACK CAPTURE: a write or delete - * landing on a TORN record (power-loss survivor) HEALS it — the incoming - * bytes replace (or remove) the unreadable ones, and the rollback target is - * the create sentinel (null). The adapter's loud floor (error + gauge) - * already fired at throw time; this narrates the heal and proceeds. Real - * storage faults still propagate. - */ -async function tornHealsToNull(read: Promise, what: string): Promise { - try { - return await read - } catch (err) { - if ((err as { code?: string }).code === 'TORN_RECORD') { - prodLog.warn( - `[StorageOperations] previous ${what} is TORN — the incoming operation ` + - `heals it; rollback target is the create sentinel` - ) - return null - } - throw err - } -} - export class SaveNounMetadataOperation implements Operation { readonly name = 'SaveNounMetadata' @@ -59,7 +34,7 @@ export class SaveNounMetadataOperation implements Operation { // Skip read for new entities — nothing to rollback to (saves 1 storage round-trip) const previousMetadata = this.isNew ? null - : await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata') + : await this.storage.getNounMetadata(this.id) // Save new metadata await this.storage.saveNounMetadata(this.id, this.metadata) @@ -100,7 +75,7 @@ export class SaveNounOperation implements Operation { // Skip read for new entities — nothing to rollback to (saves 1 storage round-trip) const previousNoun = this.isNew ? null - : await tornHealsToNull(this.storage.getNoun(this.noun.id), 'noun record') + : await this.storage.getNoun(this.noun.id) // PRESERVE stored graph state on updates. Callers stage this op with // placeholder adjacency ({connections: empty, level: 0}) because the @@ -187,11 +162,8 @@ export class DeleteNounMetadataOperation implements Operation { // Capture the FULL before-image (both legs) so the undo restores the whole // entity — a metadata-only rollback would leave the vector leg unrestored. // A null metadata read falls back to the caller's pre-delete read. - const previousNoun = await tornHealsToNull(this.storage.getNoun(this.id), 'noun record') - const previousMetadata = - (await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')) ?? - this.priorMetadata ?? - null + const previousNoun = await this.storage.getNoun(this.id) + const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null if (!previousNoun && !previousMetadata) { // Nothing to delete - no rollback needed @@ -239,7 +211,7 @@ export class SaveVerbMetadataOperation implements Operation { async execute(): Promise { // Get existing metadata (for rollback) - const previousMetadata = await tornHealsToNull(this.storage.getVerbMetadata(this.id), 'verb metadata') + const previousMetadata = await this.storage.getVerbMetadata(this.id) // Save new metadata await this.storage.saveVerbMetadata(this.id, this.metadata) @@ -275,7 +247,7 @@ export class SaveVerbOperation implements Operation { async execute(): Promise { // Get existing verb (for rollback) - const previousVerb = await tornHealsToNull(this.storage.getVerb(this.verb.id), 'verb record') + const previousVerb = await this.storage.getVerb(this.verb.id) // Save new verb await this.storage.saveVerb(this.verb) @@ -319,7 +291,7 @@ export class DeleteVerbMetadataOperation implements Operation { async execute(): Promise { // Get metadata before deletion (for rollback) - const previousMetadata = await tornHealsToNull(this.storage.getVerbMetadata(this.id), 'verb metadata') + const previousMetadata = await this.storage.getVerbMetadata(this.id) if (!previousMetadata) { // Nothing to delete - no rollback needed diff --git a/src/transaction/operations/index.ts b/src/transaction/operations/index.ts index 32a69a21..c5548e70 100644 --- a/src/transaction/operations/index.ts +++ b/src/transaction/operations/index.ts @@ -23,7 +23,6 @@ export { export { AddToVectorIndexOperation, RemoveFromVectorIndexOperation, - ReplaceInVectorIndexOperation, AddToMetadataIndexOperation, RemoveFromMetadataIndexOperation, AddToGraphIndexOperation, diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 75a63d44..6c1f0ffd 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -338,20 +338,6 @@ export interface AddParams { id?: string /** Pre-computed embedding vector (skips auto-embedding when provided) */ vector?: Vector - /** - * DEFER THE EMBEDDING (MT5, the deferred-embedding worker): the write - * acknowledges at durability — data + metadata persisted, a durable - * pending-embed marker written — and the embedding + vector-index insert - * run on the engine's single-flight background worker. HONEST SEMANTICS: - * the row is findable by id/metadata/path IMMEDIATELY; vector/semantic - * search sees it when the background embed completes (eventual vector - * index — `getIndexStatus().pendingEmbeds` counts the backlog, and - * `awaitPendingEmbeds()` is the barrier). CRASH-SAFE: markers persist - * before the ack and are recovered at the next open — a crash can DELAY - * a vector, never lose one. Refused (typed) together with `vector` — - * a supplied vector has nothing to defer. - */ - deferEmbedding?: boolean /** Multi-tenancy service identifier */ service?: string /** Type classification confidence (0-1) */ @@ -393,15 +379,6 @@ export interface AddParams { export interface UpdateParams { id: string // Entity to update data?: any // New content to re-embed - /** - * Defer the re-embedding of new `data` (see `AddParams.deferEmbedding`). - * The write acks at durability; the OLD vector keeps serving semantic - * search — stale-but-present, never absent (the flicker law) — until the - * background worker embeds the new content and swaps it in atomically. - * `data` reads return the NEW content immediately. Refused (typed) with - * an explicit `vector`. - */ - deferEmbedding?: boolean type?: NounType // Change type subtype?: string // Change subtype (set to '' or null-equivalent via dedicated unset is future work) /** @@ -1614,15 +1591,6 @@ export interface AggregationProvider { /** Serialize internal state for persistence (called during flush) */ serializeState?(): string - - /** - * Bake the committed generation into the provider's own state envelope - * before {@link serializeState} (called during flush, immediately prior). - * Lets a native-side reopen verify the envelope's honesty independently of - * the host's wrapper stamp. Optional — providers without it rely on the - * host wrapper's `sourceGeneration` alone. - */ - noteSourceGeneration?(generation: number): void } // ============= Configuration ============= @@ -2060,61 +2028,6 @@ export interface BrainyConfig { */ force?: boolean - /** - * THE ENGINE OWNS ITS FLUSH CADENCE (the persistence policy — - * SELF-ENGINE-LIFECYCLE-SPRINT, David-directed: "why do we need manual - * flushes at all?"). Under `'auto'` (the DEFAULT) the engine schedules - * single-flight background flushes itself — triggered by write count, - * elapsed time, and idle — so callers NEVER call `flush()` in a hot path - * (a production consumer's 829 per-write flushes convoyed into 45–66s - * write walls; the cadence belongs to the layer that can see dirty-node - * counts and IO pressure). `flush()` remains public as an awaitable - * durability BARRIER for the rare "must be on disk before I proceed" - * moment — calling it is never wrong, just no longer necessary. - * - * RECOVERY SEMANTICS (the documented promise): canonical records are - * durable per-write, independent of this policy — a crash between - * background flushes loses NO data. What a flush persists is DERIVED - * state (index postings, deferred HNSW nodes, counters, aggregation - * stamps); after a crash, derived state converges at the next open from - * canonical records (epoch machinery + incremental aggregation catch-up), - * paying a bounded catch-up cost proportional to the un-flushed window — - * never data loss. - * - * `'manual'` restores the pre-9.1 behavior: the engine never flushes on - * its own (except at `close()`); the caller owns the cadence. - */ - /** - * Storage-authority posture at open (10.0.0+ fleet default: `'adopt'`). - * - * `'adopt'` — a brain with NO stored authority artifact adopts LOG - * AUTHORITY at open, oracle-gated: the verification oracle replays the - * generation log against stored truth; curable divergences (pre-log - * rows, witness drift) are baseline-backfilled; the brain flips ONLY on - * a green verdict and writes the durable per-brain switch. On green, - * writes become durable-at-ack (group-committed log fsync covers every - * ack). A brain whose oracle cannot go green STAYS tree-authoritative, - * says so loudly, and records the refusal — never a silent half-state. - * - * `'defer'` — the explicit opt-out: no automatic adoption; the brain - * stays tree-authoritative until `adoptLogAuthority()` is called. The - * pre-10 behavior, documented for operators who stage their own flips. - * - * A STORED artifact always wins over this setting (checked-at-open law): - * an already-flipped brain stays flipped; an explicitly-recorded tree - * posture is honored until an operator re-runs adoption. - */ - logAuthority?: 'adopt' | 'defer' - - persistence?: { - policy?: 'auto' | 'manual' - /** Background flush after this many committed writes (default 512). */ - flushEveryWrites?: number - /** Background flush when this much time has passed since the last flush, checked at write time (default 30_000). */ - flushIntervalMs?: number - /** Background flush after the store goes quiet for this long with dirty state (default 2_000). */ - flushOnIdleMs?: number - } } // ============= Neural API Types ============= @@ -2275,79 +2188,6 @@ export interface Highlight { contentCategory?: ContentCategory } -// ============= Read barrier (waitForIndexed) ============= - -/** - * One projection leg of the read barrier (`brain.waitForIndexed(path)`) — a - * derived view of the committed data that queries are served from: - * - * - `'semantic'` — the vector index (deferred embeds land here asynchronously) - * - `'metadata'` — the field/filter index behind `find({ where })` - * - `'graph'` — the relationship adjacency index - * - `'aggregation'` — the incremental aggregate states - */ -export type IndexedProjectionPath = 'semantic' | 'metadata' | 'graph' | 'aggregation' - -/** - * Options for `brain.waitForIndexed()`. - */ -export interface WaitForIndexedOptions { - /** - * Resolve as soon as the projection has caught up to this committed - * generation (rather than the current head). Today the pending-embed set - * carries no generation stamps, so the refinement is conservative: an - * empty backlog resolves immediately (the watermark is at the head, hence - * ≥ any committed generation); a non-empty backlog waits for the full - * drain — a SUPERSET of the requested wait, never a partial one. - */ - generation?: number - - /** - * Upper bound on the wait in milliseconds. On expiry the promise REJECTS - * with {@link WaitForIndexedTimeoutError} (typed: the leg + the - * still-pending count) — never a silent partial wait. - */ - timeoutMs?: number -} - -/** - * The typed rejection of `brain.waitForIndexed(path, { timeoutMs })` on - * expiry. Carries the projection leg (`path`; `'all'` for the no-argument - * barrier) and the deferred-embed backlog size at the moment the timer fired - * (`pendingEmbeds` — the same number as - * `getIndexStatus().projections.semantic.pendingEmbeds`), so a caller can - * log an honest gauge and retry instead of guessing. A timeout means the - * projection has NOT caught up — nothing was skipped, nothing partially - * waited. - */ -export class WaitForIndexedTimeoutError extends Error { - /** The projection leg that had not caught up (`'all'` = the no-arg barrier). */ - public readonly path: IndexedProjectionPath | 'all' - - /** The expired timeout, in milliseconds. */ - public readonly timeoutMs: number - - /** Deferred embeds still pending when the timer fired — the live value of - * `getIndexStatus().projections.semantic.pendingEmbeds`. */ - public readonly pendingEmbeds: number - - constructor(path: IndexedProjectionPath | 'all', timeoutMs: number, pendingEmbeds: number) { - super( - `waitForIndexed(${path === 'all' ? '' : `'${path}'`}) timed out after ${timeoutMs}ms — ` + - `${pendingEmbeds} deferred embed${pendingEmbeds === 1 ? '' : 's'} still pending; the projection has ` + - `NOT caught up. Check getIndexStatus().projections.semantic.pendingEmbeds, then retry with a ` + - `larger timeoutMs or use awaitPendingEmbeds() for an unbounded drain.` - ) - this.name = 'WaitForIndexedTimeoutError' - this.path = path - this.timeoutMs = timeoutMs - this.pendingEmbeds = pendingEmbeds - if (Error.captureStackTrace) { - Error.captureStackTrace(this, WaitForIndexedTimeoutError) - } - } -} - // ============= Export all types ============= export * from './graphTypes.js' // Re-export NounType, VerbType, etc. \ No newline at end of file diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index d3527d77..5b5afb5e 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -129,49 +129,11 @@ export class EntityIdMapper implements EntityIdMapperProvider { // metadata channel as plain JSON; the `nextId` probe above identifies // the persisted EntityIdMapperData shape. const data = metadata as unknown as EntityIdMapperData + this.nextId = data.nextId - // TORN-STATE VALIDATION (power-loss survivor): a torn mapper file - // can carry NaN/garbage where integers belong — unvalidated, those - // NaNs reach BigInt() on the graph's int-resolution (reopen) and - // the mint path (first write after recovery) and kill both with - // RangeErrors. A torn mapper is DISCARDED with narration and the - // maps re-derive through the existing rebuild path (under log - // authority the mint-at-append records reproduce assignments - // exactly; under tree authority the metadata-index reconstruction - // rebuilds them — the same path a missing mapper file takes). - const validInt = (v: unknown): v is number => - typeof v === 'number' && Number.isSafeInteger(v) && v >= 0 - let torn = !validInt(data.nextId) - const uuidToInt = new Map() - const intToUuid = new Map() - if (!torn) { - for (const [k, v] of Object.entries(data.uuidToInt ?? {})) { - const n = Number(v) - if (!validInt(n)) { torn = true; break } - uuidToInt.set(k, n) - } - } - if (!torn) { - for (const [k, v] of Object.entries(data.intToUuid ?? {})) { - const n = Number(k) - if (!validInt(n) || typeof v !== 'string') { torn = true; break } - intToUuid.set(n, v) - } - } - if (torn) { - console.warn( - `[EntityIdMapper] persisted mapper state is TORN (non-integer ids — ` + - `power-loss survivor); discarding and re-deriving via the rebuild ` + - `path. Never a RangeError at reopen or first write.` - ) - this.nextId = 1 - this.uuidToInt = new Map() - this.intToUuid = new Map() - } else { - this.nextId = data.nextId - this.uuidToInt = uuidToInt - this.intToUuid = intToUuid - } + // Rebuild maps from serialized data + this.uuidToInt = new Map(Object.entries(data.uuidToInt).map(([k, v]) => [k, Number(v)])) + this.intToUuid = new Map(Object.entries(data.intToUuid).map(([k, v]) => [Number(k), v])) } else { // Guard: mapper file missing but entities may exist on disk. // If we start from nextId=1 with existing entities, roaring bitmap @@ -202,33 +164,14 @@ export class EntityIdMapper implements EntityIdMapperProvider { * would exceed that, throws {@link EntityIdSpaceExceeded} so the caller * loudly migrates to cor's binary mapper with `idSpace: 'u64'` * rather than silently truncating entity ids. - * - * @param generation - Brainy's commit generation current at mint time - * (contract parity with the `EntityIdMapperProvider` surface). This JS - * mapper keeps a snapshot file, not a per-record delta log, so there is - * no natural slot to store it — accepted and ignored; a native mapper - * stamps its assignment records with it. */ - getOrAssign(uuid: string, generation?: bigint): number { - void generation // Contract parity — no per-record log in the JS mapper. + getOrAssign(uuid: string): number { const existing = this.uuidToInt.get(uuid) if (existing !== undefined) { return existing } - // Assign new ID. Source guard: nextId must be a finite positive integer - // — the load path validates persisted state, but a NaN here would mint - // poison ints that reach BigInt() downstream; heal to the map-derived - // floor with narration rather than propagate. - if (!Number.isSafeInteger(this.nextId) || this.nextId < 1) { - let floor = 1 - for (const n of this.intToUuid.keys()) if (n >= floor) floor = n + 1 - console.warn( - `[EntityIdMapper] nextId was non-integer (${String(this.nextId)}) — ` + - `healed to ${floor} from the live map; torn-state survivor` - ) - this.nextId = floor - } + // Assign new ID if (this.nextId > U32_ENTITY_ID_MAX) { throw new EntityIdSpaceExceeded(this.nextId) } @@ -283,14 +226,8 @@ export class EntityIdMapper implements EntityIdMapperProvider { /** * Remove mapping for UUID - * - * @param generation - Brainy's commit generation for this removal (contract - * parity with the `EntityIdMapperProvider` surface). Accepted and ignored — - * this JS mapper removes immediately; a native mapper tombstones the - * mapping at this generation in its version chain. */ - remove(uuid: string, generation?: bigint): boolean { - void generation // Contract parity — no per-key version chain in the JS mapper. + remove(uuid: string): boolean { const intId = this.uuidToInt.get(uuid) if (intId === undefined) { return false diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 894f3fd3..f010560d 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -5,21 +5,13 @@ */ import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js' -import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError, type FieldAddress } from '../db/fieldAddressing.js' -import { splitNounMetadataRecord } from '../types/reservedFields.js' +import { SYSTEM_ENTITY_SCALARS, parseFieldAddress, UnresolvableFieldError } from '../db/fieldAddressing.js' import { ColumnStore } from '../indexes/columnStore/ColumnStore.js' import type { MetadataIndexProvider } from '../plugin.js' import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js' import { compareCodePoints } from './collation.js' import { prodLog } from './logger.js' import { getGlobalCache, UnifiedCache } from './unifiedCache.js' -import { - computeWatermarkVerdict, - makeProjectionStamp, - readStampedWatermark, - type WatermarkVerdict, - type WatermarkVerdictResult -} from './projectionWatermark.js' import { NounType, VerbType, @@ -116,15 +108,6 @@ interface FieldStats { normalizationStrategy?: 'none' | 'precision' | 'bucket' } -/** - * Storage key for the metadata projection's watermark stamp — a sidecar - * record beside the artifact (field registry + field indexes + chunked - * sparse indexes + column-store segments + id-mapper records). Written LAST - * in {@link MetadataIndexManager.flush} so stamp-after-data ordering holds - * for every byte the stamp certifies. - */ -export const METADATA_INDEX_STAMP_KEY = '__index_metadata_watermark__' - /** * Implements {@link MetadataIndexProvider}: the metadata-index surface Brainy * calls on whatever the `'metadataIndex'` provider resolves to (its own @@ -140,14 +123,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { private lastFlushTime = Date.now() private autoFlushThreshold = 10 // Start with 10 for more frequent non-blocking flushes - // --- Watermark stamp state (see utils/projectionWatermark for the law) --- - /** Generation handed in via {@link stampWatermark}, awaiting the next flush. */ - private pendingWatermark: number | null = null - /** Last watermark durably stamped by this instance or loaded at init. */ - private stampedWatermark: number | null = null - /** The three-way verdict computed at init; null until init runs. */ - private loadVerdict: WatermarkVerdictResult | null = null - // Cardinality and field statistics tracking private fieldStats = new Map() private cardinalityUpdateInterval = 100 // Update cardinality every N operations @@ -274,13 +249,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Must run first to populate fieldIndexes directory before warming cache await this.loadFieldRegistry() - // Compute the watermark verdict for the persisted artifact BEFORE any - // early return below — the verdict is recorded for every open, whether - // the workspace is empty, rebuilding, or warm. Computed and exposed - // only: today's rebuild triggers are unchanged (acting on 'catchup' — - // the incremental fold — lands with the coordinator's wiring). - await this.loadWatermarkVerdict() - // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage) await this.idMapper.init() @@ -1490,16 +1458,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @param id - Entity ID * @param entityOrMetadata - Either full entity structure or plain metadata (backward compat) * @param skipFlush - Skip automatic flush (used during batch operations) - * @param deferWrites - Batch mode: buffer postings for a later flush - * @param generation - Brainy's commit generation for this write (see the - * {@link import('../plugin.js').MetadataIndexProvider} contract). This JS - * manager keeps a single live view with no per-record delta log, so it - * has no slot to store it — the value is accepted for contract parity - * and forwarded to the shared id mapper (an injected native mapper - * stamps its assignment records with it; the JS mapper ignores it). - * The JS twin adopts full per-write stamping with the watermark train. */ - async addToIndex(id: string, entityOrMetadata: any, skipFlush: boolean = false, deferWrites: boolean = false, generation?: bigint): Promise { + async addToIndex(id: string, entityOrMetadata: any, skipFlush: boolean = false, deferWrites: boolean = false): Promise { const fields = this.extractIndexableFields(entityOrMetadata) // Sanity check for excessive indexed fields (indicates possible data issue) @@ -1547,10 +1507,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { // element, so a scalar overwrite (last-value-wins) would index only the final // element and `contains` would miss the rest. if (this.columnStore) { - // Thread the commit generation into the mint: an injected native mapper - // stamps the assignment record's delta log with the real watermark - // instead of a literal 0 (the JS mapper accepts and ignores it). - const entityIntId = this.idMapper.getOrAssign(id, generation) + const entityIntId = this.idMapper.getOrAssign(id) const fieldsMap: Record = {} for (const { field, value } of fields) { if (field === '__words__') { @@ -1642,13 +1599,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { * * @param id - Entity ID to remove * @param metadata - Optional entity or metadata structure (if not provided, requires scanning all fields - slow!) - * @param generation - Brainy's commit generation for this removal (see the - * {@link import('../plugin.js').MetadataIndexProvider} contract). Accepted - * for contract parity — this JS manager removes immediately (no tombstone - * chain) and forwards it to the shared id mapper's `remove`, where an - * injected native mapper tombstones the mapping at this generation. */ - async removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise { + async removeFromIndex(id: string, metadata?: any): Promise { if (metadata) { const fields = this.extractIndexableFields(metadata) @@ -1672,9 +1624,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Clean up ID mapper — must happen AFTER column store removal since it uses // idMapper.getInt(id). Prevents deleted IDs from persisting in the mapper // universe, which would cause ne/exists:false queries to return deleted entities. - // The generation rides along so a native mapper tombstones the mapping at - // the real commit watermark (the JS mapper ignores it). - this.idMapper.remove(id, generation) + this.idMapper.remove(id) await this.idMapper.flush() } @@ -2257,98 +2207,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @returns Promise - Entity IDs sorted by specified field * */ - /** - * Resolve the orderBy value for MANY entities in BATCHED metadata-record - * reads — the sort path's one sanctioned value source (BRAINY-PROD-LATENCY-TRIAD). - * - * THE ASYMPTOTIC LAW THIS ENFORCES: an ordered read never does per-row - * storage round-trips. The previous shape — `await getFieldValueForEntity` - * per id, each opening the VECTOR record serially — cost 62–98ms × N on a - * production filesystem brain: 3,224 rows took 199–317 SECONDS, silently. - * The metadata RECORD (smaller, cached, batch-readable) carries everything - * a sort can address: the ten system scalars top-level — EXACT values, no - * bucketing loss — and the user's bag (v2 nested or legacy flat, resolved - * through the shape-aware split). One batched read pass serves any N. - * - * The call-shape is pinned by tests (zero per-row reads, batch calls only) - * so the serial loop cannot quietly return. - * - * @param ids - Entity ids to resolve (any size; reads are chunk-batched). - * @param orderAddress - The parsed orderBy address (system or metadata scope). - * @returns id → value map; ids whose record is missing map to `undefined` - * (they sort LAST per the ordering contract — never dropped). - */ - private async resolveOrderValuesBatch( - ids: string[], - orderAddress: FieldAddress - ): Promise> { - const values = new Map() - if (ids.length === 0) return values - - // Batch door, best first: BaseStorage's getNounMetadataBatch (native - // batch or parallel reads inside), then the adapter-optional - // getMetadataBatch, then chunked-parallel single reads — NEVER serial. - const storage = this.storage as StorageAdapter & { - getNounMetadataBatch?(ids: string[]): Promise> - } - const CHUNK = 500 - const records = new Map() - for (let i = 0; i < ids.length; i += CHUNK) { - const chunk = ids.slice(i, i + CHUNK) - if (typeof storage.getNounMetadataBatch === 'function') { - const batch = await storage.getNounMetadataBatch(chunk) - for (const [id, rec] of batch) records.set(id, rec) - } else if (typeof storage.getMetadataBatch === 'function') { - const batch = await storage.getMetadataBatch(chunk) - for (const [id, rec] of batch) records.set(id, rec) - } else { - const loaded = await Promise.all( - chunk.map(async (id) => [id, await storage.getNounMetadata(id)] as const) - ) - for (const [id, rec] of loaded) if (rec) records.set(id, rec) - } - } - - for (const id of ids) { - const record = records.get(id) - if (!record) { - values.set(id, undefined) - continue - } - // Shape-aware split serves both record eras: engine scalars from the - // reserved half (EXACT timestamps — the bucketed index is never - // consulted here), user fields from the bag. - const { reserved, custom } = splitNounMetadataRecord( - record as Record - ) - if (orderAddress.scope === 'system') { - values.set( - id, - orderAddress.field === 'type' - ? reserved.noun - : (reserved as Record)[orderAddress.field] - ) - } else { - let value: unknown = custom[orderAddress.field] - if (value === undefined && orderAddress.field.includes('.')) { - // Dotted user path: traverse INSIDE the bag. - value = orderAddress.field - .split('.') - .reduce( - (o, seg) => - o && typeof o === 'object' ? (o as Record)[seg] : undefined, - custom - ) - } - values.set(id, value) - } - } - return values - } - - /** Once-per-field flag for the fallback-degradation announcement. */ - private static announcedFallbackSorts = new Set() - async getSortedIdsForFilter( filter: any, orderBy: string, @@ -2416,12 +2274,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { // ORDERING CONTRACT (cross-engine, sealed): rows missing the field are // NEVER dropped — they sort LAST in both directions — and ties break by // id ascending. The column only contains rows that HAVE the field, so - // (1) re-sort the page deterministically (value, then id) via ONE - // batched value resolution — never per-row reads — and (2) append the - // filtered rows the column omitted, id-ascending, filling any - // remaining page budget. - const pageValues = await this.resolveOrderValuesBatch(sortedUuids, orderAddress) - const page = sortedUuids.map(id => ({ id, value: pageValues.get(id) })) + // (1) re-sort the page deterministically (value, then id) with K cheap + // value reads, and (2) append the filtered rows the column omitted, + // id-ascending, filling any remaining page budget. + const page = await Promise.all( + sortedUuids.map(async id => ({ id, value: await this.getFieldValueForEntity(id, orderKey) })) + ) page.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) let result = page.map(p => p.id) @@ -2435,32 +2293,20 @@ export class MetadataIndexManager implements MetadataIndexProvider { return topK !== undefined ? result.slice(0, topK) : result } - // Fallback: no column serves this field. BOUNDED + ANNOUNCED, never - // silent (the B2 no-silent-degradation law, BRAINY-PROD-LATENCY-TRIAD): - // O(N) in row count but served by BATCHED metadata-record reads — the - // serial per-row getNoun loop that turned 3,224 rows into a 199–317s - // scan is dead, and the call-shape pin keeps it dead. + // Fallback: sparse index path (for fields not yet in column store). + // Requires a non-empty filter because it reads O(k) entity values from storage. const filteredIds = await this.getIdsForFilter(filter) if (filteredIds.length === 0) { return [] } - if ( - filteredIds.length > 500 && - !MetadataIndexManager.announcedFallbackSorts.has(orderKey) - ) { - MetadataIndexManager.announcedFallbackSorts.add(orderKey) - prodLog.warn( - `[brainy] ordered read on '${orderKey}' has no column index — served by the ` + - `batched fallback over ${filteredIds.length} rows (bounded, one batch pass; ` + - `announced once per field). A native column for this field makes it O(K).` - ) + const idValuePairs: Array<{ id: string, value: any }> = [] + for (const id of filteredIds) { + const value = await this.getFieldValueForEntity(id, orderKey) + idValuePairs.push({ id, value }) } - const fallbackValues = await this.resolveOrderValuesBatch(filteredIds, orderAddress) - const idValuePairs = filteredIds.map(id => ({ id, value: fallbackValues.get(id) })) - idValuePairs.sort((a, b) => this.compareAddressedValues(a.value, b.value, a.id, b.id, order)) const sorted = idValuePairs.map(p => p.id) @@ -2630,10 +2476,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Check if we have anything else to flush if (this.dirtyFields.size === 0) { - // Nothing dirty — but a pending watermark still stamps (the registry - // + id-mapper writes above are the only bytes this pass touched, and - // they are durable at this point). Stamp-after-data holds. - await this.writePendingStamp() return // No dirty field indexes to flush } @@ -2673,131 +2515,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { if (this.columnStore) { await this.columnStore.flush() } - - // STAMP-AFTER-DATA: the watermark stamp is the LAST write of the flush — - // every byte it certifies (field indexes, registry, id-mapper records, - // column-store segments) is durable before the stamp lands. A crash - // anywhere above leaves the artifact behind-stamped or unstamped, which - // verdicts as catchup/rescan on the next open — never a wrong adopt. - await this.writePendingStamp() } - - /** - * @description Record the committed generation this projection reflects. - * The stamp is NOT written here — it is written as the final storage write - * of the next {@link flush} (stamp-after-data ordering is a module - * guarantee, not a caller obligation). The coordinator calls this with the - * store's committed generation right before flushing. - * @param generation - The committed generation every flushed byte reflects. - */ - stampWatermark(generation: number): void { - this.pendingWatermark = generation - } - - /** - * @description The projection's current watermark: the stamp loaded at - * init (or the last stamp durably written by this instance). Null = - * unstamped (legacy artifact, first boot, or stamping never wired). - */ - watermark(): number | null { - return this.stampedWatermark - } - - /** - * @description The three-way adoption verdict computed at init — - * `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped < - * committed; the gap from {@link watermarkGap} awaits an incremental - * fold), `'rescan'` (unstamped or stamped above committed — never - * trusted). Null until init() has run. Computed and exposed only; no - * load behavior changes ride on it yet. - */ - watermarkVerdict(): WatermarkVerdict | null { - return this.loadVerdict?.verdict ?? null - } - - /** - * @description The catch-up window `(from, to]` when the init verdict was - * `'catchup'`; null otherwise. - */ - watermarkGap(): { from: number; to: number } | null { - return this.loadVerdict?.gap ?? null - } - - /** - * @description Write the pending watermark stamp as a sidecar record — - * always called AFTER the data it certifies is durable. A stamp-write - * failure is fail-safe (the artifact stays unstamped/behind → rescan or - * catchup on next open, never a wrong adopt) but is said out loud and the - * pending stamp is retained for the next flush. - */ - private async writePendingStamp(): Promise { - if (this.pendingWatermark === null) return - const watermark = this.pendingWatermark - try { - await this.storage.saveMetadata(METADATA_INDEX_STAMP_KEY, { - noun: 'IndexWatermark', - ...makeProjectionStamp(watermark) - }) - this.stampedWatermark = watermark - this.pendingWatermark = null - } catch (error) { - prodLog.error( - `[MetadataIndex] failed to write watermark stamp (generation ${watermark}) — ` + - `artifact stays behind-stamped (safe: verdicts catchup/rescan, never wrong-adopt); ` + - `retrying on next flush:`, - error - ) - } - } - - /** - * @description Read the artifact's stamp and compute the three-way verdict - * against the store's committed generation. Unstamped state on a stamped - * store verdicts `'rescan'` LOUDLY — never a silent adopt. - * - * MIGRATION COST: existing pre-stamp brains verdict `'rescan'` exactly - * once (this open re-derives from source as it already does today); the - * next flush stamps them, and every later open adopts. - */ - private async loadWatermarkVerdict(): Promise { - const committed = this.storage.committedGeneration?.() ?? null - let stamped: number | null = null - try { - const record = await this.storage.getMetadata(METADATA_INDEX_STAMP_KEY) - stamped = readStampedWatermark(record) - } catch { - // An unreadable stamp is unstamped — the fail-safe direction. - stamped = null - } - const result = computeWatermarkVerdict(stamped, committed) - this.loadVerdict = result - this.stampedWatermark = stamped - - if (result.verdict === 'rescan') { - const artifactPresent = this.fieldIndexes.size > 0 || stamped !== null - if (artifactPresent) { - prodLog.warn( - `[MetadataIndex] watermark verdict: RESCAN — persisted index is ` + - (stamped === null - ? 'unstamped (legacy pre-stamp artifact, or a crash between data and stamp)' - : `stamped at generation ${stamped}, ABOVE the store's committed generation ${committed}`) + - ` — never adopting unverifiable state` - ) - } else { - prodLog.debug( - '[MetadataIndex] watermark verdict: rescan (no persisted artifact — first boot)' - ) - } - } else if (result.verdict === 'catchup') { - prodLog.info( - `[MetadataIndex] watermark verdict: catchup — index stamped at generation ` + - `${stamped}, store committed at ${committed}; the (${stamped}, ${committed}] ` + - `window awaits an incremental fold (verdict exposed; the fold lands with the ` + - `coordinator's wiring)` - ) - } - } - + /** * Yield control back to the Node.js event loop * Prevents blocking during long-running operations diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index b8036746..359413d7 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -540,22 +540,6 @@ function rejectForgedSystemKeys(metadata: Record | undefined, s export function validateAddParams(params: AddParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'add()') - // MT5 deferred embedding: an explicit vector has nothing to defer, and a - // deferral without data has nothing to embed — both are caller bugs that - // must refuse with the fix, never be silently reinterpreted. - if ((params as AddParams & { deferEmbedding?: boolean }).deferEmbedding === true) { - if (params.vector) { - throw new Error( - `add(): deferEmbedding cannot be combined with an explicit 'vector' — ` + - `the vector is already computed; drop one of the two.` - ) - } - if (!params.data) { - throw new Error( - `add(): deferEmbedding requires 'data' (the content the background worker will embed).` - ) - } - } // Universal truth: must have data or vector if (!params.data && !params.vector) { throw new Error( @@ -597,19 +581,6 @@ export function validateAddParams(params: AddParams): void { */ export function validateUpdateParams(params: UpdateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'update()') - if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { - if (params.vector) { - throw new Error( - `update(): deferEmbedding cannot be combined with an explicit 'vector' — ` + - `the vector is already computed; drop one of the two.` - ) - } - if (!params.data) { - throw new Error( - `update(): deferEmbedding requires new 'data' — without a data change there is nothing to re-embed.` - ) - } - } // Universal truth: must have an ID if (!params.id) { throw new Error('id is required for update') diff --git a/src/utils/projectionWatermark.ts b/src/utils/projectionWatermark.ts deleted file mode 100644 index 1bd77aeb..00000000 --- a/src/utils/projectionWatermark.ts +++ /dev/null @@ -1,150 +0,0 @@ -/** - * @module utils/projectionWatermark - * @description The watermark-stamp contract shared by Brainy's persisted TS - * projections (metadata index, JS HNSW vector index, graph adjacency index). - * - * THE LAW: every persisted projection artifact carries a stamp asserting - * "this state reflects every committed generation ≤ watermark and nothing - * above it, atomically". STAMP-AFTER-DATA: the stamp is written only after - * every byte it certifies is durable — a crash between data and stamp leaves - * the artifact unstamped, which verdicts as a rescan, never a wrong adopt. - * - * At load, each owner computes a three-way verdict against the store's - * committed generation — the same rule and verdict names the aggregation - * machinery ships (see `AggregationIndex.stateAdoptionVerdict`): - * - * - `'adopt'` — stamped == committed (clean reopen, zero work), or the - * store exposes no committed generation at all (pre-stamp - * stores keep their pre-stamp behavior). - * - `'catchup'` — stamped < committed (an unclean exit after later writes, - * or a long-lived writer whose last stamp predates recent - * commits). The artifact is exact AS OF its stamp, so the - * missing window `(stamped, committed]` can be folded - * incrementally — at-least-once idempotent, bounded by - * writes since the stamp, never by store size. - * - `'rescan'` — unstamped (a legacy pre-stamp artifact, or a crash between - * data and stamp) or stamped ABOVE committed (e.g. a log - * truncation on a copied store pulled the watermark back): - * the state over-claims unverifiably — one exact rescan, - * said out loud, never a silent adopt. - * - * MIGRATION COST (stated once, honored by every owner): existing pre-stamp - * brains verdict `'rescan'` exactly once — they re-derive from source on - * that open, the next flush stamps them, and every later open adopts. - * - * The verdict is COMPUTED AND EXPOSED by each owner; acting on `'catchup'` - * (the incremental fold) lands with the owner's coordinator wiring. - */ - -/** The three-way load verdict for a persisted projection artifact. */ -export type WatermarkVerdict = 'adopt' | 'catchup' | 'rescan' - -/** - * Format version written into every projection stamp. Bump when the stamp - * record's shape changes incompatibly; readers treat an unknown version as - * unstamped (→ rescan) rather than guessing. - */ -export const PROJECTION_STAMP_FORMAT_VERSION = 1 - -/** - * @description The stamp record a projection writes into (or beside) its - * persisted artifact, always AFTER the data it certifies is durable. - */ -export interface ProjectionStamp { - /** The committed generation this artifact reflects, exactly and entirely. */ - watermark: number - /** {@link PROJECTION_STAMP_FORMAT_VERSION} at write time. */ - formatVersion: number - /** Wall-clock ms at stamp write — diagnostic only, never load-bearing. */ - stampedAt: number - /** - * Identity of the vector space for vector-bearing artifacts (the HNSW - * index). The JS index has no reachable embedding-model id in its module, - * so dimensions are the only identity it can honestly assert. - */ - modelIdentity?: { embedModelId?: string; dimensions: number | null } -} - -/** The verdict plus everything the owner needs to report or act on it. */ -export interface WatermarkVerdictResult { - verdict: WatermarkVerdict - /** Watermark read from the artifact's stamp; null = unstamped. */ - stamped: number | null - /** The store's committed generation at load; null = no capability. */ - committed: number | null - /** The catch-up window `(from, to]` when verdict is `'catchup'`, else null. */ - gap: { from: number; to: number } | null -} - -/** - * @description Build a stamp record for a projection artifact. - * @param watermark - The committed generation the artifact reflects. - * @param modelIdentity - Vector-space identity for vector-bearing artifacts. - * @returns The stamp record to persist (stamp-after-data). - */ -export function makeProjectionStamp( - watermark: number, - modelIdentity?: ProjectionStamp['modelIdentity'] -): ProjectionStamp { - const stamp: ProjectionStamp = { - watermark, - formatVersion: PROJECTION_STAMP_FORMAT_VERSION, - stampedAt: Date.now() - } - if (modelIdentity !== undefined) stamp.modelIdentity = modelIdentity - return stamp -} - -/** - * @description Read the stamped watermark out of a persisted record, treating - * anything malformed (missing, wrong type, non-finite, negative, or an - * unknown format version) as unstamped — the fail-safe direction is rescan, - * never a guessed adopt. - * @param record - The raw persisted record (or null/undefined). - * @returns The stamped watermark, or null if effectively unstamped. - */ -export function readStampedWatermark(record: unknown): number | null { - if (record === null || typeof record !== 'object') return null - const rec = record as Record - const version = rec.formatVersion - if (typeof version !== 'number' || version > PROJECTION_STAMP_FORMAT_VERSION) { - return null - } - const raw = rec.watermark - if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) return null - return raw -} - -/** - * @description The three-way adoption verdict — the single decision rule - * every stamped projection shares (mirrors the aggregation machinery's - * `stateAdoptionVerdict` exactly: same names, same directions). - * @param stamped - Watermark read from the artifact ({@link readStampedWatermark}). - * @param committed - The store's committed generation (null = no capability). - * @returns The verdict with the stamped/committed pair and the catch-up gap. - */ -export function computeWatermarkVerdict( - stamped: number | null, - committed: number | null -): WatermarkVerdictResult { - // No committed-generation capability: hash/shape checks are the only - // adoption gate, exactly the pre-stamp behavior. Never fail a store that - // cannot express the question. - if (committed === null) { - return { verdict: 'adopt', stamped, committed, gap: null } - } - if (stamped === committed) { - return { verdict: 'adopt', stamped, committed, gap: null } - } - if (stamped !== null && stamped < committed) { - return { - verdict: 'catchup', - stamped, - committed, - gap: { from: stamped, to: committed } - } - } - // Unstamped, or stamped above committed: unverifiable — rescan, loudly - // (the caller owns the loud log so it can name its projection). - return { verdict: 'rescan', stamped, committed, gap: null } -} diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index ed272109..00bddefb 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -694,12 +694,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { await this.brain.update({ id: existingId, data: embeddingData, - // MT5: the caller's write acks at durability; the re-embed (a neural - // net — it dominated the measured 5.6s p50 per file write) runs on - // the background worker and swaps in atomically. Content is readable - // and metadata-findable immediately; semantic search converges when - // the embed lands (eventual vector index, the documented contract). - deferEmbedding: true, metadata }) @@ -735,9 +729,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { data: embeddingData, // Always provide string for embeddings type: this.getFileNounType(mimeType), subtype: 'vfs-file', // Standard subtype for VFS file entities (7.30+) - // MT5: ack at durability; embedding backgrounds (see the overwrite - // branch note above). - deferEmbedding: true, metadata }) @@ -1126,9 +1117,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { data: path, // Directory path as string content type: NounType.Collection, subtype: 'vfs-directory', // Standard subtype for VFS directory entities (7.30+) - // MT5: a directory creation on a write path must not wait on the - // embedder either — same ack-at-durability contract as file writes. - deferEmbedding: true, metadata }) diff --git a/tests/conformance/golden-log-fold.test.ts b/tests/conformance/golden-log-fold.test.ts deleted file mode 100644 index c480bee5..00000000 --- a/tests/conformance/golden-log-fold.test.ts +++ /dev/null @@ -1,170 +0,0 @@ -/** - * @module tests/conformance/golden-log-fold - * @description THE GOLDEN-LOG FOLD-CONFORMANCE ORACLE (brainy leg). - * - * One deterministic v2 log — fixed ids, ints, timestamps, vectors — whose - * ENCODED BYTES and whose FOLDED STATE are both pinned by content hash. - * The second (native) reader implementation consumes the identical fixture - * (tests/fixtures/golden-log-v2.bin, written and verified here) and must - * produce the identical fold digest; the pair is normative on disagreement. - * - * What the pins catch, loudly: - * - Any byte drift in the encoder (envelope, msgpack layout, seals, CRC). - * - Any semantic drift in the fold (tombstone masking, vector landing, - * sameAsGeneration resolution, last-writer-wins ordering). - * - Any divergence between the two implementations, before the cut. - * - * The pinned hashes change ONLY with a deliberate, versioned format or - * fold-law change — never silently. Updating them requires updating the - * fixture AND the native side in the same train. - */ -import { describe, it, expect } from 'vitest' -import { createHash } from 'node:crypto' -import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' -import { join, dirname } from 'node:path' -import { - encodeFactV2, - encodeSegmentHeaderV2, - sealGroup, - decodeGroupV2, - SEGMENT_HEADER_BYTES, - type CommitFactV2, - type LogRecord -} from '../../src/db/factLogFormat.js' -import { recordDigest } from '../../src/db/logAuthority.js' - -const FIXTURE = join(__dirname, '../fixtures/golden-log-v2.bin') - -const sha256 = (b: Uint8Array): string => createHash('sha256').update(b).digest('hex') - -// Fixed identities — never regenerate. -const BRAIN = '00000000-0000-4000-8000-00000000b1a1' -const A = '00000000-0000-4000-8000-0000000000a1' -const B = '00000000-0000-4000-8000-0000000000b2' -const C = '00000000-0000-4000-8000-0000000000c3' -const V = '00000000-0000-4000-8000-0000000000d4' - -const vec = (seed: number): number[] => [seed + 0.25, seed + 0.5, seed + 0.75] - -/** The golden fact sequence — every fold-relevant behavior in nine facts. */ -function goldenFacts(): CommitFactV2[] { - const f = (generation: number, records: LogRecord[]): CommitFactV2 => ({ - generation, - timestamp: 1_700_000_000_000 + generation, - records - }) - return [ - f(1, [{ type: 'log.genesis', idSpaceWidth: 64, brainId: BRAIN, createdAt: 1_700_000_000_000 }]), - f(2, [{ type: 'noun.afterImage', id: A, entityInt: 1n, metadata: { name: 'alpha', rank: 1 }, vectorLeg: vec(1) }]), - f(3, [ - { type: 'noun.afterImage', id: B, entityInt: 2n, metadata: { name: 'beta' }, vectorLeg: null }, - { type: 'embed.pending', id: B, enqueuedAt: 1_700_000_000_003 } - ]), - // A metadata-only update: the vector rides by reference to generation 2. - f(4, [{ type: 'noun.afterImage', id: A, entityInt: 1n, metadata: { name: 'alpha', rank: 2 }, vectorLeg: { sameAsGeneration: 2 } }]), - // B's deferred vector lands. - f(5, [{ type: 'embed.landed', id: B, vector: vec(9) }]), - // A relationship. - f(6, [{ type: 'verb.afterImage', id: V, verbInt: 3n, metadata: { w: 0.5 }, vectorLeg: null, verb: 'relatedTo', sourceId: A, sourceInt: 1n, targetId: B, targetInt: 2n }]), - // C exists briefly… - f(7, [{ type: 'noun.afterImage', id: C, entityInt: 4n, metadata: { name: 'gamma' }, vectorLeg: vec(7) }]), - // …and is tombstoned (masking must hold in the fold). - f(8, [{ type: 'noun.tombstone', id: C }]), - // An all-deduped batch: a real generation with zero records. - f(9, []) - ] -} - -/** Build the golden segment: v2 header + sealed frame group. */ -function goldenSegment(): Uint8Array { - // Single-hop law: generation 2 carried A's inline vector (5 carries B's - // via embed.landed); the ref in generation 4 must verify against it. - const inline = new Set([2, 5, 7]) - const frames = goldenFacts().map((fact) => encodeFactV2(fact, { inlineVectorGenerations: inline })) - const sealed = sealGroup(frames, 4096) - const out = new Uint8Array(SEGMENT_HEADER_BYTES + sealed.length) - out.set(encodeSegmentHeaderV2(1, 4096), 0) - out.set(sealed, SEGMENT_HEADER_BYTES) - return out -} - -/** - * THE FOLD LAW (shared with the native implementation, normative): - * fold facts in generation order → per-id latest state with tombstone - * masking; embed.landed applies the vector to the id's current state; - * {sameAsGeneration: N} resolves to the inline vector the log carried at N; - * verbs fold like nouns under their own ids. Digest = recordDigest (key- - * sorted JSON sha256) of the id-sorted state map. - */ -function foldGoldenLog(bytes: Uint8Array): string { - const group = decodeGroupV2(bytes.slice(SEGMENT_HEADER_BYTES)) - const state = new Map>() - const inlineVectorAt = new Map() - for (const fact of group.facts) { - for (const rec of fact.records) { - if (rec.type === 'noun.afterImage' || rec.type === 'verb.afterImage') { - let vector: number[] | null = null - if (Array.isArray(rec.vectorLeg)) { - vector = rec.vectorLeg - inlineVectorAt.set(fact.generation, vector) - } else if (rec.vectorLeg && typeof rec.vectorLeg === 'object' && 'sameAsGeneration' in rec.vectorLeg) { - vector = inlineVectorAt.get((rec.vectorLeg as { sameAsGeneration: number }).sameAsGeneration) ?? null - } - state.set(rec.id, { - kind: rec.type === 'noun.afterImage' ? 'noun' : 'verb', - int: (rec.type === 'noun.afterImage' - ? (rec as { entityInt: bigint }).entityInt - : (rec as { verbInt: bigint }).verbInt - ).toString(), - metadata: rec.metadata, - vector, - generation: fact.generation - }) - } else if (rec.type === 'noun.tombstone' || rec.type === 'verb.tombstone') { - state.delete(rec.id) - } else if (rec.type === 'embed.landed') { - const cur = state.get(rec.id) - if (cur) state.set(rec.id, { ...cur, vector: rec.vector, generation: fact.generation }) - inlineVectorAt.set(fact.generation, rec.vector) - } - // embed.pending / genesis / blob / projection notes carry no fold state here. - } - } - const sorted = [...state.entries()].sort(([x], [y]) => (x < y ? -1 : 1)) - return recordDigest(sorted) -} - -// ── THE PINS ──────────────────────────────────────────────────────────────── -// Byte-exact encode + semantics-exact fold. These literals are the contract. -const GOLDEN_BYTES_SHA256 = 'f898ed29f6f7d41135c6c85eb07725348b20cf8efec5f050ff50ad6d54a09dad' -const GOLDEN_FOLD_DIGEST = 'fad1b1d9865d6c9c84493c5481599ebd39b7ecf4cd203af4c435dfea7cd78ed4' - -describe('golden-log fold conformance (brainy leg)', () => { - it('the encoder reproduces the golden bytes exactly', () => { - const seg = goldenSegment() - expect(seg.length % 4096, 'sealed to the sector boundary (header excluded)').toBe(SEGMENT_HEADER_BYTES % 4096) - expect(sha256(seg)).toBe(GOLDEN_BYTES_SHA256) - }) - - it('the fixture on disk is byte-identical (the shared artifact both readers consume)', () => { - const seg = goldenSegment() - if (!existsSync(FIXTURE)) { - mkdirSync(dirname(FIXTURE), { recursive: true }) - writeFileSync(FIXTURE, seg) - } - const onDisk = new Uint8Array(readFileSync(FIXTURE)) - expect(sha256(onDisk), 'fixture bytes match the encoder').toBe(GOLDEN_BYTES_SHA256) - }) - - it('folding the golden log yields the pinned state digest', () => { - expect(foldGoldenLog(goldenSegment())).toBe(GOLDEN_FOLD_DIGEST) - }) - - it('fold semantics spot-checks (human-readable guardrails beside the hash)', () => { - const group = decodeGroupV2(goldenSegment().slice(SEGMENT_HEADER_BYTES)) - expect(group.facts.length, 'nine facts, pads invisible').toBe(9) - const gens = group.facts.map((f) => f.generation) - expect(gens).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9]) - expect(group.facts[8].records).toEqual([]) - }) -}) diff --git a/tests/fixtures/golden-log-v2.bin b/tests/fixtures/golden-log-v2.bin deleted file mode 100644 index c1e4cabd..00000000 Binary files a/tests/fixtures/golden-log-v2.bin and /dev/null differ diff --git a/tests/helpers/durabilityKillMatrix.ts b/tests/helpers/durabilityKillMatrix.ts deleted file mode 100644 index c4af622a..00000000 --- a/tests/helpers/durabilityKillMatrix.ts +++ /dev/null @@ -1,210 +0,0 @@ -/** - * @module tests/helpers/durabilityKillMatrix - * @description Shared machinery for the durability kill-matrix suite - * (tests/integration/durability-kill-matrix.test.ts): open filesystem brains - * with fully explicit durability (no background cadence, no embedder), arm - * the generation store's test-only commit fault injector at one exact phase, - * abandon a "crashed" brain the way a dead process would (its RAM is gone, - * nothing flushes, nothing closes), and read the fact log / on-disk state the - * recovery assertions pin. - * - * The crash model is PROCESS DEATH: in-memory state is lost, file bytes the - * process already handed to the OS survive. One helper additionally models - * POWER LOSS for a chosen entity by removing its canonical files — legal, - * because single-op canonical writes are tmp+rename WITHOUT fsync, and a - * rename that was never fsynced may surface as "no directory entry" after - * power loss. - */ -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/brainy.js' -import type { CommitFaultPhase, GenerationStore } from '../../src/db/generationStore.js' - -/** The error a throwing fault injector uses to simulate a process crash. */ -export class SimulatedCrash extends Error { - constructor(phase: CommitFaultPhase) { - super(`simulated process crash at ${phase}`) - this.name = 'SimulatedCrash' - } -} - -/** Deterministic 384-dim vector so no test ever invokes the embedder. */ -export function vec(seed: number): number[] { - return Array.from({ length: 384 }, (_, i) => ((seed * 31 + i * 7) % 100) / 100) -} - -/** - * Map a readable label to a deterministic UUID-shaped id (entity ids must be - * UUIDs — the sharded storage layout derives the shard from the UUID hex). - */ -export function uid(label: string): string { - let h1 = 0x811c9dc5 - for (let i = 0; i < label.length; i++) { - h1 = Math.imul(h1 ^ label.charCodeAt(i), 0x01000193) >>> 0 - } - let h2 = 0xdeadbeef - for (let i = label.length - 1; i >= 0; i--) { - h2 = Math.imul(h2 ^ label.charCodeAt(i), 0x85ebca6b) >>> 0 - } - const hex = h1.toString(16).padStart(8, '0') + h2.toString(16).padStart(8, '0') - return `00000000-0000-4000-8000-${hex.slice(0, 12)}` -} - -/** Create a fresh temp directory for one brain's storage root. */ -export function makeTempDir(): string { - return fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-kill-matrix-')) -} - -/** - * Open a writer brain over `dir` with every implicit durability knob off: - * persistence policy 'manual' (the engine never flushes on its own, so every - * durable transition in a test is an explicit `flush()`/commit), deterministic - * embeddings (tests always pass explicit vectors anyway), silent logs — and - * `logAuthority: 'defer'` (the explicit opt-out of the 10.0.0 adopt-at-open - * fleet default), so the durability POSTURE is explicit per row too: rows - * pinning deferred/tree recovery semantics get exactly that, and at-ack rows - * engage log authority via `flipToAtAck`. The fleet default's open-time - * adoption would inject a baseline-backfill generation into every floor - * computation and pre-flip every row. - */ -export async function openBrain( - dir: string, - opts?: { logAuthority?: 'adopt' | 'defer' } -): Promise { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - const brain = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - persistence: { policy: 'manual' }, - logAuthority: opts?.logAuthority ?? 'defer' - }) - await brain.init() - return brain -} - -/** Typed access to the brain's private generation store (test injection point). */ -export function storeOf(brain: Brainy): GenerationStore { - return (brain as unknown as { generationStore: GenerationStore }).generationStore -} - -/** - * Arm the commit fault injector to simulate a process crash at EXACTLY one - * phase (all other phases pass through untouched). Returns the list of phases - * observed before (and including) the trip, so a test can assert the fault - * actually fired where intended. - */ -export function armCrash(brain: Brainy, phase: CommitFaultPhase): { fired: CommitFaultPhase[] } { - const fired: CommitFaultPhase[] = [] - storeOf(brain).setCommitFaultInjector((p) => { - fired.push(p) - if (p === phase) { - throw new SimulatedCrash(p) - } - }) - return { fired } -} - -/** - * Abandon a crashed brain the way process death would: its buffered RAM state - * is discarded and no background machinery may ever touch the storage - * directory again (a dead process cannot flush). The fault injector stays - * installed so any in-flight commit path still "crashes". Serialized behind - * the store's commit mutex so an interleaved background flush cannot be - * severed mid-section. - * - * NEVER calls close() — graceful close is exactly what a crash denies. - */ -export async function abandonAsCrashed(brain: Brainy): Promise { - const store = storeOf(brain) as unknown as { - withMutex(fn: () => Promise): Promise - clearPendingFlushTimer(): void - pendingGens: number[] - pendingBuffer: Map - } - await store.withMutex(async () => { - store.clearPendingFlushTimer() - store.pendingGens = [] - store.pendingBuffer.clear() - }) -} - -/** - * Every generation present in the brain's fact log, ascending — the suite's - * "what does the log claim is committed" probe. Empty when no fact log exists. - * A scan abort (gap detection) propagates — callers that PIN gap behavior - * catch it themselves. - */ -export async function factGenerations(brain: Brainy): Promise { - const scan = brain.scanFacts({ fromGeneration: 1 }) - if (!scan) return [] - const gens: number[] = [] - for await (const batch of scan.batches()) { - for (const fact of batch.facts) gens.push(fact.generation) - } - return gens.sort((a, b) => a - b) -} - -/** An ENOSPC-shaped error, matching what a full disk surfaces from node:fs. */ -export function enospcError(): NodeJS.ErrnoException { - const err = new Error("ENOSPC: no space left on device, write") as NodeJS.ErrnoException - err.code = 'ENOSPC' - err.errno = -28 - err.syscall = 'write' - return err -} - -/** - * Make the storage adapter's next raw-byte append (the fact-log append path) - * fail once with ENOSPC, then restore the original — "the disk filled for one - * append, then space was freed". Returns a probe telling how many appends - * were failed. - */ -export function failNextAppendWithEnospc(brain: Brainy): { failed: () => number } { - const storage = (brain as unknown as { - storage: { appendRawBytes(p: string, b: Uint8Array): Promise } - }).storage - const original = storage.appendRawBytes.bind(storage) - let failures = 0 - storage.appendRawBytes = async (p: string, b: Uint8Array): Promise => { - storage.appendRawBytes = original - failures++ - throw enospcError() - } - return { failed: () => failures } -} - -/** - * POWER-LOSS MODEL for one entity: remove its canonical noun files from the - * storage root. Legal disk state — a single-op write's canonical bytes are - * tmp+rename WITHOUT fsync (only `transact()` runs the write barrier), and an - * un-fsynced rename may resolve to "no directory entry" after power loss. - * Throws when nothing was removed (the caller's premise would be wrong). - */ -export function dropCanonicalNoun(dir: string, id: string): void { - const removed: string[] = [] - const walk = (p: string): void => { - for (const entry of fs.readdirSync(p, { withFileTypes: true })) { - const full = path.join(p, entry.name) - if (entry.isDirectory()) { - if (entry.name === id) { - fs.rmSync(full, { recursive: true, force: true }) - removed.push(full) - } else { - walk(full) - } - } - } - } - const nounsRoot = path.join(dir, 'entities', 'nouns') - if (fs.existsSync(nounsRoot)) walk(nounsRoot) - if (removed.length === 0) { - throw new Error(`power-loss model: no canonical files found for noun ${id} under ${nounsRoot}`) - } -} - -/** True when the staged record-set directory for `gen` exists on disk. */ -export function generationDirExists(dir: string, gen: number): boolean { - return fs.existsSync(path.join(dir, '_generations', String(gen))) -} diff --git a/tests/integration/adopt-drift-cure.test.ts b/tests/integration/adopt-drift-cure.test.ts deleted file mode 100644 index fb154574..00000000 --- a/tests/integration/adopt-drift-cure.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -/** - * @module tests/integration/adopt-drift-cure - * @description THE DRIFT-CURING BACKFILL — the actual completion of the - * default-flip ruling: existing brains whose canonical wrappers carry - * pre-hydration-law drift (denormalized fields disagreeing with their own - * metadata leg — the real depot-brain shape, uuid-v7 rows from the 9.0 era) - * must ADOPT AUTOMATICALLY: the backfill rewrites canonical in the law - * shape (metadata leg = the authority; floats preserved), the oracle then - * verifies the rewrite before flipping. Same safety, zero operator chores. - * Log-ahead divergences still refuse as before. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -type RawBox = { - storage: { - readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }> - writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise - } -} - -describe('adoption cures hydration-law drift automatically', () => { - it('a drifted wrapper (stale denormalized fields) adopts green with floats preserved', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-drift-cure-')) - dirs.push(dir) - const brain = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - logAuthority: 'defer' - }) - await brain.init() - brains.push(brain) - const id = await brain.add({ - data: 'early-era row with drift', - type: NounType.Document, - metadata: { k: 1 } - }) - await brain.flush() - const before = await brain.get(id, { includeVectors: true }) - const floats = [...(before!.vector as number[])] - expect(floats.length).toBeGreaterThan(0) - - // Manufacture the depot shape: the stored wrapper's denormalized fields - // disagree with the metadata leg (pre-hydration-law drift) — an as-is - // identity re-commit preserves this forever; the law-shape rewrite cures it. - const storage = (brain as unknown as RawBox).storage - const raw = await storage.readNounRaw(id) - const wrapper = raw.vector as Record - await storage.writeNounRaw(id, { - metadata: raw.metadata, - vector: { - ...wrapper, - noun: 'thing', // stale denormalized type (metadata leg says document) - legacyField: 'pre-law residue', - createdAt: '1999-01-01T00:00:00.000Z' - } - }) - // Confirm the drift is oracle-visible before the cure. - expect((await brain.verifyLogAuthority()).verdict, 'drift detected').toBe('red') - - // THE PIN: adoption cures it without any operator step. - const report = await brain.adoptLogAuthority() - expect(report.verdict).toBe('green') - expect(brain.logAuthority().authority).toBe('log') - - // Nothing degraded: floats byte-identical, metadata intact, row serves. - const after = await brain.get(id, { includeVectors: true }) - expect(after!.vector as number[], 'floats preserved through the cure').toEqual(floats) - expect((after!.metadata as { k: number }).k).toBe(1) - expect((await brain.find({ where: { k: 1 }, limit: 5 })).map((r) => r.id)).toContain(id) - }, 120000) - - it('log-ahead divergences still refuse — the backfill never papers over a log the witness denies', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-logahead-')) - dirs.push(dir) - const brain = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - logAuthority: 'defer' - }) - await brain.init() - brains.push(brain) - const id = await brain.add({ data: 'row', type: NounType.Document, metadata: { k: 1 } }) - await brain.flush() - - // Log-ahead shape: canonical loses the record while the log still - // claims it live (log-live-canonical-absent — NOT curable by baseline). - const storage = (brain as unknown as RawBox).storage - await storage.writeNounRaw(id, { metadata: null, vector: null }) - - await expect(brain.adoptLogAuthority()).rejects.toThrow( - /log-ahead|witness denies|log claims/i - ) - expect(brain.logAuthority().authority).toBe('tree') - }, 120000) -}) diff --git a/tests/integration/aggregation-lifecycle-catchup.test.ts b/tests/integration/aggregation-lifecycle-catchup.test.ts deleted file mode 100644 index d4f9e6bf..00000000 --- a/tests/integration/aggregation-lifecycle-catchup.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -/** - * @module tests/integration/aggregation-lifecycle-catchup - * @description THE AGGREGATION LIFECYCLE PINS (SELF-ENGINE-LIFECYCLE-SPRINT / - * BRAINY-PROD-LATENCY-TRIAD asks (a)+(b)). The production disease: the - * aggregation stamp persisted ONLY at close(), so a long-lived writer that - * flushes but never closes left its stamp behind after every write window — - * and the exact-match adoption rule then forced a WHOLE-STORE backfill walk - * (per-entity work, measured >60s and door-starving on a 9k-row production - * brain) on the first stats call after any unclean exit. - * - * The cures pinned here: - * (a) `brain.flush()` persists aggregation state, stamped at the committed - * generation — the stamp tracks every flush, not just close(). - * (b) BEHIND-stamp state is ADOPTED and reconciled INCREMENTALLY over its - * exact missing window (fact-log affected ids + time-travel before/after - * reads) — the full walk never runs for an unclean exit. Pinned by call - * shape (the walk spy), not by latency. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const AGG = { - name: 'by_subtype', - source: { type: NounType.Document }, - groupBy: ['system.subtype'] as string[], - metrics: { count: { op: 'count' as const } } -} - -const dirs: string[] = [] -const brains: Brainy[] = [] - -async function open(dir: string): Promise { - const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await b.init() - brains.push(b) - return b -} - -function countFor(results: Array<{ groupKey: Record; metrics: Record }>, subtype: string): number { - const row = results.find(r => r.groupKey['system.subtype'] === subtype) - return row ? Number(row.metrics.count) : 0 -} - -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -describe('aggregation lifecycle — flush stamps, behind-stamp catches up incrementally', () => { - it('(a) brain.flush() persists aggregation state stamped at the committed generation', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-agg-flush-')) - dirs.push(dir) - const brain = await open(dir) - brain.defineAggregate(AGG) - await brain.add({ data: 'a', type: NounType.Document, subtype: 'invoice', metadata: {} }) - await brain.add({ data: 'b', type: NounType.Document, subtype: 'invoice', metadata: {} }) - await brain.queryAggregate(AGG.name) // settle backfill-on-define - - await brain.flush() - - const internals = brain as unknown as { - storage: { - getMetadata(k: string): Promise<{ sourceGeneration?: number } | null> - committedGeneration?(): number - } - } - const persisted = await internals.storage.getMetadata('__aggregation_state_by_subtype__') - expect(persisted, 'state persisted by flush(), not only close()').toBeTruthy() - expect( - persisted!.sourceGeneration, - 'stamp equals the committed generation at flush time' - ).toBe(internals.storage.committedGeneration?.()) - }) - - it('(b) an unclean exit reconciles incrementally — exact counts, ZERO full-store walks', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-agg-catchup-')) - dirs.push(dir) - - // Session 1: define + write + flush (stamps at G), then MORE writes of - // every kind (add / update-that-moves-groups / delete) and a clean close - // — but we then REWIND the persisted aggregation artifact to its at-G - // bytes, which is byte-for-byte the unclean-exit state: stamp G, store - // committed at G+k. - let brain = await open(dir) - brain.defineAggregate(AGG) - await brain.add({ data: 'a', type: NounType.Document, subtype: 'invoice', metadata: {} }) - await brain.add({ data: 'b', type: NounType.Document, subtype: 'invoice', metadata: {} }) - const moving = await brain.add({ data: 'c', type: NounType.Document, subtype: 'draft', metadata: {} }) - const doomed = await brain.add({ data: 'd', type: NounType.Document, subtype: 'draft', metadata: {} }) - await brain.queryAggregate(AGG.name) - await brain.flush() - - const internals = brain as unknown as { - storage: { - getMetadata(k: string): Promise | null> - saveMetadata(k: string, v: Record): Promise - } - } - const stateAtG = JSON.parse( - JSON.stringify(await internals.storage.getMetadata('__aggregation_state_by_subtype__')) - ) - - // The missing window: one add, one group-moving update, one delete. - await brain.add({ data: 'e', type: NounType.Document, subtype: 'invoice', metadata: {} }) - await brain.update({ id: moving, subtype: 'invoice' }) - await brain.remove(doomed) - await brain.close() - brains.pop() - - // Rewind the aggregation artifact to the at-G bytes (the unclean exit). - { - const reopenForRewind = await open(dir) - const rw = reopenForRewind as unknown as typeof internals - await rw.storage.saveMetadata('__aggregation_state_by_subtype__', stateAtG) - await reopenForRewind.close() - brains.pop() - } - - // Session 2: reopen — adoption must see BEHIND and reconcile, never walk. - brain = await open(dir) - brain.defineAggregate(AGG) - const walkSpy = vi.spyOn( - brain as unknown as { runAggregationBackfillWalk(): Promise }, - 'runAggregationBackfillWalk' - ) - - const results = await brain.queryAggregate(AGG.name) - - // Ground truth after the window: invoice = a,b,e + moved c = 4; draft = 0 - // (c moved out, d deleted). - expect(countFor(results as never, 'invoice'), 'invoice count exact after catch-up').toBe(4) - expect(countFor(results as never, 'draft'), 'draft count exact after catch-up').toBe(0) - - // THE CALL-SHAPE PIN: the whole-store walk never ran. - expect(walkSpy, 'full backfill walk must not run for a behind-stamp reopen').not.toHaveBeenCalled() - - vi.restoreAllMocks() - }, 120000) -}) diff --git a/tests/integration/asof-semantic-recall.test.ts b/tests/integration/asof-semantic-recall.test.ts deleted file mode 100644 index 35805326..00000000 --- a/tests/integration/asof-semantic-recall.test.ts +++ /dev/null @@ -1,140 +0,0 @@ -/** - * @module tests/integration/asof-semantic-recall - * @description AS-OF SEMANTIC RECALL — the time-travel row of the release: - * vector/semantic search at a pinned past generation, served EXACTLY. - * - * The contract pinned here (brainy-alone leg; the accelerated-provider leg - * carries the same semantics at scale): - * 1. PAST VECTORS ARE THE PAST'S VECTORS: a later re-embed/update never - * leaks into an earlier pin — asOf(G) ranks by the vectors as they - * stood at G, byte-exact. - * 2. TOMBSTONE MASKING: a row deleted after G is FOUND at G; a row deleted - * at or before G is ABSENT at G. - * 3. THE DEFERRED-EMBED CELL of the visibility matrix: at pins before the - * vector landed the row's VECTOR LEG serves the stub (text/metadata - * legs may still surface it — triple intelligence by design); the real - * vector serves only at and after its landing pin. No backward leak. - * 4. TYPED REFUSAL beyond the log head — never a silent latest. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const brains: Brainy[] = [] - -async function memBrain(): Promise { - const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) - await b.init() - brains.push(b) - return b -} - -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) -}) - -describe('as-of semantic recall', () => { - it('PAST VECTORS EXACT: a later update never leaks into an earlier pin', async () => { - const brain = await memBrain() - const id = await brain.add({ - data: 'crimson apples in the orchard', - type: NounType.Document, - metadata: { epoch: 'old' } - }) - const g1 = brain.generation() - const v1 = [...(((await brain.get(id, { includeVectors: true }))!.vector) as number[])] - - await brain.update({ id, data: 'deep blue ocean currents', metadata: { epoch: 'new' } }) - const g2 = brain.generation() - const v2 = (await brain.get(id, { includeVectors: true }))!.vector as number[] - expect(v2, 'the update really re-embedded').not.toEqual(v1) - - // The pin: at G1 the row carries its ORIGINAL vector and content. - const dbPast = await brain.asOf(g1) - const past = await dbPast.get(id, { includeVectors: true }) - expect(past, 'row exists at G1').toBeTruthy() - expect(past!.vector as number[], 'as-of vector is byte-exact the OLD vector').toEqual(v1) - expect((past!.metadata as { epoch: string }).epoch).toBe('old') - - // Semantic search at G1 finds it via the OLD content; at G2 via the new. - const hitsOld = await dbPast.find({ query: 'crimson apples in the orchard', limit: 3 }) - expect(hitsOld.map((r) => r.id), 'old content recalls at G1').toContain(id) - const dbNow = await brain.asOf(g2) - const hitsNew = await dbNow.find({ query: 'deep blue ocean currents', limit: 3 }) - expect(hitsNew.map((r) => r.id), 'new content recalls at G2').toContain(id) - await dbPast.release() - await dbNow.release() - }) - - it('TOMBSTONE MASKING: deleted-after-G is found at G; deleted-before-G is absent', async () => { - const brain = await memBrain() - const doomed = await brain.add({ - data: 'ephemeral meteor shower observation', - type: NounType.Document, - metadata: {} - }) - const keeper = await brain.add({ - data: 'permanent granite mountain survey', - type: NounType.Document, - metadata: {} - }) - const gBoth = brain.generation() - await brain.remove(doomed) - const gAfter = brain.generation() - - const dbBoth = await brain.asOf(gBoth) - const atBoth = await dbBoth.find({ query: 'ephemeral meteor shower observation', limit: 5 }) - expect(atBoth.map((r) => r.id), 'pre-delete pin still recalls the row').toContain(doomed) - - const dbAfter = await brain.asOf(gAfter) - const atAfter = await dbAfter.find({ query: 'ephemeral meteor shower observation', limit: 5 }) - expect(atAfter.map((r) => r.id), 'post-delete pin masks the tombstoned row').not.toContain(doomed) - expect((await dbAfter.find({ query: 'permanent granite mountain survey', limit: 5 })).map((r) => r.id)).toContain(keeper) - await dbBoth.release() - await dbAfter.release() - }) - - it('DEFERRED-EMBED CELL: semantically absent before the vector landed, present after — never a stub match', async () => { - const brain = await memBrain() - // Anchor row so the semantic search always has a corpus. - await brain.add({ data: 'unrelated anchor topic entirely', type: NounType.Document, metadata: {} }) - - const id = await brain.add({ - data: 'deferred saffron sunrise essay', - type: NounType.Document, - deferEmbedding: true, - metadata: {} - }) - const gAck = brain.generation() - await brain.awaitPendingEmbeds() - const gLanded = brain.generation() - expect(gLanded, 'the landed vector is its own generation').toBeGreaterThan(gAck) - - // At the ack generation: metadata-visible, and the VECTOR LEG carries - // the stub (the visibility matrix's AT-EMBED cell governs the vector - // leg — find({query})'s text/metadata legs may legitimately still - // surface the row, that is triple intelligence working as designed; - // what must NEVER happen is a stub vector ranking as a real one). - const dbAck = await brain.asOf(gAck) - const metaHits = await dbAck.find({ where: {}, limit: 10 }) - expect(metaHits.map((r) => r.id), 'metadata-visible at ack pin').toContain(id) - const ackRow = await dbAck.get(id, { includeVectors: true }) - expect((ackRow!.vector as number[]).length, 'the as-of vector at the ack pin is the stub — no vector leaked backward').toBe(0) - - // At the landed generation: fully recallable. - const dbLanded = await brain.asOf(gLanded) - const landedRow = await dbLanded.get(id, { includeVectors: true }) - expect((landedRow!.vector as number[]).length, 'the real vector serves at the landed pin').toBeGreaterThan(0) - const semLanded = await dbLanded.find({ query: 'deferred saffron sunrise essay', limit: 5 }) - expect(semLanded.map((r) => r.id), 'recallable at the landed pin').toContain(id) - await dbAck.release() - await dbLanded.release() - }) - - it('TYPED REFUSAL beyond the head — never a silent latest', async () => { - const brain = await memBrain() - await brain.add({ data: 'one row', type: NounType.Document, metadata: {} }) - const head = brain.generation() - await expect(brain.asOf(head + 100)).rejects.toThrow(/generation|beyond|future|exceed/i) - }) -}) diff --git a/tests/integration/brain-relocation.test.ts b/tests/integration/brain-relocation.test.ts deleted file mode 100644 index 827bb959..00000000 --- a/tests/integration/brain-relocation.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -/** - * @module tests/integration/brain-relocation - * @description LC8 — RELOCATABLE BRAIN DIRECTORY. A brain's directory moved - * wholesale to a new path (rename/copy — backup-restore, disk migration, - * container re-mount) must open and serve IDENTICALLY: no absolute paths may - * hide in any persisted artifact. Pinned across every intelligence: point - * reads, metadata find, semantic find, graph traversal, aggregation — plus - * continued writes with monotonic generations and time-travel reads over - * pre-move history. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync, renameSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType, VerbType } from '../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] - -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -const AGG = { - name: 'by_kind', - source: { type: NounType.Document }, - groupBy: ['kind'] as string[], - metrics: { count: { op: 'count' as const } } -} - -describe('LC8 — a moved brain directory opens and serves identically', () => { - it('rename the directory: all three intelligences serve, writes continue, history travels', async () => { - const home = mkdtempSync(join(tmpdir(), 'brainy-reloc-')) - dirs.push(home) - const oldPath = join(home, 'brain-old') - const newPath = join(home, 'brain-new') - - // Season a brain: rows, a relation, an aggregate, then flush + close. - let brain = new Brainy({ storage: { type: 'filesystem', path: oldPath }, requireSubtype: false }) - await brain.init() - brains.push(brain) - brain.defineAggregate(AGG) - const alpha = await brain.add({ - data: 'alpha document about mountain geology', - type: NounType.Document, - metadata: { kind: 'report', n: 1 } - }) - const beta = await brain.add({ - data: 'beta document about coastal erosion', - type: NounType.Document, - metadata: { kind: 'report', n: 2 } - }) - await brain.relate({ from: alpha, to: beta, verb: VerbType.RelatedTo }) - await brain.queryAggregate(AGG.name) // settle backfill - const preMoveGen = brain.generation() - await brain.flush() - await brain.close() - brains.pop() - - // The move: wholesale directory rename. - renameSync(oldPath, newPath) - - // Reopen at the NEW path — everything serves. - brain = new Brainy({ storage: { type: 'filesystem', path: newPath }, requireSubtype: false }) - await brain.init() - brains.push(brain) - brain.defineAggregate(AGG) - - // Point read + metadata find. - expect((await brain.get(alpha))!.data).toContain('mountain geology') - const found = await brain.find({ where: { kind: 'report' }, limit: 10 }) - expect(found.map((r) => r.id).sort()).toEqual([alpha, beta].sort()) - - // Semantic find. - const sem = await brain.find({ query: 'alpha document about mountain geology', limit: 3 }) - expect(sem.map((r) => r.id)).toContain(alpha) - - // Graph traversal. - const related = await brain.related(alpha) - expect(related.map((r) => r.to)).toContain(beta) - - // Aggregation. - const agg = (await brain.queryAggregate(AGG.name)) as Array<{ - groupKey: Record - metrics: Record - }> - const reportRow = agg.find((g) => g.groupKey['kind'] === 'report') - expect(Number(reportRow?.metrics.count)).toBe(2) - - // Writes continue with monotonic generations. - const gamma = await brain.add({ - data: 'gamma addendum after the move', - type: NounType.Document, - metadata: { kind: 'report', n: 3 } - }) - expect(brain.generation()).toBeGreaterThan(preMoveGen) - expect((await brain.get(gamma))!.data).toContain('addendum') - - // Time travel across the move boundary: the pre-move pin sees exactly - // the pre-move world (no gamma), served from relocated history. - const dbPast = await brain.asOf(preMoveGen) - expect(await dbPast.get(gamma)).toBeNull() - expect((await dbPast.get(alpha))!.data).toContain('mountain geology') - await dbPast.release() - }, 120000) -}) diff --git a/tests/integration/db-mvcc.test.ts b/tests/integration/db-mvcc.test.ts index 0efc453e..959d0053 100644 --- a/tests/integration/db-mvcc.test.ts +++ b/tests/integration/db-mvcc.test.ts @@ -96,15 +96,11 @@ describe('8.0 Db API — generational MVCC', () => { } /** Open (and track) a filesystem brain rooted at a fresh temp directory. */ - async function openFsBrain( - dir?: string, - logAuthority?: 'adopt' | 'defer' - ): Promise<{ brain: Brainy; dir: string }> { + async function openFsBrain(dir?: string): Promise<{ brain: Brainy; dir: string }> { const rootDirectory = dir ?? makeTempDir() const brain = new Brainy({ requireSubtype: false, - storage: { type: 'filesystem', path: rootDirectory }, - ...(logAuthority ? { logAuthority } : {}) + storage: { type: 'filesystem', path: rootDirectory } }) await brain.init() brains.push(brain) @@ -651,13 +647,7 @@ describe('8.0 Db API — generational MVCC', () => { // ========================================================================== it('proof 8 — a crash before the manifest rename recovers to the exact pre-transaction state', async () => { const dir = makeTempDir() - // 'defer' (tree authority): this proof pins the TREE commit-point - // contract — the manifest rename is the commit, so a crash before it - // rolls back. Under the adopt-at-open default (log authority) the same - // crash point legitimately REPLAYS the fsynced fact at reopen and the - // transaction lands — that contract is pinned in the durability kill - // matrix's at-ack rows, not here. - const { brain: first } = await openFsBrain(dir, 'defer') + const { brain: first } = await openFsBrain(dir) await first.transact([ { @@ -699,10 +689,9 @@ describe('8.0 Db API — generational MVCC', () => { // the realistic worst case for the recovery path. await first.close() - // Reopen ('defer' again — a reopen under the adopt default would adopt - // and change the recovery path): recovery rolls the uncommitted - // generation back and rebuilds the indexes from the repaired records. - const { brain: second } = await openFsBrain(dir, 'defer') + // Reopen: recovery rolls the uncommitted generation back and rebuilds + // the indexes from the repaired records. + const { brain: second } = await openFsBrain(dir) const recovered = await second.get(uid('crash-e')) expect((recovered?.metadata as { v: number }).v).toBe(1) expect(await second.get(uid('crash-new'))).toBeNull() @@ -1173,16 +1162,13 @@ describe('8.0 Db API — generational MVCC', () => { const brain = await openMemoryBrain() // Model-B: a single-op write is its OWN generation and IS logged (no meta — - // tx metadata is a transact()-only concept). Relative baseline: under the - // adopt-at-open fleet default the open-time baseline backfill is itself a - // logged single-op generation, so the log is not empty on a fresh brain — - // every pin below is expressed against that baseline. - const baseGens = (await brain.transactionLog()).map((entry) => entry.generation) + // tx metadata is a transact()-only concept). It is generation 1 on a fresh + // brain (init-time infrastructure writes are the un-versioned gen-0 baseline). await brain.add({ id: uid('txlog-solo'), type: NounType.Document, data: 'solo', vector: vec(99), subtype: 'note' }) const soloLog = await brain.transactionLog() - const soloGen = brain.generation() - expect(soloLog.map((entry) => entry.generation)).toEqual([soloGen, ...baseGens]) + expect(soloLog.map((entry) => entry.generation)).toEqual([1]) expect(soloLog[0].meta).toBeUndefined() + const soloGen = 1 const first = await brain.transact( [{ op: 'add', id: uid('txlog-a'), type: NounType.Document, data: 'a', vector: vec(100), metadata: {} }], @@ -1195,14 +1181,12 @@ describe('8.0 Db API — generational MVCC', () => { const third = await brain.transact([{ op: 'update', id: uid('txlog-a'), metadata: { v: 3 } }]) const entries = await brain.transactionLog() - // Newest first: the three transacts, then the single-op solo write, then - // whatever the open baseline logged (the adopt-at-open backfill). + // Newest first: the three transacts, then the single-op solo write (gen 1). expect(entries.map((entry) => entry.generation)).toEqual([ third.generation, second.generation, first.generation, - soloGen, - ...baseGens + soloGen ]) expect(entries[1].meta).toEqual({ author: 'job-2' }) expect(entries[2].meta).toEqual({ author: 'job-1' }) @@ -1254,24 +1238,21 @@ describe('8.0 Db API — generational MVCC', () => { const brain = await openMemoryBrain() const a = uid('ov-a') const b = uid('ov-b') - // Pin RELATIVELY at the transact's own generation (not an absolute 1 — - // the adopt-at-open baseline backfill owns the first generation). - const tx = await brain.transact([ - { op: 'add', id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }, - { op: 'add', id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } } - ]) - const txGen = tx.generation - await tx.release() - const at1 = await brain.asOf(txGen) + await ( + await brain.transact([ + { op: 'add', id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }, + { op: 'add', id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } } + ]) + ).release() + const at1 = await brain.asOf(1) // A single-op REMOVE of `b` lands AFTER the pin and is NOT flushed (pending). await brain.remove(b) const liveIds = (await brain.find({})).map((r) => r.id) const pastIds = (await at1.find({})).map((r) => r.id) - // Live: `b` is gone. Historical (pinned at the transact's generation): the - // un-flushed removal is overlaid out, so `b` is still present at its - // pinned state. + // Live: `b` is gone. Historical (pinned at gen 1): the un-flushed removal is + // overlaid out, so `b` is still present at its pinned state. expect(liveIds).toContain(a) expect(liveIds).not.toContain(b) expect(pastIds).toContain(a) @@ -1281,14 +1262,11 @@ describe('8.0 Db API — generational MVCC', () => { it('Model-B retention — explicit caps reclaim single-op history; committed history survives reopen', async () => { const { brain, dir } = await openFsBrain() - // Relative baseline: the adopt-at-open backfill holds the first - // generation(s), so the 6 writes below land at base+1..base+6. - const base = brain.generation() const a = uid('ret-a') await brain.add({ id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }) for (let v = 2; v <= 6; v++) await brain.update({ id: a, metadata: { v } }) await brain.flush() // persist the per-write generations to disk - expect(brain.generation()).toBe(base + 6) + expect(brain.generation()).toBe(6) // Cap to the 2 most recent generations — older single-op history is reclaimed. const res = await brain.compactHistory({ maxGenerations: 2 }) diff --git a/tests/integration/db-temporal.test.ts b/tests/integration/db-temporal.test.ts index 335a1681..d17f5c16 100644 --- a/tests/integration/db-temporal.test.ts +++ b/tests/integration/db-temporal.test.ts @@ -36,9 +36,6 @@ import { GenerationCompactedError } from '../../src/db/errors.js' import type { GenerationStore } from '../../src/db/generationStore.js' import { NounType } from '../../src/types/graphTypes.js' -/** The VFS root — re-committed by the adopt-at-open baseline backfill. */ -const VFS_ROOT = '00000000-0000-0000-0000-000000000000' - /** Deterministic 384-dim vector so no test ever invokes the embedder. */ function vec(seed: number): number[] { return Array.from({ length: 384 }, (_, i) => ((seed * 31 + i * 7) % 100) / 100) @@ -136,11 +133,7 @@ describe('8.0 Db API — temporal range verbs', () => { expect(viaDb).toEqual(viaGen) expect(viaDb.fromGeneration).toBe(g1) expect(viaDb.nouns).toEqual([a, b].sort()) // a (updated after g1) + b (added after g1) - // (0, now] also includes a's creation — still {a, b} among user rows. The - // adopt-at-open baseline backfill re-commits the VFS root as a real - // generation, so the full-epoch window legitimately reports it too; - // filter it to keep this pin about the user writes. - expect(viaEpoch.nouns.filter((n) => n !== VFS_ROOT)).toEqual([a, b].sort()) + expect(viaEpoch.nouns).toEqual([a, b].sort()) // (0, now] also includes a's creation, still {a, b} // direction guard: an older view cannot be `since` a newer lower bound const older = await brain.asOf(1) @@ -170,11 +163,7 @@ describe('8.0 Db API — temporal range verbs', () => { } const all = await brain.transactionLog() - // Newest first — compared above the open baseline (the adopt-at-open - // backfill logs its own generation(s) below the first user write). - expect(all.map((e) => e.generation).filter((g) => g >= gens[0])).toEqual( - [...gens].reverse() - ) + expect(all.map((e) => e.generation)).toEqual([...gens].reverse()) // newest first // INCLUSIVE both ends — gens[1] AND gens[3] are present (contrast since's exclusive lower). const windowed = await brain.transactionLog({ from: gens[1], to: gens[3] }) @@ -345,22 +334,19 @@ describe('8.0 Db API — temporal range verbs', () => { // 7. Granularity (Model-B) --------------------------------------------------- it('granularity: single-operation writes ARE versioned and visible to the temporal verbs', async () => { const brain = await openMemoryBrain() - // Relative baseline: the adopt-at-open backfill already logged its own - // generation(s) — pin the DELTA this test's writes add, not a count. - const baseCount = (await brain.transactionLog()).length const a = uid('gran-a') const r1 = await brain.transact([ { op: 'add', id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } } ]) await r1.release() - expect((await brain.transactionLog()).length).toBe(baseCount + 1) + expect((await brain.transactionLog()).length).toBe(1) // Model-B: a single-op write is its OWN immutable generation — logged, // diffable, and time-travelable, exactly like a transact() of one op. await brain.update({ id: a, metadata: { v: 2 } }) // The single-op update appended a generation/log entry. - expect((await brain.transactionLog()).length).toBe(baseCount + 2) + expect((await brain.transactionLog()).length).toBe(2) expect(brain.generation()).toBe(r1.generation + 1) // diff sees the single-op update as a modification of `a`. diff --git a/tests/integration/deferred-embedding.test.ts b/tests/integration/deferred-embedding.test.ts deleted file mode 100644 index 819ddbad..00000000 --- a/tests/integration/deferred-embedding.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -/** - * @module tests/integration/deferred-embedding - * @description MT5 — THE DEFERRED-EMBEDDING CONTRACT (A3 of the service-class - * pair, BRAINY-PROD-LATENCY-TRIAD). The production disease: a VFS file write - * ran a neural network synchronously while the caller waited (5.6s p50 per - * small file). The contract pinned here: - * - * 1. ACK AT DURABILITY: a deferred write never calls the embedder on the - * caller's path — the row is id/metadata-findable immediately, with a - * durable pending marker and an honest `pendingEmbeds` gauge. - * 2. EVENTUAL VECTOR INDEX: `awaitPendingEmbeds()` is the barrier — after - * it, the vector is real, indexed, and the marker is reaped. - * 3. STALE-BEATS-ABSENT on deferred updates: the OLD vector keeps serving - * until the atomic swap (the flicker law, never a dark window). - * 4. CRASH-SAFE: markers survive a session that dies mid-defer; the next - * open recovers and lands the vector. A crash DELAYS a vector, never - * loses one. - * 5. TYPED REFUSALS: deferEmbedding + vector, and deferEmbedding without - * data, are caller bugs that refuse with the fix in the message. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] - -async function memBrain(): Promise { - const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) - await b.init() - brains.push(b) - return b -} - -afterEach(async () => { - vi.restoreAllMocks() - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -describe('MT5 — deferred embedding', () => { - it('ACK LAW: add({deferEmbedding}) never embeds on the caller path; row findable immediately; barrier lands the vector and reaps the marker', async () => { - const brain = await memBrain() - const embedSpy = vi.spyOn(brain, 'embed') - - const id = await brain.add({ - data: 'deferred content', - type: NounType.Document, - deferEmbedding: true, - metadata: { tag: 'deferred' } - }) - - // The caller's path never ran the embedder. - expect(embedSpy, 'no embed on the ack path').not.toHaveBeenCalled() - - // Immediately findable by metadata; vector is the stub; gauge honest. - const found = await brain.find({ where: { tag: 'deferred' }, limit: 5 }) - expect(found.map((r) => r.id)).toContain(id) - expect((await brain.getIndexStatus()).pendingEmbeds).toBeGreaterThanOrEqual(1) - - // The barrier: vector lands, marker reaped, index carries the row. - await brain.awaitPendingEmbeds() - expect(embedSpy).toHaveBeenCalled() - const after = await brain.get(id, { includeVectors: true }) - expect((after!.vector as number[]).length, 'real vector after the barrier').toBeGreaterThan(0) - expect(brain.pendingEmbedCount()).toBe(0) - expect((await brain.getIndexStatus()).pendingEmbeds).toBe(0) - }) - - it('STALE-BEATS-ABSENT: a deferred update serves the OLD vector until the atomic swap; data reads NEW immediately', async () => { - const brain = await memBrain() - const id = await brain.add({ data: 'original content', type: NounType.Document, metadata: {} }) - const before = await brain.get(id, { includeVectors: true }) - const oldVector = [...(before!.vector as number[])] - expect(oldVector.length).toBeGreaterThan(0) - - await brain.update({ id, data: 'completely different content', deferEmbedding: true }) - - // Data is new IMMEDIATELY; the vector is still the old one (present, - // never absent) until the worker swaps it. - const mid = await brain.get(id, { includeVectors: true }) - expect(mid!.data).toBe('completely different content') - expect(mid!.vector as number[], 'old vector keeps serving').toEqual(oldVector) - - await brain.awaitPendingEmbeds() - const after = await brain.get(id, { includeVectors: true }) - expect((after!.vector as number[]).length).toBeGreaterThan(0) - expect(after!.vector as number[], 'vector swapped after the barrier').not.toEqual(oldVector) - }) - - it('CRASH-SAFE: a session dying mid-defer leaves the durable marker; the next open recovers and lands the vector', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-defer-crash-')) - dirs.push(dir) - - // Session 1: the embedder hangs → the worker can never complete; close() - // does not wait for it (crash-equivalent for the embed leg). - let brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await brain.init() - brains.push(brain) - vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) - const id = await brain.add({ - data: 'survives the crash', - type: NounType.Document, - deferEmbedding: true, - metadata: { k: 1 } - }) - expect(brain.pendingEmbedCount()).toBe(1) - await brain.close() - brains.pop() - vi.restoreAllMocks() - - // Session 2: recovery lists the marker and resumes in the background. - brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await brain.init() - brains.push(brain) - expect(brain.pendingEmbedCount(), 'marker recovered at open').toBe(1) - - await brain.awaitPendingEmbeds() - const after = await brain.get(id, { includeVectors: true }) - expect((after!.vector as number[]).length, 'the delayed vector landed').toBeGreaterThan(0) - expect(brain.pendingEmbedCount()).toBe(0) - }, 120000) - - it('VFS ACK LAW: writeFile resolves even when the embedder HANGS forever — the ack never depends on a neural net', async () => { - const brain = await memBrain() - // The strongest form of the pin: an embedder that never resolves. If any - // part of the writeFile ack path awaited an embed, this test would hang. - // (The background worker legitimately picks the deferred embeds up later - // — it may even interleave on the event loop during writeFile's other - // awaits — but the CALLER'S promise must never depend on it.) - const hang = vi - .spyOn(brain, 'embed') - .mockImplementation(() => new Promise(() => {})) - - await brain.vfs.writeFile('/notes/today.md', '# The day\nA deferred capture.') - - // Acked with the embedder hung: content + metadata fully readable. - const content = await brain.vfs.readFile('/notes/today.md') - expect(content.toString()).toContain('A deferred capture.') - expect(brain.pendingEmbedCount()).toBeGreaterThanOrEqual(1) - - // Un-hang, abandon the poisoned in-flight run (its embed promise never - // resolves — production is covered by the worker's 60s hang guard; the - // test takes the white-box shortcut for speed), drain, verify. - hang.mockRestore() - ;(brain as unknown as { _embedWorkerFlight: Promise | null })._embedWorkerFlight = null - await brain.awaitPendingEmbeds() - expect(brain.pendingEmbedCount()).toBe(0) - }) - - it('TYPED REFUSALS: defer+vector and defer-without-data both refuse with the fix', async () => { - const brain = await memBrain() - await expect( - brain.add({ - data: 'x', - vector: new Array(384).fill(0.1), - type: NounType.Document, - deferEmbedding: true, - metadata: {} - }) - ).rejects.toThrow(/deferEmbedding cannot be combined/) - - const id = await brain.add({ data: 'y', type: NounType.Document, metadata: {} }) - await expect( - brain.update({ id, deferEmbedding: true, metadata: { z: 1 } }) - ).rejects.toThrow(/requires new 'data'/) - }) -}) diff --git a/tests/integration/durability-kill-matrix.test.ts b/tests/integration/durability-kill-matrix.test.ts deleted file mode 100644 index 70962dda..00000000 --- a/tests/integration/durability-kill-matrix.test.ts +++ /dev/null @@ -1,701 +0,0 @@ -/** - * @module tests/integration/durability-kill-matrix - * @description THE DURABILITY KILL MATRIX — for every step of the commit - * path, inject a crash AT that step (the generation store's test-only fault - * injector), then reopen the same storage directory with a brand-new Brainy - * and assert the recovery contract BY CONSTRUCTION, not by timing: - * - * - an ACKED write survives the crash (never a lost ack), and - * - an UN-ACKED write leaves no torn state (fully present or fully absent, - * never half). - * - * The crash simulation is honest process death: the crashed brain is NEVER - * closed — `abandonAsCrashed` discards its buffered RAM state exactly as a - * dead process would, and recovery on the next open is the only repair that - * runs. File bytes already handed to the OS survive (process-crash model); - * one row additionally models POWER LOSS by removing an entity's un-fsynced - * canonical files (legal: single-op canonical writes are tmp+rename without - * fsync). - * - * Matrix rows (fault point → durability barrier position): - * - * BEFORE the barrier (nothing durable records the write): - * singleop-after-execute · singleop-after-fact-append · flush-after-staging - * AFTER partial durability (staged/synced bytes exist, manifest did not advance): - * flush-before-manifest · before-manifest-rename (transact) · - * transact-after-fact-sync - * AFTER the commit point: - * after-manifest-rename (transact) - * MODE VARIANTS: singleop-after-fact-append under durable-at-ack. - * DISK FULL: one ENOSPC'd append — loud typed rejection, reads keep - * serving, a later write succeeds. - * - * Where the observed recovery contract differs from the ideal, the pin states - * the OBSERVED behavior with a comment; where the observed behavior violates - * "never a torn state / never a lost ack", the pin asserts the CONTRACT and - * is marked `.fails` — a release-blocking finding, deliberately not weakened. - */ -import { describe, it, expect, afterEach } from 'vitest' -import * as fs from 'node:fs' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { - abandonAsCrashed, - armCrash, - dropCanonicalNoun, - factGenerations, - failNextAppendWithEnospc, - generationDirExists, - makeTempDir, - openBrain, - storeOf, - uid, - vec -} from '../helpers/durabilityKillMatrix.js' - -describe('durability kill matrix — crash at every commit-path step, recover by reopen', () => { - const dirs: string[] = [] - const liveBrains: Brainy[] = [] - // Crashed brains are deliberately NEVER closed (a dead process cannot - // close); they are severed by abandonAsCrashed inside each test. - - function trackDir(): string { - const dir = makeTempDir() - dirs.push(dir) - return dir - } - - async function openLive(dir: string): Promise { - const brain = await openBrain(dir) - liveBrains.push(brain) - return brain - } - - afterEach(async () => { - for (const brain of liveBrains.splice(0)) { - try { - await brain.close() - } catch { - // already closed / crashed mid-close — teardown only - } - } - for (const dir of dirs.splice(0)) { - await fs.promises.rm(dir, { recursive: true, force: true }) - } - }) - - /** Baseline arrangement: one durable row + explicit flush = the durable floor. */ - async function arrangeBaseline(label: string): Promise<{ - dir: string - brain: Brainy - baselineId: string - floor: number - }> { - const dir = trackDir() - const brain = await openBrain(dir) // NOT tracked live — most rows crash it - const baselineId = uid(`${label}-baseline`) - await brain.add({ - id: baselineId, - data: 'baseline row', - type: NounType.Document, - vector: vec(1), - metadata: { v: 1 } - }) - await brain.flush() - return { dir, brain, baselineId, floor: storeOf(brain).committedGeneration() } - } - - /** - * Flip a brain to durable-at-ack (log-authority) mode. - * - * NOT via `adoptLogAuthority()` (and the helper opens every brain with - * `logAuthority: 'defer'`, opting out of the 10.0.0 adopt-at-open fleet - * default): the sanctioned path runs the oracle and a baseline backfill, - * which appends its own generation — shifting the floor arithmetic every - * row pins. This helper flips the SAME switch the sanctioned path flips - * (`setLogDurability('at-ack')`) and persists the SAME authority artifact, - * so a reopened brain also runs in log-authority mode. The durability - * semantics under test are governed entirely by that switch. - */ - async function flipToAtAck(brain: Brainy): Promise { - const storage = ( - brain as unknown as { - storage: { - writeRawObject(p: string, d: unknown): Promise - syncRawObjects(p: string[]): Promise - } - } - ).storage - await storage.writeRawObject('_system/log-authority.json', { - authority: 'log', - flippedAt: Date.now() - }) - await storage.syncRawObjects(['_system/log-authority.json']) - storeOf(brain).setLogDurability('at-ack') - } - - // ========================================================================== - // Rows BEFORE the durability barrier — the write never became durable-acked - // ========================================================================== - - it('singleop-after-execute — un-acked write is atomic (present-whole), baseline and log stay at the floor', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('sae') - const crashedId = uid('sae-crashed') - const arm = armCrash(brain, 'singleop-after-execute') - await expect( - brain.add({ - id: crashedId, - data: 'never acked', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - ).rejects.toThrow('simulated process crash at singleop-after-execute') - expect(arm.fired).toContain('singleop-after-execute') - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // Baseline intact. - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - // The log holds nothing beyond the committed watermark (no fact was ever - // appended for the crashed write). - expect(await factGenerations(reopened)).toEqual([floor]) - expect(storeOf(reopened).committedGeneration()).toBe(floor) - // The un-acked write: Model-B applies the live canonical write BEFORE the - // ack, so under process death its bytes survive — the row is PRESENT and - // WHOLE by id (atomic, not torn). Under power loss the same un-fsynced - // bytes may instead vanish entirely; both end states are atomic. NOTE the - // divergence: the row is get()-visible but find()-invisible (no index - // entry survived, no generation/fact records it, and no repair is pending - // — a permanent canonical orphan; see the suite report). - const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null - expect(orphan).not.toBeNull() - expect(orphan!.metadata.v).toBe(2) // whole, byte-consistent — never torn - const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }> - expect(found.map((f) => f.id)).toContain(baselineId) - expect(found.map((f) => f.id)).not.toContain(crashedId) - // A fresh write succeeds with a monotonic generation. The crashed - // generation number is REUSED (nothing durable references it): the - // counter reopened at the floor. - expect(reopened.generation()).toBe(floor) - const freshId = uid('sae-fresh') - await reopened.add({ - id: freshId, - data: 'fresh after recovery', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(floor + 1) - expect(((await reopened.get(freshId)) as { metadata: { v: number } }).metadata.v).toBe(3) - }) - - it('singleop-after-fact-append (deferred mode) — the appended fact is truncated back at reopen', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('sfa') - const crashedId = uid('sfa-crashed') - const arm = armCrash(brain, 'singleop-after-fact-append') - await expect( - brain.add({ - id: crashedId, - data: 'never acked', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - ).rejects.toThrow('simulated process crash at singleop-after-fact-append') - expect(arm.fired).toContain('singleop-after-fact-append') - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // The fact WAS appended to the log file before the crash (process death - // keeps file bytes) — open() must truncate it back to the manifest - // watermark, and does. - expect(await factGenerations(reopened)).toEqual([floor]) - expect(storeOf(reopened).committedGeneration()).toBe(floor) - // Baseline intact; un-acked row atomic (present-whole via canonical, as - // in the singleop-after-execute row). - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null - expect(orphan).not.toBeNull() - expect(orphan!.metadata.v).toBe(2) - // Fresh write with a monotonic generation (crashed number reused — the - // truncated fact freed it). - expect(reopened.generation()).toBe(floor) - const freshId = uid('sfa-fresh') - await reopened.add({ - id: freshId, - data: 'fresh', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(floor + 1) - expect(await factGenerations(reopened)).toEqual([floor, floor + 1]) - }) - - it('flush-after-staging — the ACKED write survives (drop-without-restore); only the window history is lost', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('fas') - const ackedId = uid('fas-acked') - await brain.add({ - id: ackedId, - data: 'acked before flush', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - const ackedGen = storeOf(brain).generation() - const arm = armCrash(brain, 'flush-after-staging') - await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-after-staging') - expect(arm.fired).toContain('flush-after-staging') - // The crashed flush left the staged record-set dir on disk, above the manifest. - expect(generationDirExists(dir, ackedGen)).toBe(true) - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // Recovery DROPPED the staged group-commit dir WITHOUT restoring its - // before-images — restoring would silently revert an acknowledged write. - expect(generationDirExists(dir, ackedGen)).toBe(false) - expect(storeOf(reopened).committedGeneration()).toBe(floor) - // NEVER A LOST ACK: the acknowledged write is present and whole. - const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null - expect(acked).not.toBeNull() - expect(acked!.metadata.v).toBe(2) - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - // Recovery rolled generations back → index reconciliation ran → the acked - // row is find()-visible too. - const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }> - expect(found.map((f) => f.id)).toEqual(expect.arrayContaining([baselineId, ackedId])) - // The window's HISTORY is the documented cost: its fact is truncated back - // (the acked row now lives only in canonical bytes, not the log). - expect(await factGenerations(reopened)).toEqual([floor]) - // The crashed generation number is NOT reused (its dropped dir was seen - // at open): fresh writes continue above it. - expect(reopened.generation()).toBe(ackedGen) - const freshId = uid('fas-fresh') - await reopened.add({ - id: freshId, - data: 'fresh', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1) - }) - - // ========================================================================== - // Rows AFTER partial durability — staged/synced bytes exist, no manifest - // ========================================================================== - - it('flush-before-manifest — staged bytes + synced facts above the manifest are dropped/truncated; the acked write stays', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('fbm') - const ackedId = uid('fbm-acked') - await brain.add({ - id: ackedId, - data: 'acked before flush', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - const ackedGen = storeOf(brain).generation() - const arm = armCrash(brain, 'flush-before-manifest') - await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-before-manifest') - // The earlier flush phase passed through untripped before the target fired. - expect(arm.fired).toContain('flush-after-staging') - expect(arm.fired).toContain('flush-before-manifest') - expect(generationDirExists(dir, ackedGen)).toBe(true) - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // Per the recovery contract in open(): groupCommit record-sets above the - // manifest are dropped WITHOUT restore, and the (fsynced!) facts above - // the manifest are truncated back. The acked live write stays. - expect(generationDirExists(dir, ackedGen)).toBe(false) - expect(storeOf(reopened).committedGeneration()).toBe(floor) - expect(await factGenerations(reopened)).toEqual([floor]) - const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null - expect(acked).not.toBeNull() // never a lost ack - expect(acked!.metadata.v).toBe(2) - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - // Fresh write above the crashed generation (number not reused). - expect(reopened.generation()).toBe(ackedGen) - const freshId = uid('fbm-fresh') - await reopened.add({ - id: freshId, - data: 'fresh', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1) - }) - - it('before-manifest-rename (transact) — fully staged, never committed: rolled back byte-identically', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('bmr') - const newId = uid('bmr-new') - const arm = armCrash(brain, 'before-manifest-rename') - await expect( - brain.transact([ - { op: 'update', id: baselineId, metadata: { v: 2 } }, - { - op: 'add', - id: newId, - type: NounType.Document, - data: 'uncommitted', - vector: vec(2), - metadata: { v: 2 } - } - ]) - ).rejects.toThrow('simulated process crash at before-manifest-rename') - expect(arm.fired).toContain('before-manifest-rename') - const txGen = storeOf(brain).generation() - expect(generationDirExists(dir, txGen)).toBe(true) - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // Rolled back cleanly: the update is undone, the add is ABSENT everywhere. - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - expect(await reopened.get(newId)).toBeNull() - const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }> - expect(found.map((f) => f.id)).not.toContain(newId) - expect(generationDirExists(dir, txGen)).toBe(false) - expect(storeOf(reopened).committedGeneration()).toBe(floor) - expect(await factGenerations(reopened)).toEqual([floor]) - // The crashed generation number is never reissued (counter persisted - // before the crash point). - expect(reopened.generation()).toBe(txGen) - const freshId = uid('bmr-fresh') - await reopened.add({ - id: freshId, - data: 'fresh', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1) - }) - - it('transact-after-fact-sync — the fsynced fact of an uncommitted transact is truncated back; rollback is clean', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('tfs') - const newId = uid('tfs-new') - const arm = armCrash(brain, 'transact-after-fact-sync') - await expect( - brain.transact([ - { op: 'update', id: baselineId, metadata: { v: 2 } }, - { - op: 'add', - id: newId, - type: NounType.Document, - data: 'uncommitted', - vector: vec(2), - metadata: { v: 2 } - } - ]) - ).rejects.toThrow('simulated process crash at transact-after-fact-sync') - expect(arm.fired).toContain('transact-after-fact-sync') - const txGen = storeOf(brain).generation() - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // The batch's fact was appended AND fsynced before the crash — open() - // must truncate it back to the manifest watermark (the generation never - // committed), and the before-images must restore byte-identically. - expect(await factGenerations(reopened)).toEqual([floor]) - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - expect(await reopened.get(newId)).toBeNull() - expect(storeOf(reopened).committedGeneration()).toBe(floor) - expect(generationDirExists(dir, txGen)).toBe(false) - // Counter: the staged dir was seen at open, so the number is not reused. - expect(reopened.generation()).toBe(txGen) - const freshId = uid('tfs-fresh') - await reopened.add({ - id: freshId, - data: 'fresh', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1) - }) - - // ========================================================================== - // Row AFTER the commit point — the transaction must be kept - // ========================================================================== - - it('after-manifest-rename (transact) — the manifest rename landed: the transaction is COMMITTED and fully present', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('amr') - const newId = uid('amr-new') - const arm = armCrash(brain, 'after-manifest-rename') - await expect( - brain.transact([ - { op: 'update', id: baselineId, metadata: { v: 2 } }, - { - op: 'add', - id: newId, - type: NounType.Document, - data: 'committed by the rename', - vector: vec(2), - metadata: { v: 2 } - } - ]) - ).rejects.toThrow('simulated process crash at after-manifest-rename') - expect(arm.fired).toContain('after-manifest-rename') - const txGen = storeOf(brain).generation() - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // COMMITTED: both operations present, atomically. - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(2) - const added = (await reopened.get(newId)) as { metadata: { v: number } } | null - expect(added).not.toBeNull() - expect(added!.metadata.v).toBe(2) - expect(storeOf(reopened).committedGeneration()).toBe(txGen) - // The fact was synced before the commit point and sits at/below the - // manifest — it is KEPT. - expect(await factGenerations(reopened)).toEqual([floor, txGen]) - // Fresh writes continue above the committed generation. - const freshId = uid('amr-fresh') - await reopened.add({ - id: freshId, - data: 'fresh', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1) - }) - - // ========================================================================== - // Durable-at-ack (log-authority) mode variants - // ========================================================================== - - it('singleop-after-fact-append (at-ack mode) — the intact fact is REPLAYED at reopen; the write commits', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('aaf') - await flipToAtAck(brain) - const crashedId = uid('aaf-crashed') - const arm = armCrash(brain, 'singleop-after-fact-append') - await expect( - brain.add({ - id: crashedId, - data: 'fact fsynced, never acked', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - ).rejects.toThrow('simulated process crash at singleop-after-fact-append') - expect(arm.fired).toContain('singleop-after-fact-append') - await abandonAsCrashed(brain) - - const reopened = await openLive(dir) - // LOG-AUTHORITY RECOVERY CONTRACT: under 'log' authority, an intact - // fact above the manifest is adopted at open — REPLAYED into canonical - // and committed — never truncated. (At-least-once at the fact layer: a - // crashed-pre-ack write whose fact survived intact becomes committed; - // that is a valid write landing, never a torn or lost state.) - expect(await factGenerations(reopened)).toEqual([floor, floor + 1]) - expect(storeOf(reopened).committedGeneration()).toBe(floor + 1) - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - const replayed = (await reopened.get(crashedId)) as { metadata: { v: number } } | null - expect(replayed).not.toBeNull() - expect(replayed!.metadata.v).toBe(2) - // Fresh write lands monotonically ABOVE the replayed generation. - const freshId = uid('aaf-fresh') - await reopened.add({ - id: freshId, - data: 'fresh', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await reopened.flush() - expect(storeOf(reopened).committedGeneration()).toBe(floor + 2) - }) - - // THE AT-ACK CONTRACT, END TO END (was a release-blocking finding; fixed - // by log-authority replay-at-open): under power loss the un-fsynced - // tmp+rename canonical bytes legally vanish while the fsynced fact - // survives — recovery REPLAYS that fact into canonical, so the acked - // write lives. This is the sentence 'durable-at-ack' actually promises. - it( - 'at-ack POWER LOSS — an ACKED write whose fact is fsynced SURVIVES reopen via log replay', - async () => { - const { dir, brain, baselineId } = await arrangeBaseline('apl') - await flipToAtAck(brain) - const ackedId = uid('apl-acked') - // No fault injector: this write ACKS normally — in at-ack mode the ack - // returned only after a covering log fsync. - await brain.add({ - id: ackedId, - data: 'acked, fact fsynced', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - // Crash before any flush: RAM is gone… - await abandonAsCrashed(brain) - // …and power loss takes the un-fsynced canonical rename with it. The - // fsynced fact log survives — it is the write's only durable copy. - dropCanonicalNoun(dir, ackedId) - - const reopened = await openLive(dir) - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - // THE AT-ACK CONTRACT: the acknowledged write survives the crash. - // Observed today: open() truncates its fact back to the manifest - // watermark and the write is gone everywhere. - const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null - expect(acked).not.toBeNull() - expect(acked!.metadata.v).toBe(2) - } - ) - - // ========================================================================== - // Disk full — one ENOSPC'd append - // ========================================================================== - - it('disk full — an ENOSPC append rejects loudly and typed; reads keep serving; a later write succeeds', async () => { - const { dir, brain, baselineId, floor } = await arrangeBaseline('nospc') - liveBrains.push(brain) // this row never crashes the brain - void dir - const failedId = uid('nospc-failed') - const probe = failNextAppendWithEnospc(brain) - // LOUD, TYPED, never a silent success: the raw ENOSPC surfaces to the - // caller with its errno code intact. - await expect( - brain.add({ - id: failedId, - data: 'no space', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - ).rejects.toMatchObject({ code: 'ENOSPC' }) - expect(probe.failed()).toBe(1) - // The store still serves reads. - expect(((await brain.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - // Space "restored" (the failing patch self-cleared): a later write succeeds - // end to end, including its fact and an explicit durability barrier. - const laterId = uid('nospc-later') - await brain.add({ - id: laterId, - data: 'space restored', - type: NounType.Document, - vector: vec(3), - metadata: { v: 3 } - }) - await brain.flush() - expect(((await brain.get(laterId)) as { metadata: { v: number } }).metadata.v).toBe(3) - expect(storeOf(brain).committedGeneration()).toBeGreaterThan(floor) - // FIXED BEHAVIOR (was: the rejected generation stayed buffered and the - // next flush committed it with NO fact — a silent log gap): the failure - // path un-buffers the generation and returns the counter reservation, - // so the later write takes floor+1 and the log is gap-free. - expect(storeOf(brain).committedGeneration()).toBe(floor + 1) - expect(await factGenerations(brain)).toEqual([floor, floor + 1]) - // Canonical residue of the rejected write (execute ran before the - // append failed) is the documented Model-B crash-equivalent orphan — - // uncommitted, absent from the log, same shape as a crash at execute. - expect(((await brain.get(failedId)) as { metadata: { v: number } } | null)?.metadata.v).toBe(2) - }) - - // THE NO-SILENT-COMMIT CONTRACT (was a release-blocking finding; fixed by - // un-buffering on append failure): a loudly-rejected write never becomes - // durably committed and the log never carries a gap. Canonical residue - // (the execute-before-commit orphan) is the documented Model-B - // crash-equivalent, pinned in the row above — NOT a commit. - it('disk full — a write rejected for a failed fact append is NOT silently committed', async () => { - const { brain, floor } = await arrangeBaseline('nogap') - liveBrains.push(brain) - const failedId = uid('nogap-failed') - failNextAppendWithEnospc(brain) - await expect( - brain.add({ - id: failedId, - data: 'no space', - type: NounType.Document, - vector: vec(2), - metadata: { v: 2 } - }) - ).rejects.toMatchObject({ code: 'ENOSPC' }) - await brain.flush() - // THE CONTRACT: nothing was committed behind the caller's back — the - // log carries no gap and no generation for the rejected write. (get() - // still serves the canonical execute-residue orphan — the documented - // Model-B crash-equivalent, pinned in the row above.) - expect(storeOf(brain).committedGeneration()).toBe(floor) - expect(await factGenerations(brain)).toEqual([floor]) - }) - - // ========================================================================== - // Block-layer power-loss findings (first dm-flakey run) — the three cures - // ========================================================================== - - it('at-ack POWER LOSS BELOW THE MANIFEST — an unclean open folds the WHOLE log; acked writes committed before the flush still survive vanished canonical', async () => { - const { dir, brain, baselineId } = await arrangeBaseline('wlf') - await flipToAtAck(brain) - const ackedA = uid('wlf-a') - const ackedB = uid('wlf-b') - await brain.add({ id: ackedA, data: 'below manifest one', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) - await brain.add({ id: ackedB, data: 'below manifest two', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) - // The group-commit flush advances the manifest OVER these generations — - // but live canonical bytes are tmp+rename without per-file fsync, so a - // power cut can still take them. The fsynced facts are the durable copy. - await (brain as unknown as { flush(): Promise }).flush() - await abandonAsCrashed(brain) // no clean close → no clean-shutdown marker - dropCanonicalNoun(dir, ackedA) - dropCanonicalNoun(dir, ackedB) - - const reopened = await openLive(dir) - // The whole-log fold restores BOTH rows from facts ≤ manifest. - expect(((await reopened.get(ackedA)) as { metadata: { v: number } }).metadata.v).toBe(2) - expect(((await reopened.get(ackedB)) as { metadata: { v: number } }).metadata.v).toBe(3) - expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) - }) - - it('clean-shutdown marker: a clean close writes it, the next open consumes it (no fold on the happy path)', async () => { - const { dir, brain } = await arrangeBaseline('csm') - await flipToAtAck(brain) - await brain.close() - liveBrains.splice(liveBrains.indexOf(brain), 1) - // The adapter stores raw objects gzipped — accept either spelling. - const markerExists = () => - fs.existsSync(join(dir, '_system', 'clean-shutdown.json')) || - fs.existsSync(join(dir, '_system', 'clean-shutdown.json.gz')) - expect(markerExists(), 'clean close stamps the marker').toBe(true) - - const reopened = await openLive(dir) - expect(markerExists(), 'open consumes the marker').toBe(false) - await reopened.close() - liveBrains.splice(liveBrains.indexOf(reopened), 1) - expect(markerExists(), 'the next clean close re-stamps it').toBe(true) - }) - - it('torn writer lock (empty file) — open treats it as stale and recovers; never a permanent lockout', async () => { - const { dir, brain } = await arrangeBaseline('tlk') - await brain.close() - liveBrains.splice(liveBrains.indexOf(brain), 1) - // The power-loss shape: the lock file exists but is EMPTY (torn write). - fs.writeFileSync(join(dir, 'locks', '_writer.lock'), '') - - const reopened = await openLive(dir) // must not throw 'contended' - const fresh = uid('tlk-fresh') - await reopened.add({ id: fresh, data: 'lock recovered', type: NounType.Document, vector: vec(4), metadata: { v: 4 } }) - expect(await reopened.get(fresh)).not.toBeNull() - }) - - it('pair guard: a metadata index without stampWatermark never crashes flush', async () => { - const { brain } = await arrangeBaseline('psg') - liveBrains.push(brain) - // The native pair swaps the metadata manager; the replacement may not - // carry the stamp method — flush must treat that as verdict-side rescan, - // never a TypeError at the fan-out. - ;(brain as unknown as { metadataIndex: { stampWatermark?: unknown } }).metadataIndex.stampWatermark = undefined - await expect((brain as unknown as { flush(): Promise }).flush()).resolves.toBeUndefined() - }) -}) diff --git a/tests/integration/embed-markers-in-log.test.ts b/tests/integration/embed-markers-in-log.test.ts deleted file mode 100644 index 2dcad2f1..00000000 --- a/tests/integration/embed-markers-in-log.test.ts +++ /dev/null @@ -1,320 +0,0 @@ -/** - * @module tests/integration/embed-markers-in-log - * @description DEFERRED-EMBED MARKERS ARE LOG RECORDS — the sidecar is dead. - * The pending-embed lifecycle lives IN the generation log as first-class v2 - * records: `embed.pending` rides the deferred write's OWN commit fact (same - * generation, one atomic append — a marker can never be orphaned from its - * write nor the write from its marker) and `embed.landed` rides the - * background worker's landing commit. Recovery is REPLAY, NOT LISTING: the - * open-time fold arms every pending without a matching landed (minus rows - * the log later tombstoned). The pins: - * - * (a) SAME-FACT ATOMICITY: a deferred add's commit fact carries the - * embed.pending record BESIDE its noun after-image — one generation, - * one frame — and no sidecar file is ever written. - * (b) LANDING: after the barrier, the log carries embed.landed (inline - * vector, per the v2 format) riding the landing commit's own fact, and - * a fresh fold of the whole log nets ZERO pending. - * (c) CRASH RECOVERY VIA THE LOG: kill mid-defer (hung embedder, flushed - * durability, crash-style abandon), reopen — the fold re-arms exactly - * one pending with NO sidecar file existing anywhere, and the vector - * then lands. - * (d) LEGACY BRIDGE: a sidecar marker file left by a pre-log build is - * folded in at open, migrated into the log as an embed.pending record, - * and the file is deleted — one-time, durable, idempotent. - * (e) VFS ACK LAW (unchanged contract, new mechanism): writeFile acks - * under a forever-hung embedder while its pending marker sits durably - * in the log. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as path from 'node:path' -import * as zlib from 'node:zlib' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import type { CommitFact } from '../../src/db/factLog.js' -import { - makeTempDir, - openBrain, - abandonAsCrashed, - vec, - uid -} from '../helpers/durabilityKillMatrix.js' - -/** The retired sidecar prefix — asserted ABSENT (or bridged away) on disk. */ -const SIDECAR_DIR = ['_system', 'pending_embeds'] as const - -const sidecarDir = (dir: string): string => path.join(dir, ...SIDECAR_DIR) - -/** Every committed fact in the brain's log, generation-ascending. */ -async function allFacts(brain: Brainy): Promise { - const scan = ( - brain as unknown as { - scanFacts(o?: { fromGeneration?: number }): { - batches(): AsyncGenerator<{ facts: CommitFact[] }> - } | null - } - ).scanFacts({ fromGeneration: 1 }) - expect(scan, 'filesystem storage hosts a fact log').not.toBeNull() - const facts: CommitFact[] = [] - for await (const batch of scan!.batches()) facts.push(...batch.facts) - return facts -} - -/** The recovery fold, reimplemented independently: pending arms, landed - * disarms, a noun tombstone disarms (a deleted row owes no vector). */ -function foldPending(facts: CommitFact[]): Set { - const pending = new Set() - for (const fact of facts) { - for (const record of fact.records ?? []) { - if (record.type === 'embed.pending') pending.add(record.id) - else if (record.type === 'embed.landed') pending.delete(record.id) - } - for (const op of fact.ops) { - if (op.kind === 'noun' && op.record === null) pending.delete(op.id) - } - } - return pending -} - -/** Hang the embedder forever (the ack-law adversary). */ -function hangEmbedder(brain: Brainy): ReturnType { - return vi - .spyOn(brain as unknown as { embed(d: unknown): Promise }, 'embed') - .mockImplementation(() => new Promise(() => {})) -} - -/** Abandon a hung worker pass (its embed promise never resolves; production - * is covered by the worker's 60s hang guard — the test takes the white-box - * shortcut for speed, same idiom as the deferred-embedding suite). */ -function abandonHungWorker(brain: Brainy): void { - ;(brain as unknown as { _embedWorkerFlight: Promise | null })._embedWorkerFlight = null -} - -describe('deferred-embed markers in the log — the sidecar is dead', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - const trackDir = (): string => { - const dir = makeTempDir() - dirs.push(dir) - return dir - } - const track = (brain: Brainy): Brainy => { - brains.push(brain) - return brain - } - - afterEach(async () => { - vi.restoreAllMocks() - for (const b of brains.splice(0)) { - abandonHungWorker(b) - await b.close().catch(() => {}) - } - for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) - }) - - it('(a) SAME-FACT ATOMICITY: the deferred add\'s ONE commit fact carries embed.pending beside its after-image; no sidecar file exists', async () => { - const dir = trackDir() - const brain = track(await openBrain(dir)) - hangEmbedder(brain) // hold the pending state open for the scan - - const id = await brain.add({ - data: 'deferred content whose marker rides the fact', - type: NounType.Document, - deferEmbedding: true, - metadata: { pin: 'a' } - }) - expect(brain.pendingEmbedCount()).toBe(1) - - const facts = await allFacts(brain) - const carrying = facts.filter((f) => - (f.records ?? []).some((r) => r.type === 'embed.pending' && r.id === id) - ) - expect(carrying, 'exactly ONE fact carries the pending marker').toHaveLength(1) - const fact = carrying[0] - // The SAME fact (same generation, one atomic append) carries the write's - // own after-image — marker and write are inseparable by construction. - const afterImage = fact.ops.find((op) => op.kind === 'noun' && op.id === id) - expect(afterImage, 'the marker rides the write\'s own fact').toBeDefined() - expect(afterImage!.record, 'an after-image, not a tombstone').not.toBeNull() - const marker = (fact.records ?? []).find((r) => r.type === 'embed.pending' && r.id === id) - expect(marker && marker.type === 'embed.pending' && marker.enqueuedAt).toBeGreaterThan(0) - - // The sidecar is dead: nothing under the retired prefix, ever. - expect(fs.existsSync(sidecarDir(dir)), 'no sidecar directory is created').toBe(false) - }) - - it('(b) LANDING: after the barrier the log carries embed.landed (inline vector) on the landing commit\'s own fact, and a fresh fold nets zero pending', async () => { - const dir = trackDir() - const brain = track(await openBrain(dir)) - - const id = await brain.add({ - data: 'content that lands in the background', - type: NounType.Document, - deferEmbedding: true, - metadata: { pin: 'b' } - }) - await brain.awaitPendingEmbeds() - expect(brain.pendingEmbedCount()).toBe(0) - - const facts = await allFacts(brain) - const landingFacts = facts.filter((f) => - (f.records ?? []).some((r) => r.type === 'embed.landed' && r.id === id) - ) - expect(landingFacts, 'exactly ONE landing fact').toHaveLength(1) - const landed = (landingFacts[0].records ?? []).find( - (r) => r.type === 'embed.landed' && r.id === id - ) - expect(landed && landed.type === 'embed.landed' && landed.vector.length).toBeGreaterThan(0) - // The landing commit's own after-image rides the same fact — the worker's - // vector swap and its durable "pending consumed" are one atomic append. - const landingAfterImage = landingFacts[0].ops.find((op) => op.kind === 'noun' && op.id === id) - expect(landingAfterImage, 'the landed marker rides the swap\'s own fact').toBeDefined() - expect(landingAfterImage!.record).not.toBeNull() - - // A fresh fold of the WHOLE log — the exact recovery computation — nets zero. - expect(foldPending(facts).size).toBe(0) - expect(fs.existsSync(sidecarDir(dir))).toBe(false) - }) - - it('(c) CRASH RECOVERY VIA THE LOG: kill mid-defer, reopen — one pending re-armed from the fold, NO sidecar file anywhere, and the vector then lands', async () => { - const dir = trackDir() - - // Session 1: embedder hung, deferred add acked, durability flushed, then - // a crash-style abandon (RAM gone, no close, no background machinery). - const first = await openBrain(dir) - brains.push(first) - hangEmbedder(first) - const id = await first.add({ - data: 'survives the kill through the log', - type: NounType.Document, - deferEmbedding: true, - metadata: { pin: 'c' } - }) - expect(first.pendingEmbedCount()).toBe(1) - await first.flush() // the durability barrier: fact (with marker) + manifest - expect(fs.existsSync(sidecarDir(dir)), 'no sidecar before the kill').toBe(false) - await abandonAsCrashed(first) - brains.splice(brains.indexOf(first), 1) - vi.restoreAllMocks() - - // Session 2: recovery folds the log — embedder hung BEFORE init so the - // re-armed pending is observable, not raced away by the fast worker. - const second = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - persistence: { policy: 'manual' } - }) - const hang = hangEmbedder(second) - await second.init() - track(second) - expect(second.pendingEmbedCount(), 'the fold re-armed the pending').toBe(1) - expect(fs.existsSync(sidecarDir(dir)), 'recovery used the LOG, not files').toBe(false) - - // Un-hang and drain: a crash DELAYED the vector, never lost it. - hang.mockRestore() - abandonHungWorker(second) - await second.awaitPendingEmbeds() - expect(second.pendingEmbedCount()).toBe(0) - const after = await second.get(id, { includeVectors: true }) - expect(after, 'the deferred row survived the crash').toBeTruthy() - expect((after!.vector as number[]).length, 'the delayed vector landed').toBeGreaterThan(0) - expect(foldPending(await allFacts(second)).size, 'the landing is durable in the log').toBe(0) - }) - - it('(d) LEGACY BRIDGE: a pre-log sidecar marker folds in at open, migrates into the log, and the file dies — one-time and durable', async () => { - const dir = trackDir() - - // Session 1: a normal committed row (the entity the legacy marker names). - const first = await openBrain(dir) - brains.push(first) - const id = uid('legacy-defer') - await first.add({ - id, - data: 'legacy deferred content', - type: NounType.Document, - vector: vec(9), - metadata: { pin: 'd' } - }) - await first.flush() - await first.close() - brains.splice(brains.indexOf(first), 1) - - // A pre-log build's sidecar marker, hand-written exactly as the old - // writeRawObject persisted it (the filesystem adapter compresses raw - // objects by default: gzipped JSON at `.gz`). - fs.mkdirSync(sidecarDir(dir), { recursive: true }) - const sidecarFile = path.join(sidecarDir(dir), id) - fs.writeFileSync( - `${sidecarFile}.gz`, - zlib.gzipSync(JSON.stringify({ id, enqueuedAt: 1234567890 }, null, 2)) - ) - - // Session 2: the bridge fires at open. Embedder hung BEFORE init so the - // folded pending is observable. - const second = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - persistence: { policy: 'manual' } - }) - const hang = hangEmbedder(second) - await second.init() - track(second) - expect(second.pendingEmbedCount(), 'the legacy marker folded in').toBe(1) - expect(fs.existsSync(sidecarFile), 'the sidecar file was deleted').toBe(false) - expect(fs.existsSync(`${sidecarFile}.gz`), 'the compressed variant too').toBe(false) - const migrated = await allFacts(second) - expect( - migrated.some((f) => (f.records ?? []).some((r) => r.type === 'embed.pending' && r.id === id)), - 'the marker now lives IN the log' - ).toBe(true) - - // Drain: the bridged pending embeds and lands like any other. - hang.mockRestore() - abandonHungWorker(second) - await second.awaitPendingEmbeds() - expect(second.pendingEmbedCount()).toBe(0) - const facts = await allFacts(second) - expect( - facts.some((f) => (f.records ?? []).some((r) => r.type === 'embed.landed' && r.id === id)), - 'the bridged pending landed durably' - ).toBe(true) - expect(foldPending(facts).size).toBe(0) - await second.flush() - await second.close() - brains.splice(brains.indexOf(second), 1) - - // Session 3: nothing resurrects — the bridge was one-time, the clear durable. - const third = track(await openBrain(dir)) - expect(third.pendingEmbedCount(), 'no zombie pending on the next open').toBe(0) - expect(fs.existsSync(sidecarDir(dir)) && fs.readdirSync(sidecarDir(dir)).length > 0).toBe(false) - }) - - it('(e) VFS ACK LAW: writeFile acks under a forever-hung embedder while its pending marker sits durably in the log', async () => { - const dir = trackDir() - const brain = track(await openBrain(dir)) - const hang = hangEmbedder(brain) - - await brain.vfs.writeFile('/notes/today.md', '# The day\nA deferred capture.') - - // Acked with the embedder hung: content + metadata fully readable. - const content = await brain.vfs.readFile('/notes/today.md') - expect(content.toString()).toContain('A deferred capture.') - expect(brain.pendingEmbedCount()).toBeGreaterThanOrEqual(1) - - // The marker is already durable IN the log while the embedder hangs — - // the exact state a crash here would recover from. - expect(foldPending(await allFacts(brain)).size).toBeGreaterThanOrEqual(1) - expect(fs.existsSync(sidecarDir(dir))).toBe(false) - - // Un-hang, abandon the poisoned pass, drain, verify. - hang.mockRestore() - abandonHungWorker(brain) - await brain.awaitPendingEmbeds() - expect(brain.pendingEmbedCount()).toBe(0) - expect(foldPending(await allFacts(brain)).size).toBe(0) - }) -}) diff --git a/tests/integration/fact-log-contracts.test.ts b/tests/integration/fact-log-contracts.test.ts index 874504c9..eb579da9 100644 --- a/tests/integration/fact-log-contracts.test.ts +++ b/tests/integration/fact-log-contracts.test.ts @@ -4,13 +4,14 @@ * * (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt * process end (no flush, no close — reopen from disk). - * - transact(): HOLDS — the fact is fsync'd before transact returns. - * - single-op: HOLDS (was pinned `it.fails` until the ack-at-log - * destination landed): the 10.0.0 adopt-at-open fleet default flips a - * fresh brain to log authority at open, so single-op acks await the - * covering group fsync (durable-at-ack) and recovery REPLAYS intact - * facts above the manifest at the next open. The contract is now - * permanent on every path. + * - transact(): HOLDS TODAY — the fact is fsync'd before transact returns. + * - single-op: PINNED AS `it.fails` — today's group-commit batches + * DURABILITY (ack precedes the group fsync; a hard kill loses the fact + * AND the generation together, coherently — the documented Model-B + * contract, fine while the tree is authoritative). The destination + * (ack-at-log) requires group commit to become LATENCY batching: the + * ack waits for the shared fsync. When that lands, this pin flips red — + * remove `.fails` and the contract is permanent. No cliff to discover. * * (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment * rotation yields exactly its snapshot — byte-identical facts, no gaps, @@ -62,11 +63,9 @@ describe('fsync-before-ack contract (fact durability at the ack boundary)', () = expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) }) - // THE ACK-AT-LOG CONTRACT, HELD (was `.fails` until it landed): under the - // adopt-at-open fleet default this brain runs durable-at-ack from open — - // the ack waits for the covering log fsync, and the log-authority recovery - // path replays the intact fact at the next open instead of truncating it. - it('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => { + // PINNED (flips red when group commit becomes latency batching — then + // remove `.fails` and the ack-at-log contract is permanent on every path). + it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => { await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } }) const ackedHead = brain.scanFacts()!.headGeneration // Abrupt end immediately after the ack — before any flush window. diff --git a/tests/integration/fact-log-v2-cutover.test.ts b/tests/integration/fact-log-v2-cutover.test.ts deleted file mode 100644 index 6e8d9fb6..00000000 --- a/tests/integration/fact-log-v2-cutover.test.ts +++ /dev/null @@ -1,397 +0,0 @@ -/** - * @module tests/integration/fact-log-v2-cutover - * @description The fact log's LIVE WRITE FORMAT cutover to v2, end-to-end - * through real brains: (a) a NEW brain's tail segment carries a v2 header - * (formatVersion 2, sealSize 4096), opens with the log.genesis record - * (id-space width 64 + the manifest-persisted brainId), and scanFacts yields - * the same CommitFact shape a v1 brain would — reconstruction included, - * proven by digest-equality against canonical after a reopen; (b) MIXED - * logs: an existing v1 segment stays readable forever beside a v2 tail - * (cutover-by-rotation; the v1 segment is never rewritten); (c) MINT: - * after-image records carry the metadata index id mapper's exact int - * assignments (white-box compare); (d) SEALS: every flush leaves the tail - * sector-aligned, and pads are invisible to scans; (e) REPLAY: the - * log-authority recovery path resurrects an acked write from a v2 tail - * after a crash-style abandon. - */ -import { describe, it, expect, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as path from 'node:path' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { - parseSegmentHeader, - decodeGroupV2, - SEGMENT_HEADER_BYTES, - FACT_LOG_FORMAT_V1, - FACT_LOG_FORMAT_V2, - type LogGenesisRecord, - type NounAfterImageRecord -} from '../../src/db/factLogFormat.js' -import type { CommitFact, FactIntMinter, FactLog } from '../../src/db/factLog.js' -import { - makeTempDir, - openBrain, - storeOf, - abandonAsCrashed, - factGenerations, - vec, - uid -} from '../helpers/durabilityKillMatrix.js' - -/** The VFS root — created at init by a baseline (generation-less) write. */ -const VFS_ROOT = '00000000-0000-0000-0000-000000000000' -const FACTS_DIR = ['_generations', 'facts'] as const -const MANIFEST_PATH = '_generations/facts/manifest.json' - -/** White-box internals this suite instruments. */ -type BrainInternals = { - storage: { - readRawObject(p: string): Promise - readNounRaw(id: string): Promise<{ metadata: unknown | null; vector: unknown | null }> - } - metadataIndex: { - getIdMapper(): { getInt(uuid: string): number | undefined } - } -} -const internals = (brain: Brainy): BrainInternals => brain as unknown as BrainInternals - -/** The facts manifest as stored (additive brainId included). */ -interface StoredFactsManifest { - segments: Array<{ file: string }> - tailSegment: string | null - brainId?: string -} - -async function readManifest(brain: Brainy): Promise { - const manifest = (await internals(brain).storage.readRawObject( - MANIFEST_PATH - )) as StoredFactsManifest | null - expect(manifest, 'the facts manifest exists').toBeTruthy() - return manifest! -} - -/** Raw on-disk bytes of one fact segment file. */ -function segmentBytes(dir: string, file: string): Uint8Array { - return new Uint8Array(fs.readFileSync(path.join(dir, ...FACTS_DIR, file))) -} - -async function allFacts(brain: Brainy): Promise { - const scan = (brain as unknown as { scanFacts(): { batches(): AsyncGenerator<{ facts: CommitFact[] }> } | null }).scanFacts() - expect(scan, 'this storage hosts a fact log').not.toBeNull() - const facts: CommitFact[] = [] - for await (const batch of scan!.batches()) facts.push(...batch.facts) - return facts -} - -/** The live FactLog instance (white-box: the minter strip in scenario b). */ -function factLogOf(brain: Brainy): FactLog & { intMinter: FactIntMinter | null } { - const log = storeOf(brain).getFactLog() - expect(log, 'filesystem storage hosts a fact log').not.toBeNull() - return log as FactLog & { intMinter: FactIntMinter | null } -} - -describe('fact log v2 cutover — live writes land in the v2 segment format', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - const trackDir = (): string => { - const dir = makeTempDir() - dirs.push(dir) - return dir - } - const track = (brain: Brainy): Brainy => { - brains.push(brain) - return brain - } - - afterEach(async () => { - for (const b of brains.splice(0)) { - await (b as unknown as { close?: () => Promise }).close?.().catch(() => {}) - } - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) - }) - - it('(a) NEW BRAIN: v2 tail header, genesis-first, and scanFacts parity with canonical across a reopen', async () => { - const dir = trackDir() - const brain = track(await openBrain(dir)) - const idA = uid('v2-new-a') - const idB = uid('v2-new-b') - await brain.add({ id: idA, data: 'alpha', type: NounType.Document, vector: vec(1), metadata: { n: 1 } }) - await brain.add({ id: idB, data: 'beta', type: NounType.Document, vector: vec(2), metadata: { n: 2 } }) - await brain.flush() - - // The tail segment's raw header bytes: formatVersion 2, sealSize 4096. - const manifest = await readManifest(brain) - expect(manifest.tailSegment).toBeTruthy() - expect(manifest.brainId, 'the brain id was minted into the manifest').toBeTruthy() - const bytes = segmentBytes(dir, manifest.tailSegment!) - const header = parseSegmentHeader(bytes.subarray(0, SEGMENT_HEADER_BYTES)) - expect(header.formatVersion).toBe(FACT_LOG_FORMAT_V2) - expect(header.sealSize).toBe(4096) - - // Genesis is the FIRST record of the FIRST fact — and appears exactly once. - const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 }) - expect(group.facts.length).toBeGreaterThanOrEqual(2) - const firstRecord = group.facts[0].records[0] - expect(firstRecord.type).toBe('log.genesis') - const genesis = firstRecord as LogGenesisRecord - expect(genesis.idSpaceWidth).toBe(64) - expect(genesis.brainId).toBe(manifest.brainId) - const genesisCount = group.facts - .flatMap((f) => f.records) - .filter((r) => r.type === 'log.genesis').length - expect(genesisCount).toBe(1) - - // Shape parity + reconstruction fidelity: REOPEN (so the tail decodes - // from disk, not from the in-session originals) and compare each add's - // CommitFact op against canonical byte truth — metadata leg (bigint - // timestamps normalized back to numbers) AND the reconstructed vector - // wrapper must equal what readNounRaw returns, exactly as a v1 log's - // byte-faithful capture would. - await (brain as unknown as { close: () => Promise }).close() - brains.splice(brains.indexOf(brain), 1) - const reopened = track(await openBrain(dir)) - const facts = await allFacts(reopened) - const gens = facts.map((f) => f.generation) - expect([...gens].sort((a, b) => a - b)).toEqual(gens) - expect(new Set(gens).size).toBe(gens.length) - - const logGens = new Set( - ((await (reopened as unknown as { transactionLog(): Promise> }).transactionLog()) ?? []).map( - (e) => e.generation - ) - ) - for (const g of gens) expect(logGens.has(g), `generation ${g} is a real commit`).toBe(true) - - for (const id of [idA, idB]) { - const fact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null)) - expect(fact, `the add fact for ${id} survives the reopen`).toBeDefined() - const op = fact!.ops.find((o) => o.id === id)! - expect(op.kind).toBe('noun') - const canonical = await internals(reopened).storage.readNounRaw(id) - expect(op.record!.metadata).toStrictEqual(canonical.metadata) - // ENTITY TRUTH comparison: canonical wrappers denormalize HNSW residue - // (connections + the randomly-assigned level) that the log record - // deliberately reconstructs empty — strip both sides (the oracle's - // normalizer law) so a nonzero random level can't fake a divergence. - const strip = (w: unknown) => { - const { connections: _c, level: _l, ...rest } = w as Record - return rest - } - expect(strip(op.record!.vector)).toStrictEqual(strip(canonical.vector)) - } - }) - - it('(b) MIXED LOG: an existing v1 segment stays readable forever beside the v2 tail (cutover by rotation, v1 bytes untouched)', async () => { - // ROUTE: a REAL v1 segment is written by the v1 writer itself — the live - // FactLog with its minter stripped (the exact pre-cutover code path, - // still shipped for minter-less configurations) — then the minter is - // restored mid-session and the next append performs the cutover - // rotation. Stronger than hand-crafted bytes: both formats come from - // their real writers, on one log. - const dir = trackDir() - const brain = track(await openBrain(dir)) - const log = factLogOf(brain) - const minter = log.intMinter - expect(minter, 'the brain wired the int minter at init').toBeTruthy() - - log.intMinter = null // the pre-cutover writer - const idOld1 = uid('v1-old-1') - const idOld2 = uid('v1-old-2') - await brain.add({ id: idOld1, data: 'old one', type: NounType.Document, vector: vec(3), metadata: { era: 'v1' } }) - await brain.add({ id: idOld2, data: 'old two', type: NounType.Document, vector: vec(4), metadata: { era: 'v1' } }) - await brain.flush() - - const before = await readManifest(brain) - expect(before.segments).toHaveLength(0) - const v1TailFile = before.tailSegment! - const v1Bytes = segmentBytes(dir, v1TailFile) - expect(parseSegmentHeader(v1Bytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe( - FACT_LOG_FORMAT_V1 - ) - - log.intMinter = minter // the cutover lands mid-session - const idNew = uid('v2-new') - await brain.add({ id: idNew, data: 'new era', type: NounType.Document, vector: vec(5), metadata: { era: 'v2' } }) - await brain.flush() - - // The v1 tail was SEALED (bytes untouched), the new tail is v2. - const after = await readManifest(brain) - expect(after.segments.map((s) => s.file)).toContain(v1TailFile) - expect(after.tailSegment).not.toBe(v1TailFile) - const sealedBytes = segmentBytes(dir, v1TailFile) - expect(parseSegmentHeader(sealedBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe( - FACT_LOG_FORMAT_V1 - ) - expect( - Buffer.compare(Buffer.from(sealedBytes), Buffer.from(v1Bytes)), - 'the sealed v1 segment is byte-identical — never rewritten' - ).toBe(0) - const tailBytes = segmentBytes(dir, after.tailSegment!) - expect(parseSegmentHeader(tailBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe( - FACT_LOG_FORMAT_V2 - ) - // NOT a brand-new log: no genesis on a rotated-in v2 tail. - const tailGroup = decodeGroupV2(tailBytes.subarray(SEGMENT_HEADER_BYTES), { - expectedIdSpaceWidth: 64 - }) - expect( - tailGroup.facts.flatMap((f) => f.records).some((r) => r.type === 'log.genesis') - ).toBe(false) - - // One scan spans both formats, shape-identically, in generation order. - const liveFacts = await allFacts(brain) - const liveGens = liveFacts.map((f) => f.generation) - expect([...liveGens].sort((a, b) => a - b)).toEqual(liveGens) - for (const id of [idOld1, idOld2, idNew]) { - const fact = liveFacts.find((f) => f.ops.some((op) => op.id === id)) - expect(fact, `fact for ${id} is scannable`).toBeDefined() - const op = fact!.ops.find((o) => o.id === id)! - expect(op.kind).toBe('noun') - expect(op.record).not.toBeNull() - } - - // The MIXED log survives a reopen and keeps appending (v2 tail). - await (brain as unknown as { close: () => Promise }).close() - brains.splice(brains.indexOf(brain), 1) - const reopened = track(await openBrain(dir)) - const reFacts = await allFacts(reopened) - expect(reFacts.map((f) => f.generation)).toEqual(liveGens) - // The v1 fact still reads exactly as the v1 decoder always read it. - // (Not compared byte-strict against canonical: the v1 CAPTURE has a - // known pre-existing wart — write-cache-warm objects carry - // undefined-valued engine keys that msgpack preserves as nil while the - // durable JSON drops them. v1 bytes are frozen; the v2 encoder - // sanitizes to durable truth instead — pinned in scenario (a).) - const oldOp = reFacts - .find((f) => f.ops.some((op) => op.id === idOld1))! - .ops.find((o) => o.id === idOld1)! - const canonicalOld = await internals(reopened).storage.readNounRaw(idOld1) - const oldMeta = oldOp.record!.metadata as Record - expect(oldMeta.noun).toBe('document') - expect((oldMeta.metadata as Record).era).toBe('v1') - const oldWrapper = oldOp.record!.vector as { id: string; vector: number[] } - const canonicalWrapper = canonicalOld.vector as { id: string; vector: number[] } - expect(oldWrapper.id).toBe(idOld1) - expect(oldWrapper.vector).toStrictEqual(canonicalWrapper.vector) - await reopened.add({ id: uid('post-reopen'), data: 'still writing', type: NounType.Document, vector: vec(6), metadata: {} }) - expect((await factGenerations(reopened)).length).toBe(liveGens.length + 1) - }) - - it('(c) MINT-AT-APPEND: after-image records carry the id mapper\'s EXACT int assignments — distinct, nonzero, reproducible', async () => { - const dir = trackDir() - const brain = track(await openBrain(dir)) - const idA = uid('mint-a') - const idB = uid('mint-b') - await brain.add({ id: idA, data: 'mint one', type: NounType.Document, vector: vec(7), metadata: { m: 1 } }) - await brain.add({ id: idB, data: 'mint two', type: NounType.Document, vector: vec(8), metadata: { m: 2 } }) - await brain.flush() - - const manifest = await readManifest(brain) - const bytes = segmentBytes(dir, manifest.tailSegment!) - const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 }) - const afterImages = new Map() - for (const fact of group.facts) { - for (const record of fact.records) { - if (record.type === 'noun.afterImage') afterImages.set(record.id, record) - } - } - const recA = afterImages.get(idA) - const recB = afterImages.get(idB) - expect(recA, 'idA has a decoded after-image').toBeDefined() - expect(recB, 'idB has a decoded after-image').toBeDefined() - expect(recA!.entityInt).toBeGreaterThan(0n) - expect(recB!.entityInt).toBeGreaterThan(0n) - expect(recA!.entityInt).not.toBe(recB!.entityInt) - - // White-box: the ints on the wire ARE the metadata index mapper's - // assignments — the exact ints a mapper rebuild must reproduce. - const mapper = internals(brain).metadataIndex.getIdMapper() - expect(recA!.entityInt).toBe(BigInt(mapper.getInt(idA)!)) - expect(recB!.entityInt).toBe(BigInt(mapper.getInt(idB)!)) - }) - - it('(d) SEALS AT SYNC: every flush leaves the tail sector-aligned; pads are invisible to scans', async () => { - const dir = trackDir() - const brain = track(await openBrain(dir)) - await brain.add({ id: uid('seal-1'), data: 'one', type: NounType.Document, vector: vec(10), metadata: {} }) - await brain.flush() - - const manifest = await readManifest(brain) - const tailPath = path.join(dir, ...FACTS_DIR, manifest.tailSegment!) - const sizeAfterFirstFlush = fs.statSync(tailPath).size - expect(sizeAfterFirstFlush).toBeGreaterThan(0) - expect(sizeAfterFirstFlush % 4096, 'tail is sector-aligned after flush').toBe(0) - const countAfterFirstFlush = (await factGenerations(brain)).length - - for (let i = 0; i < 3; i++) { - await brain.add({ id: uid(`seal-more-${i}`), data: `more ${i}`, type: NounType.Document, vector: vec(11 + i), metadata: { i } }) - } - await brain.flush() - const sizeAfterSecondFlush = fs.statSync(tailPath).size - expect(sizeAfterSecondFlush).toBeGreaterThan(sizeAfterFirstFlush) - expect(sizeAfterSecondFlush % 4096, 'still aligned after more writes + flush').toBe(0) - - // Pads count toward bytes, never toward facts. - expect((await factGenerations(brain)).length).toBe(countAfterFirstFlush + 3) - }) - - it('(e) REPLAY COMPAT: the log-authority recovery path resurrects an acked write from a v2 tail after a crash-style abandon', async () => { - // The flip idiom from the log-authority suite: seed writes, baseline - // backfill LAST (the init-time VFS root never got a fact), flush, then - // the sanctioned guarded flip — the oracle goes green over an ALL-V2 - // log, which is itself the reproduction proof for the v2 record path. - const dir = mkdtempSync(join(tmpdir(), 'brainy-v2-cutover-')) - dirs.push(dir) - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - const open = async (): Promise => { - const b = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - silent: true, - dimensions: 384 - }) - await b.init() - return track(b) - } - - const brain = await open() - const kept = await brain.add({ data: 'alpha document', type: 'document', metadata: { n: 1 } }) - const removed = await brain.add({ data: 'beta document', type: 'document', metadata: { n: 2 } }) - await brain.update({ id: kept, metadata: { n: 10 } }) - await brain.remove(removed) - const root = await brain.get(VFS_ROOT) - expect(root, 'the VFS root exists').toBeTruthy() - await brain.update({ id: VFS_ROOT, metadata: root!.metadata }) // baseline backfill — final write - await brain.flush() - - const report = await (brain as unknown as { adoptLogAuthority(): Promise<{ verdict: string }> }).adoptLogAuthority() - expect(report.verdict, 'the oracle is green over a pure-v2 log').toBe('green') - - // An at-ack write: its v2 fact is fsynced (sector-sealed) at ack. - const survivor = await brain.add({ - data: 'survives power loss', - type: 'document', - metadata: { s: 1 } - }) - - // Crash-style abandon: RAM state gone, no flush, no close. - await abandonAsCrashed(brain) - - // Reopen: open() finds the acked fact ABOVE the manifest watermark in - // the v2 tail (peekFactsAbove → v2 decode) and REPLAYS it into - // canonical — an acked write is never lost. - const reopened = await open() - expect( - (reopened as unknown as { logAuthority(): { authority: string } }).logAuthority().authority - ).toBe('log') - const resurrected = await reopened.get(survivor) - expect(resurrected, 'the acked write survived the crash').toBeTruthy() - expect((resurrected as { metadata?: { s?: number } }).metadata?.s).toBe(1) - expect((await factGenerations(reopened)).length).toBeGreaterThan(0) - }) -}) diff --git a/tests/integration/find-matchall-cold.test.ts b/tests/integration/find-matchall-cold.test.ts deleted file mode 100644 index 158cb163..00000000 --- a/tests/integration/find-matchall-cold.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -/** - * @module tests/integration/find-matchall-cold - * @description THE MATCH-ALL SILENT-EMPTY PIN: `find({ where: {} })` is a - * match-all query — zero predicates constrain nothing — yet it used to route - * through the index-filter branch, where `getIdsForFilter({})` answers `[]` - * by contract. Result: 0 rows while storage held rows (worst on a freshly - * reopened brain, where it masqueraded as data loss), the forbidden answer - * class — a silent empty instead of served-or-refused. These tests pin the - * law: an empty `where` routes exactly like an absent `where`, serving from - * truth-complete sources (a storage page bounded to the offset+limit window, - * or the column store's top-K sort under orderBy) — warm AND cold, on the - * live brain, the Db pin path, pagination.count, streaming.entities, and the - * semantic path (`{ query, where: {} }` must not short-circuit to `[]`). - * The one deliberate refusal: `removeMany({ where: {} })` throws — a - * match-all BULK DELETE must be asked for explicitly, never inherited. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] - -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -async function open(dir: string): Promise { - const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await b.init() - brains.push(b) - return b -} - -/** Seed three plain documents with a sortable numeric field. */ -async function seed(brain: Brainy): Promise { - const ids: string[] = [] - ids.push(await brain.add({ data: 'alpha row', type: NounType.Document, metadata: { n: 1 } })) - ids.push(await brain.add({ data: 'beta row', type: NounType.Document, metadata: { n: 2 } })) - ids.push(await brain.add({ data: 'gamma row', type: NounType.Document, metadata: { n: 3 } })) - await brain.flush() - return ids -} - -describe('find({ where: {} }) — match-all serves, warm and cold', () => { - it('the repro: a freshly reopened filesystem brain serves match-all (not a silent 0)', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-cold-')) - dirs.push(dir) - const brain = await open(dir) - await seed(brain) - await brain.close() - brains.pop() - - const reopened = await open(dir) - const rows = await reopened.find({ where: {}, limit: 10 }) - expect(rows.length, 'match-all serves every stored row on the cold brain').toBe(3) - - // The predicate paths that always worked cold stay working — same brain. - expect((await reopened.find({ where: { n: 1 }, limit: 10 })).length).toBe(1) - expect((await reopened.find({ where: { 'system.type': 'document' }, limit: 10 })).length).toBe(3) - }, 120000) - - it('match-all + orderBy on a metadata field serves sorted after reopen', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-order-')) - dirs.push(dir) - const brain = await open(dir) - await seed(brain) - await brain.close() - brains.pop() - - const reopened = await open(dir) - const rows = await reopened.find({ where: {}, orderBy: 'n', order: 'desc', limit: 10 }) - expect(rows.length, 'sorted match-all serves every stored row cold').toBe(3) - expect( - rows.map((r) => (r.metadata as { n: number }).n), - 'orderBy is honored on the cold match-all page' - ).toEqual([3, 2, 1]) - }, 120000) - - it('warm brain unchanged: match-all, sorted match-all, and predicates all serve in-session', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-warm-')) - dirs.push(dir) - const brain = await open(dir) - await seed(brain) - - expect((await brain.find({ where: {}, limit: 10 })).length).toBe(3) - const sorted = await brain.find({ where: {}, orderBy: 'n', order: 'asc', limit: 2 }) - expect(sorted.map((r) => (r.metadata as { n: number }).n)).toEqual([1, 2]) - expect((await brain.find({ where: { n: 2 }, limit: 10 })).length).toBe(1) - // Pagination window respected: match-all never over-serves the page. - expect((await brain.find({ where: {}, limit: 2, offset: 2 })).length).toBe(1) - }, 120000) - - it('the semantic path: find({ query, where: {} }) must not short-circuit to []', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-query-')) - dirs.push(dir) - const brain = await open(dir) - await seed(brain) - await brain.close() - brains.pop() - - const reopened = await open(dir) - // Before the fix, the pre-resolved empty filter matched nothing and the - // vector search was skipped entirely — a silent [] for every such query. - const rows = await reopened.find({ query: 'alpha row', where: {}, limit: 10 }) - expect(rows.length, 'an unconstraining where must not empty a semantic query').toBeGreaterThan(0) - }, 120000) - - it('the Db pin path: asOf(g).find({ where: {} }) serves at the pinned generation after reopen', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-asof-')) - dirs.push(dir) - const brain = await open(dir) - await brain.add({ data: 'first', type: NounType.Document, metadata: { n: 1 } }) - await brain.add({ data: 'second', type: NounType.Document, metadata: { n: 2 } }) - await brain.flush() - const gTwo = brain.generation() - await brain.add({ data: 'third', type: NounType.Document, metadata: { n: 3 } }) - await brain.flush() - await brain.close() - brains.pop() - - const reopened = await open(dir) - // Current-generation pin (delegates to the live find fast path). - const now = reopened.now() - expect((await now.find({ where: {}, limit: 10 })).length).toBe(3) - - // Historical pin: the record-overlay path must serve match-all too. - const past = await reopened.asOf(gTwo) - try { - const rows = await past.find({ where: {}, limit: 10 }) - expect(rows.length, 'match-all at the pinned generation sees exactly the rows of that generation').toBe(2) - } finally { - await past.release() - } - }, 120000) - - it('pagination.count({ where: {} }) counts every row instead of a silent 0', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-count-')) - dirs.push(dir) - const brain = await open(dir) - await seed(brain) - await brain.close() - brains.pop() - - const reopened = await open(dir) - // The law: an empty where counts exactly like an absent where (the - // unfiltered total — which by long-standing count semantics includes - // system entities such as the VFS root, hence >= the 3 user rows). - const emptyWhere = await reopened.pagination.count({ where: {} }) - expect(emptyWhere).toBe(await reopened.pagination.count({})) - expect(emptyWhere).toBeGreaterThanOrEqual(3) - }, 120000) - - it('streaming.entities({ where: {} }) streams every row instead of nothing', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-stream-')) - dirs.push(dir) - const brain = await open(dir) - await seed(brain) - await brain.close() - brains.pop() - - const reopened = await open(dir) - const streamed: string[] = [] - for await (const entity of reopened.streaming.entities({ where: {} })) { - streamed.push(entity.id) - } - expect(streamed.length, 'an unconstraining where streams the full store').toBeGreaterThanOrEqual(3) - }, 120000) - - it('removeMany({ where: {} }) refuses loudly — match-all bulk delete is never implicit', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-matchall-remove-')) - dirs.push(dir) - const brain = await open(dir) - await seed(brain) - - await expect(brain.removeMany({ where: {} })).rejects.toThrow(/matches EVERYTHING/) - // Nothing was deleted by the refused call. - expect((await brain.find({ where: {}, limit: 10 })).length).toBe(3) - }, 120000) -}) diff --git a/tests/integration/log-authority-adopt.test.ts b/tests/integration/log-authority-adopt.test.ts deleted file mode 100644 index 5e810b1f..00000000 --- a/tests/integration/log-authority-adopt.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * @module tests/integration/log-authority-adopt - * @description THE SANCTIONED FLIP, END TO END: adoptLogAuthority() cures - * its own curable divergences by baseline backfill — a FRESH brain (whose - * generation-0 VFS root never entered the log) flips WITHOUT any manual - * white-box backfill. Before this, no fresh brain could ever flip: the - * oracle reported the bootstrap row as pre-log-record and the flip refused. - * Log-AHEAD divergences stay incurable and refuse loudly (witness wins). - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] - -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -async function open(dir: string, logAuthority?: 'adopt' | 'defer'): Promise { - const b = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - ...(logAuthority ? { logAuthority } : {}) - }) - await b.init() - brains.push(b) - return b -} - -describe('adoptLogAuthority — the sanctioned flip with self-backfill', () => { - it('a fresh brain flips directly: the backfill cures the generation-0 baseline', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-adopt-')) - dirs.push(dir) - const brain = await open(dir) - const idA = await brain.add({ data: 'first row', type: NounType.Document, metadata: { n: 1 } }) - await brain.add({ data: 'second row', type: NounType.Document, metadata: { n: 2 } }) - await brain.flush() - - const report = await brain.adoptLogAuthority() - expect(report.verdict, 'the flip receipt is a green oracle').toBe('green') - expect(brain.logAuthority().authority).toBe('log') - - // The switch survives reopen; the brain keeps serving identically. - await brain.close() - brains.pop() - const reopened = await open(dir) - expect(reopened.logAuthority().authority).toBe('log') - expect(await reopened.get(idA), 'records serve at reopen').toBeTruthy() - const rows = await reopened.find({ where: {}, limit: 10 }) - expect(rows.length, 'match-all serves on the reopened flipped brain').toBeGreaterThanOrEqual(2) - // And a fresh oracle run on the flipped brain stays green. - expect((await reopened.verifyLogAuthority()).verdict).toBe('green') - }, 120000) - - it('witness drift (out-of-generation canonical rewrite) is cured by the backfill, then flips', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-adopt-drift-')) - dirs.push(dir) - const brain = await open(dir) - const id = await brain.add({ data: 'drifter', type: NounType.Document, metadata: { v: 1 } }) - await brain.flush() - - // Simulate maintenance rewriting canonical OUTSIDE a generation (the - // witness-drift class): mutate the stored record directly. - const storage = (brain as unknown as { - storage: { - readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }> - writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise - } - }).storage - const raw = await storage.readNounRaw(id) - await storage.writeNounRaw(id, { - metadata: { ...(raw.metadata as Record), drifted: true }, - vector: raw.vector - }) - expect((await brain.verifyLogAuthority()).verdict, 'drift detected').toBe('red') - - const report = await brain.adoptLogAuthority() - expect(report.verdict).toBe('green') - expect(brain.logAuthority().authority).toBe('log') - }, 120000) - - // THE OPT-OUT CONTRACT (`logAuthority: 'defer'`): no automatic adoption — - // the fresh brain stays tree-authoritative and writes NO artifact (a - // deferred posture is config, not stored state); the EXPLICIT - // adoptLogAuthority() then flips it exactly as before the fleet default. - it("opt-out: 'defer' stays tree with no artifact until the explicit adoptLogAuthority() flips it", async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-adopt-defer-')) - dirs.push(dir) - const brain = await open(dir, 'defer') - await brain.add({ data: 'deferred row', type: NounType.Document, metadata: { n: 1 } }) - await brain.flush() - - expect(brain.logAuthority().authority, "'defer' skips open-time adoption").toBe('tree') - const storage = (brain as unknown as { - storage: { readRawObject(p: string): Promise } - }).storage - const artifact = await storage.readRawObject('_system/log-authority.json').catch(() => null) - expect(artifact, "'defer' writes no authority artifact").toBeNull() - - const report = await brain.adoptLogAuthority() - expect(report.verdict, 'the explicit flip still lands on green').toBe('green') - expect(brain.logAuthority().authority).toBe('log') - const stored = (await storage.readRawObject('_system/log-authority.json')) as { - authority?: string - } | null - expect(stored?.authority, 'the explicit flip stores the artifact').toBe('log') - }, 120000) -}) diff --git a/tests/integration/log-authority.test.ts b/tests/integration/log-authority.test.ts deleted file mode 100644 index a828c9a3..00000000 --- a/tests/integration/log-authority.test.ts +++ /dev/null @@ -1,399 +0,0 @@ -/** - * @module tests/integration/log-authority - * @description The guarded log-authority core, end-to-end: the per-brain - * authority switch (stored artifact, checked at open only), the - * verification oracle (replay the fact log, diff latest per-id state - * against the canonical tree, NAME every divergence by class), the guarded - * flip (refuses on red with the cure in the message; lands on green and - * engages durable-at-ack immediately), and the switch surviving reopen. - * - * THE 10.0.0 FLEET DEFAULT is ADOPT-AT-OPEN (`logAuthority: 'adopt'`): a - * fresh brain with no stored artifact runs the oracle at open, backfills - * curable divergences, and flips to log authority on green — so a - * default-config brain opens ALREADY log-authoritative and durable-at-ack. - * The first two pins hold that default and its explicit opt-out - * (`logAuthority: 'defer'`, the pre-10 tree behavior). Every test below - * them that exercises the ORACLE or the EXPLICIT flip opens its brain with - * `'defer'` — otherwise the open-time adoption would have pre-flipped the - * brain and pre-cured the very divergences under test. - * - * KNOWN GAPS PINNED WITH `.fails` (real findings, not test bugs — see the - * comments on each): a fresh brain is NOT log-complete by construction - * today, because the VFS root is written at init as a baseline - * (generation-less) write that never gets a fact, so the oracle reports it - * as a `pre-log-record`. The open-time adoption (and adoptLogAuthority()) - * CURES this by baseline backfill — a re-commit, not construction — so the - * by-construction pin stays `.fails` on a deferred brain. Tests that need - * a green oracle on a deferred brain perform that backfill explicitly (an - * identity update of the root as the FINAL write — final, because - * derived-index maintenance rewrites canonical noun records outside - * generations, so an earlier fact's after-image goes stale; see the module - * tail comment on `backfillBaseline`). - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import type { OracleReport } from '../../src/db/logAuthority.js' - -/** The VFS root — created at init by a baseline (generation-less) write. */ -const VFS_ROOT = '00000000-0000-0000-0000-000000000000' -const AUTHORITY_ARTIFACT = '_system/log-authority.json' - -/** White-box view of the internals this suite instruments (read-only spies - * plus the sanctioned direct-storage writes for aging/drifting a brain). */ -type BrainInternals = { - generationStore: { - getFactLog(): { ensureSynced(): Promise } | null - logDurability: 'deferred' | 'at-ack' - } - storage: { - readRawObject(path: string): Promise - saveNoun(n: unknown): Promise - saveNounMetadata(id: string, m: Record): Promise - getNounMetadata(id: string): Promise | null> - writeNounRaw(id: string, r: { metadata: null; vector: null }): Promise - } -} - -const internals = (brain: Brainy): BrainInternals => - brain as unknown as BrainInternals - -/** Count calls to the fact log's ensureSynced without changing behavior. */ -function spyEnsureSynced(brain: Brainy): { calls: () => number } { - const factLog = internals(brain).generationStore.getFactLog() - expect(factLog, 'filesystem storage hosts a fact log').not.toBeNull() - let calls = 0 - const original = factLog!.ensureSynced.bind(factLog) - factLog!.ensureSynced = async () => { - calls++ - return original() - } - return { calls: () => calls } -} - -/** - * The minimal baseline backfill: an identity update of the VFS root, so the - * one canonical record the log never saw (the init-time baseline write) gets - * a fact carrying its current state. MUST be the final write of the setup — - * derived-index maintenance (HNSW/enumeration denormalization) rewrites the - * root's canonical noun record outside any generation, so a root fact taken - * before later writes digests stale and reports `state-differs`. - */ -async function backfillBaseline(brain: Brainy): Promise { - const root = await brain.get(VFS_ROOT) - expect(root, 'the VFS root exists on a fresh brain').toBeTruthy() - await brain.update({ id: VFS_ROOT, metadata: root!.metadata }) -} - -/** Seed a brain with the standard write mix: 2 adds, an update, a remove. */ -async function seedWrites(brain: Brainy): Promise<{ kept: string; removed: string }> { - const kept = await brain.add({ data: 'alpha document', type: 'document', metadata: { n: 1 } }) - const removed = await brain.add({ data: 'beta document', type: 'document', metadata: { n: 2 } }) - await brain.update({ id: kept, metadata: { n: 10 } }) - await brain.remove(removed) - return { kept, removed } -} - -describe('log authority — the switch, the oracle, the guarded flip', () => { - const dirs: string[] = [] - const brains: Brainy[] = [] - - /** - * Open a brain over `dir`. Omit `logAuthority` to exercise the FLEET - * DEFAULT (adopt-at-open); pass `'defer'` for the tests that need a - * tree-authoritative brain so the oracle/explicit-flip path is actually - * the thing under test (the default would pre-flip and pre-backfill). - */ - const openBrain = async ( - dir?: string, - logAuthority?: 'adopt' | 'defer' - ): Promise<{ brain: Brainy; dir: string }> => { - const d = dir ?? mkdtempSync(join(tmpdir(), 'brainy-log-authority-')) - if (!dir) dirs.push(d) - const brain = new Brainy({ - storage: { type: 'filesystem', path: d }, - requireSubtype: false, - silent: true, - dimensions: 384, - ...(logAuthority ? { logAuthority } : {}) - }) - brains.push(brain) - await brain.init() - return { brain, dir: d } - } - - afterEach(async () => { - for (const b of brains.splice(0)) { - await (b as unknown as { close?: () => Promise }).close?.().catch(() => {}) - } - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) - }) - - // THE RULED DEFAULT (10.0.0): with no config and no stored artifact, a - // fresh brain ADOPTS log authority at open — oracle green (the open-time - // baseline backfill cures the generation-0 VFS root), artifact on disk, - // durable-at-ack live from the first write. - it('DEFAULT IS ADOPT-AT-OPEN: a fresh brain opens already log-authoritative — artifact stored, plain acks await the covering log fsync', async () => { - const { brain } = await openBrain() // no logAuthority config = the fleet default - - const authority = brain.logAuthority() - expect(authority.authority).toBe('log') - expect(typeof authority.flippedAt).toBe('number') - expect(authority.oracle, 'the open-time flip records its green oracle summary').toBeDefined() - - const artifact = (await internals(brain) - .storage.readRawObject(AUTHORITY_ARTIFACT) - .catch(() => null)) as { authority?: string } | null - expect(artifact, 'the adoption wrote the switch artifact').not.toBeNull() - expect(artifact!.authority).toBe('log') - - // The MODE assertion (not a timing one): in log authority a single-op - // ack awaits the log's covering-fsync path. - expect(internals(brain).generationStore.logDurability).toBe('at-ack') - const spy = spyEnsureSynced(brain) - await brain.add({ data: 'log mode write', type: 'document', metadata: { n: 1 } }) - expect(spy.calls(), 'adopted default: add() awaits the covering fsync').toBeGreaterThanOrEqual(1) - }) - - // THE EXPLICIT OPT-OUT: `logAuthority: 'defer'` is the pre-10 behavior — - // tree authority, NO artifact written (a deferred posture is config, not - // stored state), and single-op acks never await a log fsync. - it("OPT-OUT ('defer'): the brain stays tree-authoritative, stores no artifact, and plain acks never await a log fsync", async () => { - const { brain } = await openBrain(undefined, 'defer') - - expect(brain.logAuthority().authority).toBe('tree') - expect(brain.logAuthority().flippedAt).toBeUndefined() - - const artifact = await internals(brain) - .storage.readRawObject(AUTHORITY_ARTIFACT) - .catch(() => null) - expect(artifact, "'defer' writes no switch artifact").toBeNull() - - // The MODE assertion (not a timing one): in tree authority a single-op - // ack must never call the log's covering-fsync path. - const spy = spyEnsureSynced(brain) - await brain.add({ data: 'tree mode write', type: 'document', metadata: { n: 1 } }) - expect(spy.calls(), 'tree mode: add() does not call ensureSynced').toBe(0) - expect(internals(brain).generationStore.logDurability).toBe('deferred') - }) - - // KNOWN GAP (marked .fails — remove the marker when fixed in src): the - // intended contract is that a fresh brain is log-complete by construction, - // because every write dual-writes a fact. Today the VFS root - // (00000000-0000-0000-0000-000000000000) is created at init by a baseline - // write with NO generation and NO fact, yet it is enumerated by the - // canonical walk — so the oracle on a fresh brain is red with exactly one - // `pre-log-record` mismatch on the root. The adopt-at-open default (and - // adoptLogAuthority()) CURES this by baseline backfill — a re-commit, - // which is why this pin opens with 'defer': it holds the BY-CONSTRUCTION - // intent, which the backfill masks but does not deliver. - it.fails('ORACLE INTENT: a fresh brain is log-complete by construction — verdict green with zero mismatches', async () => { - const { brain } = await openBrain(undefined, 'defer') - await seedWrites(brain) - await brain.flush() - - const report = await brain.verifyLogAuthority() - expect(report.verdict).toBe('green') - expect(report.mismatches).toEqual([]) - }) - - it('a fresh, un-backfilled brain diverges ONLY on the init-time baseline record — every user write is exactly reproduced', async () => { - // 'defer': the adopt-at-open default would have backfilled the baseline - // already — this pin needs the brain genuinely un-backfilled. - const { brain } = await openBrain(undefined, 'defer') - await seedWrites(brain) - await brain.flush() - - const report = await brain.verifyLogAuthority() - // Tolerant pin (stays true after the baseline gap is fixed in src): - // whatever the verdict, no USER record may ever diverge — the only - // admissible mismatch is the init-time baseline root, as pre-log-record. - expect( - report.mismatches.every( - (m) => m.id === VFS_ROOT && m.reason === 'pre-log-record' && m.kind === 'noun' - ), - 'the only divergence on a fresh brain is the baseline root record' - ).toBe(true) - expect(report.matched).toBe(report.nounsChecked - report.mismatches.length) - expect(report.mismatchListTruncated).toBe(false) - }) - - it('THE ORACLE GOES GREEN on a log-complete brain: adds + update + remove, every canonical row exactly reproduced', async () => { - // 'defer' + manual backfill: the exact-count pins below (5 generations) - // depend on the log holding ONLY this test's writes — the adopt-at-open - // default would inject its own backfill generation at init. - const { brain } = await openBrain(undefined, 'defer') - await seedWrites(brain) - await backfillBaseline(brain) // final write — see the helper's contract - await brain.flush() - - const report = await brain.verifyLogAuthority() - expect(report.verdict).toBe('green') - expect(report.mismatches).toEqual([]) - expect(report.mismatchListTruncated).toBe(false) - // Live count: the kept document + the VFS root (the removed one is a - // tombstone in the log and absent from canonical — checked, not counted). - expect(report.nounsChecked).toBe(2) - expect(report.matched).toBe(2) - // 5 committed generations: add, add, update, remove, root backfill. - expect(report.generationsScanned).toBe(5) - }) - - it('THE ORACLE NAMES pre-log records: a canonical row no fact ever recorded reports pre-log-record, by id', async () => { - const { brain } = await openBrain(undefined, 'defer') - await seedWrites(brain) - await backfillBaseline(brain) - await brain.flush() - expect((await brain.verifyLogAuthority()).verdict, 'sanity: green before aging').toBe('green') - - // Simulate an aged brain: write one canonical record DIRECTLY at the - // storage layer (the write path never sees it, so no fact exists) — - // the pre-log shape: flat metadata, no _fmt stamp, 384-dim vector. - const legacyId = '00000000-0000-4000-8000-00000000a6ed' - const storage = internals(brain).storage - await storage.saveNoun({ - id: legacyId, - vector: new Array(384).fill(0.01), - connections: new Map(), - level: 0 - }) - await storage.saveNounMetadata(legacyId, { - noun: 'document', - confidence: 0.75, - createdAt: 1700000000000, - updatedAt: 1700000000000, - _rev: 1, - legacyField: 'legacy-value' - }) - - const report = await brain.verifyLogAuthority() - expect(report.verdict).toBe('red') - expect(report.mismatches).toHaveLength(1) - expect(report.mismatches[0]).toEqual({ - id: legacyId, - kind: 'noun', - reason: 'pre-log-record' - }) - }) - - it('THE FLIP REFUSES ON A LOG-AHEAD DIVERGENCE: the witness denies what the log claims — nothing written, nothing changed', async () => { - // Contract update (adoptLogAuthority's baseline backfill): curable - // divergences — pre-log records and witness drift — are re-committed - // and the flip proceeds; ONLY log-AHEAD divergences (the log claims - // state canonical denies) refuse, because no backfill can make the log - // un-claim a live row. This test stages exactly that incurable shape. - // 'defer': the brain must still be tree-authoritative (no artifact) so - // the refusal's nothing-written pins below have meaning. - const { brain } = await openBrain(undefined, 'defer') - const { kept } = await seedWrites(brain) - await backfillBaseline(brain) - await brain.flush() - - // The log says `kept` is live; its canonical record vanishes behind the - // write path's back (log-live-canonical-absent — the witness wins). - const storage = internals(brain).storage - await storage.writeNounRaw(kept, { metadata: null, vector: null }) - - let error: Error | null = null - try { - await brain.adoptLogAuthority() - } catch (err) { - error = err as Error - } - expect(error, 'the flip rejects on a log-ahead divergence').not.toBeNull() - expect(error!.message).toMatch(/witness denies/) - expect(error!.message).toMatch(/log-live-canonical-absent/) - - // Nothing changed: authority still tree, no artifact, deferred durability. - expect(brain.logAuthority().authority).toBe('tree') - const artifact = await storage.readRawObject(AUTHORITY_ARTIFACT).catch(() => null) - expect(artifact, 'a refused flip writes no artifact').toBeNull() - expect(internals(brain).generationStore.logDurability).toBe('deferred') - }) - - it('THE FLIP LANDS ON GREEN: the report is the receipt, the artifact is on disk, and durable-at-ack engages immediately', async () => { - // 'defer': this pin exercises the EXPLICIT flip — the adopt-at-open - // default would have landed it before the test began. - const { brain } = await openBrain(undefined, 'defer') - await seedWrites(brain) - await backfillBaseline(brain) - await brain.flush() - - const report: OracleReport = await brain.adoptLogAuthority() - expect(report.verdict).toBe('green') - - const authority = brain.logAuthority() - expect(authority.authority).toBe('log') - expect(typeof authority.flippedAt).toBe('number') - expect(authority.oracle).toBeDefined() - expect(authority.oracle!.nounsChecked).toBe(report.nounsChecked) - expect(authority.oracle!.generationsScanned).toBe(report.generationsScanned) - - const artifact = (await internals(brain) - .storage.readRawObject(AUTHORITY_ARTIFACT) - .catch(() => null)) as { authority?: string } | null - expect(artifact, 'the switch artifact exists on disk').not.toBeNull() - expect(artifact!.authority).toBe('log') - - // Durable-at-ack engaged in THIS session: the next single-op ack awaits - // a covering log fsync. - expect(internals(brain).generationStore.logDurability).toBe('at-ack') - const spy = spyEnsureSynced(brain) - await brain.add({ data: 'post-flip write', type: 'document', metadata: { n: 3 } }) - expect(spy.calls(), 'log mode: add() awaits the covering fsync').toBeGreaterThanOrEqual(1) - }) - - it('THE SWITCH SURVIVES REOPEN: authority restored at open with no re-verification, durable-at-ack active in the new session', async () => { - const { brain, dir } = await openBrain(undefined, 'defer') - await seedWrites(brain) - await backfillBaseline(brain) - await brain.flush() - await brain.adoptLogAuthority() - const flipReceipt = brain.logAuthority() - await (brain as unknown as { close: () => Promise }).close() - - // Reopen with 'defer' too: the restored authority below can then ONLY - // come from the stored artifact (a stored artifact always wins; had the - // default re-adopted, flippedAt/oracle would differ from the receipt). - const { brain: reopened } = await openBrain(dir, 'defer') - const restored = reopened.logAuthority() - expect(restored.authority).toBe('log') - // No re-verification happened at open: the restored record IS the stored - // flip receipt, oracle summary and timestamp intact. - expect(restored.flippedAt).toBe(flipReceipt.flippedAt) - expect(restored.oracle).toEqual(flipReceipt.oracle) - - // Mode restored at open: an ack in the new session awaits the log fsync. - expect(internals(reopened).generationStore.logDurability).toBe('at-ack') - const spy = spyEnsureSynced(reopened) - await reopened.add({ data: 'new session write', type: 'document', metadata: { n: 4 } }) - expect(spy.calls(), 'reopened log mode: add() awaits the covering fsync').toBeGreaterThanOrEqual(1) - }) - - it('STATE-DIFFERS: canonical drift the write path never saw is named, by id', async () => { - const { brain } = await openBrain(undefined, 'defer') - const { kept } = await seedWrites(brain) - await backfillBaseline(brain) - await brain.flush() - expect((await brain.verifyLogAuthority()).verdict, 'sanity: green before drift').toBe('green') - - // Drift one canonical metadata record DIRECTLY at the storage layer — - // the log never hears about it. This is the witness-drift case the - // oracle exists to catch. - const storage = internals(brain).storage - const current = await storage.getNounMetadata(kept) - expect(current, 'the seeded record has stored metadata').toBeTruthy() - await storage.saveNounMetadata(kept, { ...current!, driftedByTest: true }) - - const report = await brain.verifyLogAuthority() - expect(report.verdict).toBe('red') - expect(report.mismatches).toHaveLength(1) - expect(report.mismatches[0]).toEqual({ - id: kept, - kind: 'noun', - reason: 'state-differs' - }) - }) -}) diff --git a/tests/integration/recovery-walk-tolerance.test.ts b/tests/integration/recovery-walk-tolerance.test.ts deleted file mode 100644 index 6a37e3bd..00000000 --- a/tests/integration/recovery-walk-tolerance.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -/** - * @module tests/integration/recovery-walk-tolerance - * @description The rc6-red cures — the typed/tolerant boundary redrawn where - * block-layer fault injection proved it belonged: - * 1. WALKS ARE HEALERS: an init-time recovery/rebuild/pagination walk that - * meets a torn record narrates+counts (the adapter's loud floor) and - * HEALS PAST it — the open succeeds, remaining rows serve. rc6 died - * typed here; rc5 survived silently; the cure is loud survival. - * 2. IDENTITY READS STAY TYPED: get-by-id of the torn record itself still - * throws TornRecordError — a caller who asked for THAT record can act. - * 3. TORN MAPPER STATE (the NaN→BigInt source): a mapper file carrying - * garbage integers is discarded with narration; reopen succeeds and the - * FIRST WRITE after recovery mints sanely — never a RangeError. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync, readdirSync, writeFileSync, existsSync, statSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { gzipSync } from 'node:zlib' -import { Brainy, TornRecordError } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -async function open(dir: string): Promise { - const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await b.init() - brains.push(b) - return b -} - -/** Find one entity metadata file under entities/nouns and tear it. */ -function tearOneNounMetadata(dir: string, excludeId?: string): string { - const nounsRoot = join(dir, 'entities', 'nouns') - const walk = (d: string): string | null => { - for (const e of readdirSync(d, { withFileTypes: true })) { - const p = join(d, e.name) - if (e.isDirectory()) { - if (excludeId && e.name === excludeId) continue - const hit = walk(p) - if (hit) return hit - } else if (/^metadata\.json(\.gz)?$/.test(e.name)) { - writeFileSync(p, Buffer.from([0x1f, 0x8b, 0x00, 0xde, 0xad])) // torn gz - return p - } - } - return null - } - const torn = walk(nounsRoot) - if (!torn) throw new Error('layout probe: no noun metadata file found to tear') - // The id is the parent directory name. - return torn.split('/').slice(-2, -1)[0] -} - -describe('recovery-walk tolerance (the rc6-red cures)', () => { - it('a torn entity record does not kill the open: recovery walks heal past it, remaining rows serve, identity read throws typed', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-walk-tol-')) - dirs.push(dir) - let brain = await open(dir) - const keeper = await brain.add({ data: 'keeper row', type: NounType.Document, metadata: { k: 1 } }) - await brain.add({ data: 'victim row', type: NounType.Document, metadata: { k: 2 } }) - await brain.flush() - await brain.close() - brains.pop() - - const tornId = tearOneNounMetadata(dir, keeper) - - // THE PIN: the open succeeds (rc6 died right here), the keeper serves, - // and walks (find) heal past the victim. - brain = await open(dir) - expect((await brain.get(keeper))!.data).toContain('keeper row') - const rows = await brain.find({ where: {}, limit: 10 }) - expect(rows.map((r) => r.id)).toContain(keeper) - - // Identity read of the victim itself: typed, catchable — the caller - // asked for THAT record; under log authority the replay may have - // already HEALED it from the fact log (also a valid outcome) — accept - // healed-or-typed, never silent-absent-without-narration. - try { - const victim = await brain.get(tornId) - // Healed by replay: the record must be real (log authority rewrote it). - expect(victim).not.toBeNull() - } catch (err) { - expect(err).toBeInstanceOf(TornRecordError) - } - }, 120000) - - it('a torn mapper file (NaN ints) discards with narration; reopen succeeds and the first write mints sanely', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-torn-mapper-')) - dirs.push(dir) - let brain = await open(dir) - await brain.add({ data: 'pre-crash row', type: NounType.Document, metadata: { k: 1 } }) - await brain.flush() - await brain.close() - brains.pop() - - // The power-cut shape: the persisted mapper carries garbage integers. - const sys = join(dir, '_system') - const mapperPath = readdirSync(sys) - .filter((f) => /entityIdMapper/.test(f)) - .map((f) => join(sys, f))[0] - expect(mapperPath, 'layout probe: mapper artifact exists').toBeTruthy() - const torn = { nextId: 'NaN-garbage', uuidToInt: { x: 'junk' }, intToUuid: { junk: 42 } } - if (mapperPath.endsWith('.gz')) writeFileSync(mapperPath, gzipSync(JSON.stringify(torn))) - else writeFileSync(mapperPath, JSON.stringify(torn)) - expect(statSync(mapperPath).size).toBeGreaterThan(0) - - // Reopen MUST succeed; the first write after recovery must mint sanely - // (rc6's fresh-write RangeError shape), and graph int resolution at - // reopen must not throw (rc6's reopen shape). - brain = await open(dir) - const fresh = await brain.add({ data: 'post-recovery write', type: NounType.Document, metadata: { k: 2 } }) - expect((await brain.get(fresh))!.data).toContain('post-recovery') - await brain.flush() - expect(Number.isSafeInteger(brain.generation())).toBe(true) - }, 120000) -}) diff --git a/tests/integration/reprojection-doors-open.test.ts b/tests/integration/reprojection-doors-open.test.ts deleted file mode 100644 index 343bc536..00000000 --- a/tests/integration/reprojection-doors-open.test.ts +++ /dev/null @@ -1,257 +0,0 @@ -/** - * @module tests/integration/reprojection-doors-open - * @description The reprojection engine against a REAL brain on filesystem - * storage: a toy secondary projection (bucket counts with its own watermark - * artifact, stamp-after-data per src/utils/projectionWatermark.ts) folds the - * brain's committed facts through the engine, wired with the callback-form - * {@link FactLogSource} over `brain.scanFacts`. - * - * Proves the three doors-open rows: - * (i) folding to caught-up matches ground-truth counts; - * (ii) mid-fold, `find()` and `get()` still answer, and a door bump - * preempts the advance at the next boundary (mechanism-pinned via - * batch counts, not wall-clock); - * (iii) a crash mid-fold (abandon; reopen; re-advance) resumes from the - * durable stamp — never refolds from zero. - */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync, existsSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { - ReprojectionEngine, - type ProjectionAdapter -} from '../../src/reprojection/reprojectionEngine.js' -import { FactLogSource } from '../../src/reprojection/factLogSource.js' -import { makeProjectionStamp, readStampedWatermark } from '../../src/utils/projectionWatermark.js' -import type { CommitFact } from '../../src/db/factLog.js' - -/** 50 rows, 5 buckets, 10 each. */ -const ROWS = 50 -const BUCKETS = 5 -const GROUND_TRUTH: Record = { b0: 10, b1: 10, b2: 10, b3: 10, b4: 10 } - -/** - * The toy secondary projection: latest bucket per entity id, persisted as a - * data file plus a SEPARATE stamp artifact written stamp-after-data via the - * shared projectionWatermark helpers. Idempotent by construction (latest- - * state per id), so at-least-once redelivery on resume is harmless. - */ -class BucketCountProjection implements ProjectionAdapter { - readonly family = 'bucket-counts' - /** Every generation this INSTANCE applied — the refold detector for (iii). */ - readonly appliedGenerations: number[] = [] - private latest: Map - private wm: number | null - - private constructor( - private readonly dir: string, - wm: number | null, - latest: Map - ) { - this.wm = wm - this.latest = latest - } - - /** Load from the artifact dir — data is trusted only under a valid stamp. */ - static async open(dir: string): Promise { - mkdirSync(dir, { recursive: true }) - const stampPath = join(dir, 'stamp.json') - const dataPath = join(dir, 'data.json') - let wm: number | null = null - if (existsSync(stampPath)) { - wm = readStampedWatermark(JSON.parse(readFileSync(stampPath, 'utf8'))) - } - const latest = new Map( - wm !== null && existsSync(dataPath) - ? (JSON.parse(readFileSync(dataPath, 'utf8')) as Array<[string, string | null]>) - : [] - ) - return new BucketCountProjection(dir, wm, latest) - } - - /** Non-null bucket tallies from the latest-state map. */ - counts(): Record { - const out: Record = {} - for (const bucket of this.latest.values()) { - if (bucket !== null) out[bucket] = (out[bucket] ?? 0) + 1 - } - return out - } - - watermark(): number | null { - return this.wm - } - - async applyBatch(facts: CommitFact[], upTo: number): Promise { - for (const fact of facts) { - this.appliedGenerations.push(fact.generation) - for (const op of fact.ops) { - if (op.kind !== 'noun') continue - if (op.record === null) { - this.latest.set(op.id, null) // tombstone - continue - } - // The stored noun record nests user metadata under `.metadata`. - const stored = op.record.metadata as Record | null - const user = (stored?.metadata ?? stored) as Record | null - const bucket = typeof user?.bucket === 'string' ? user.bucket : null - this.latest.set(op.id, bucket) - } - } - // Durability THEN stamp — the projectionWatermark law. - writeFileSync(join(this.dir, 'data.json'), JSON.stringify([...this.latest])) - writeFileSync(join(this.dir, 'stamp.json'), JSON.stringify(makeProjectionStamp(upTo))) - this.wm = upTo - } - - async discard(): Promise { - rmSync(this.dir, { recursive: true, force: true }) - } -} - -describe('reprojection doors-open — a real brain, a toy secondary projection', () => { - let brainDir: string - let projRoot: string - let brain: Brainy - const ids: string[] = [] - - const openBrain = async (dir: string): Promise => { - const b = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - /** - * The production wiring, callback form: the engine's `from` is an EXCLUSIVE - * lower bound, `scanFacts` bounds are inclusive — hence `from + 1`; the - * first batch is returned and the handle closed (short batches at segment - * boundaries are legal — only EMPTY means caught up). - */ - const sourceFor = (b: Brainy): FactLogSource => - new FactLogSource(async (from, limit) => { - const scan = b.scanFacts({ fromGeneration: from + 1, batchSize: limit }) - if (!scan) throw new Error('this brain hosts no fact log — cannot reproject') - const iterator = scan.batches() - try { - const first = await iterator.next() - return first.done ? [] : first.value.facts - } finally { - if (typeof iterator.return === 'function') await iterator.return(undefined) - } - }) - - beforeAll(async () => { - brainDir = mkdtempSync(join(tmpdir(), 'brainy-reproj-')) - projRoot = mkdtempSync(join(tmpdir(), 'brainy-reproj-artifacts-')) - brain = await openBrain(brainDir) - for (let i = 0; i < ROWS; i++) { - ids.push( - await brain.add({ - data: `record ${i} filed in bucket ${i % BUCKETS}`, - type: 'document', - metadata: { bucket: `b${i % BUCKETS}` } - }) - ) - } - }, 240_000) - - afterAll(async () => { - await brain?.close().catch(() => {}) - rmSync(brainDir, { recursive: true, force: true }) - rmSync(projRoot, { recursive: true, force: true }) - }) - - it('(i) folds to caught-up through the engine and matches ground-truth counts', async () => { - const projection = await BucketCountProjection.open(join(projRoot, 'i')) - const engine = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 8 }) - engine.register(projection) - - const result = await engine.advance(projection.family, { budgetMs: 60_000 }) - - expect(result.status).toBe('caught-up') - expect(result.watermark).toBeGreaterThanOrEqual(ROWS) // one generation per add, at least - expect(result.applied).toBeGreaterThanOrEqual(ROWS) - expect(engine.quarantined(projection.family)).toEqual([]) - expect(projection.counts()).toEqual(GROUND_TRUTH) - // The stamp on disk is the adapter's own — stamped exactly at the fold head. - const reloaded = await BucketCountProjection.open(join(projRoot, 'i')) - expect(reloaded.watermark()).toBe(result.watermark) - expect(reloaded.counts()).toEqual(GROUND_TRUTH) - }) - - it('(ii) doors stay open mid-fold: find() and get() answer, and a bump preempts the advance', async () => { - const projection = await BucketCountProjection.open(join(projRoot, 'ii')) - const engine = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 4 }) - engine.register(projection) - const head = brain.scanFacts()!.headGeneration - - const inFlight = engine.advance(projection.family, { budgetMs: 60_000 }) - // The read hook: foreground door traffic announces itself, then reads — - // both interleave with the running fold on the same event loop. - engine.doorSignal.bump() - const found = await brain.find({ query: 'record filed in bucket', limit: 3 }) - const got = await brain.get(ids[0]) - const result = await inFlight - - // The doors answered mid-fold. - expect(found.length).toBeGreaterThan(0) - expect(got).toBeTruthy() - const gotMeta = got!.metadata as Record | undefined - expect((gotMeta?.bucket ?? (gotMeta?.metadata as Record)?.bucket)).toBe('b0') - - // THE PREEMPTION PIN — mechanism, not wall-clock: the bump landed before - // the first installment boundary, so the advance yielded after exactly - // one batch (≤ batchSize facts), far short of the head. - expect(result.status).toBe('preempted') - expect(result.applied).toBeGreaterThan(0) - expect(result.applied).toBeLessThanOrEqual(4) - expect(projection.appliedGenerations.length).toBe(result.applied) - expect(projection.watermark()).not.toBeNull() - expect(projection.watermark()!).toBeLessThan(head) - - // Resuming folds the remainder; nothing was lost to the preemption. - const resumed = await engine.advance(projection.family, { budgetMs: 60_000 }) - expect(resumed.status).toBe('caught-up') - expect(projection.counts()).toEqual(GROUND_TRUTH) - }) - - it('(iii) crash mid-fold: reopen and re-advance resumes from the stamp, never refolds from zero', async () => { - const projDir = join(projRoot, 'iii') - const before = await BucketCountProjection.open(projDir) - const engine1 = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 4 }) - engine1.register(before) - - // A zero budget folds exactly one guaranteed batch, then stops. - const partial = await engine1.advance(before.family, { budgetMs: 0 }) - expect(partial.status).toBe('budget-exhausted') - const stamped = before.watermark() - expect(stamped).not.toBeNull() - expect(stamped!).toBeGreaterThan(0) - - // CRASH: abandon the engine and adapter mid-fold; reopen the brain cold. - await brain.close() - brain = await openBrain(brainDir) - - const after = await BucketCountProjection.open(projDir) - expect(after.watermark()).toBe(stamped) // the stamp survived the crash - - const engine2 = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 4 }) - engine2.register(after) - const resumed = await engine2.advance(after.family, { budgetMs: 60_000 }) - expect(resumed.status).toBe('caught-up') - - // NEVER REFOLDS FROM ZERO: every generation the resumed instance applied - // sits strictly above the crash stamp. - expect(after.appliedGenerations.length).toBeGreaterThan(0) - expect(Math.min(...after.appliedGenerations)).toBeGreaterThan(stamped!) - // And the combined state — durable prefix plus resumed fold — is exact. - expect(after.counts()).toEqual(GROUND_TRUTH) - }) -}) diff --git a/tests/integration/reserved-root-mint.test.ts b/tests/integration/reserved-root-mint.test.ts deleted file mode 100644 index f9577842..00000000 --- a/tests/integration/reserved-root-mint.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** - * @module tests/integration/reserved-root-mint - * @description THE RESERVED-ROOT MINT EXEMPTION (the release's final fix): - * existing brains mint the VFS root (the all-zeros UUID) as int 0 by - * construction at genesis — the one legitimate zero in the id space. The - * adoption path must accept it (every real depot brain refused adoption - * over this); a zero mint for ANY OTHER id remains a corrupt-mint refusal. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const ROOT = '00000000-0000-0000-0000-000000000000' -const dirs: string[] = [] -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -type MapperBox = { - metadataIndex: { - getIdMapper(): { - uuidToInt: Map - intToUuid: Map - dirty?: boolean - } - } -} - -describe('reserved-root mint exemption', () => { - it('adoption succeeds on a brain whose VFS root carries int 0 (the depot-brain shape)', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-root0-')) - dirs.push(dir) - // Build the brain in 'defer' so we control the adoption moment. - const brain = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - logAuthority: 'defer' - }) - await brain.init() - brains.push(brain) - await brain.add({ data: 'depot row', type: NounType.Document, metadata: { k: 1 } }) - - // The genesis-era shape: the root's mint is 0 (white-box — real depot - // brains carry this in their persisted mapper). - const mapper = (brain as unknown as MapperBox).metadataIndex.getIdMapper() - const currentInt = mapper.uuidToInt.get(ROOT) - if (currentInt !== undefined) mapper.intToUuid.delete(currentInt) - mapper.uuidToInt.set(ROOT, 0) - mapper.intToUuid.set(0, ROOT) - - // THE PIN: adoption goes green — the backfill re-commits the root with - // its legitimate int 0 instead of refusing the whole brain. - const report = await brain.adoptLogAuthority() - expect(report.verdict).toBe('green') - expect(brain.logAuthority().authority).toBe('log') - // And the brain keeps serving + writing after the flip. - const fresh = await brain.add({ data: 'post-adopt', type: NounType.Document, metadata: { k: 2 } }) - expect((await brain.get(fresh))!.data).toContain('post-adopt') - }, 120000) - - it('a zero mint for a NON-root id still refuses at the mint seam, loudly and typed', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-nonroot0-')) - dirs.push(dir) - const brain = new Brainy({ - storage: { type: 'filesystem', path: dir }, - requireSubtype: false, - logAuthority: 'defer' - }) - await brain.init() - brains.push(brain) - const victim = await brain.add({ data: 'poisoned mint target', type: NounType.Document, metadata: {} }) - - // Corrupt shape: some OTHER id maps to 0. (A full update() SELF-HEALS - // this — the index cycle re-mints before the fact is written, which is - // the correct outcome — so the pin holds the guard at its real seam: - // the fact log's minter, which is what stands between a surviving zero - // and the wire.) - const mapper = (brain as unknown as MapperBox).metadataIndex.getIdMapper() - const currentInt = mapper.uuidToInt.get(victim) - if (currentInt !== undefined) mapper.intToUuid.delete(currentInt) - mapper.uuidToInt.set(victim, 0) - mapper.intToUuid.set(0, victim) - - const factLog = (brain as unknown as { - generationStore: { getFactLog(): { intMinter(kind: string, id: string): bigint } } - }).generationStore.getFactLog() - expect(() => factLog.intMinter('noun', victim)).toThrow( - /reserved for the VFS root|minted ints are positive/ - ) - // And the reserved root itself passes the same seam with 0. - mapper.uuidToInt.set(ROOT, 0) - mapper.intToUuid.set(0, ROOT) - expect(factLog.intMinter('noun', ROOT)).toBe(0n) - }, 120000) -}) diff --git a/tests/integration/transact-durability-barrier.test.ts b/tests/integration/transact-durability-barrier.test.ts index 8a670ba5..9311ce67 100644 --- a/tests/integration/transact-durability-barrier.test.ts +++ b/tests/integration/transact-durability-barrier.test.ts @@ -43,13 +43,6 @@ describe('transact durability barrier — entity writes fsync before the counter }) await brain.init() - // Drain the pending tier BEFORE instrumenting: the adopt-at-open fleet - // default re-commits the init-time baseline as a buffered single-op - // generation, and transact() flushes buffered single-ops first — that - // flush's manifest sync would otherwise be recorded ahead of the - // transact's own commit point and break the first-index ordering pins. - await brain.flush() - // Instrument the real filesystem storage: record every fsync batch in order, // and count barrier open/flush, delegating to the originals. syncCalls = [] diff --git a/tests/integration/wait-for-indexed.test.ts b/tests/integration/wait-for-indexed.test.ts deleted file mode 100644 index 711ddc99..00000000 --- a/tests/integration/wait-for-indexed.test.ts +++ /dev/null @@ -1,219 +0,0 @@ -/** - * @module tests/integration/wait-for-indexed - * @description THE READ BARRIER — `brain.waitForIndexed(path?, opts?)`. A - * consumer that writes and then semantically recalls gets ONE honest barrier - * instead of guessing. The contract pinned here: - * - * 1. SEMANTIC LEG: a deferred add followed by `waitForIndexed('semantic')` - * resolves only after the vector landed — the row is vector-searchable - * the moment the barrier returns. - * 2. TYPED TIMEOUT: `timeoutMs` expiry REJECTS with - * WaitForIndexedTimeoutError carrying the leg + the pending count and - * naming the gauge — never a silent partial wait. - * 3. NO-ARG: every projection at the head; today that means the deferred - * embed backlog is drained. - * 4. SYNCHRONOUS LEGS: metadata/graph/aggregation resolve immediately by - * design today (they update inside the write path) — even while the - * semantic backlog is wedged. - * 5. GAUGES: getIndexStatus().projections carries the per-leg numbers, and - * the top-level pendingEmbeds compat field agrees with the semantic one. - * 6. GENERATION REFINEMENT: an empty backlog satisfies any generation - * immediately; a non-empty one falls back to the full drain. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy, WaitForIndexedTimeoutError } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const brains: Brainy[] = [] - -async function memBrain(): Promise { - const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) - await b.init() - brains.push(b) - return b -} - -/** - * Abandon a poisoned in-flight embed run (its embed promise never resolves — - * production is covered by the worker's 60s hang guard; the test takes the - * white-box shortcut for speed), then drain so teardown never wedges. - */ -async function unwedge(brain: Brainy): Promise { - ;(brain as unknown as { _embedWorkerFlight: Promise | null })._embedWorkerFlight = null - await brain.awaitPendingEmbeds() -} - -afterEach(async () => { - vi.restoreAllMocks() - for (const b of brains.splice(0)) await b.close().catch(() => {}) -}) - -describe('waitForIndexed — the read barrier', () => { - it("SEMANTIC LEG: deferred add → waitForIndexed('semantic') resolves and the row is vector-searchable after", async () => { - const brain = await memBrain() - const embedSpy = vi.spyOn(brain, 'embed') - - const id = await brain.add({ - data: 'the quarterly revenue report for the northern region', - type: NounType.Document, - deferEmbedding: true, - metadata: { kind: 'report' } - }) - expect(embedSpy, 'no embed on the ack path').not.toHaveBeenCalled() - expect(brain.pendingEmbedCount()).toBeGreaterThanOrEqual(1) - - await brain.waitForIndexed('semantic') - - // The barrier's meaning: backlog drained, vector real, row searchable. - expect(brain.pendingEmbedCount(), 'barrier means drained').toBe(0) - const after = await brain.get(id, { includeVectors: true }) - expect((after!.vector as number[]).length, 'real vector after the barrier').toBeGreaterThan(0) - const hits = await brain.find({ - query: 'the quarterly revenue report for the northern region', - searchMode: 'semantic', - limit: 5 - }) - expect(hits.map((r) => r.id), 'vector-searchable after the barrier').toContain(id) - }) - - it('TYPED TIMEOUT: a hung embedder + timeoutMs rejects with the typed error naming the pending count and the gauge', async () => { - const brain = await memBrain() - const hang = vi - .spyOn(brain, 'embed') - .mockImplementation(() => new Promise(() => {})) - - await brain.add({ - data: 'never lands while the embedder hangs', - type: NounType.Document, - deferEmbedding: true, - metadata: {} - }) - expect(brain.pendingEmbedCount()).toBe(1) - - let caught: unknown - try { - await brain.waitForIndexed('semantic', { timeoutMs: 200 }) - } catch (e) { - caught = e - } - - expect(caught, 'expiry REJECTS — never a silent partial wait').toBeInstanceOf( - WaitForIndexedTimeoutError - ) - const err = caught as WaitForIndexedTimeoutError - expect(err.path).toBe('semantic') - expect(err.timeoutMs).toBe(200) - expect(err.pendingEmbeds).toBeGreaterThanOrEqual(1) - // The message names what was still pending and the gauge to check. - expect(err.message).toContain(`${err.pendingEmbeds} deferred embed`) - expect(err.message).toContain('getIndexStatus().projections.semantic.pendingEmbeds') - - hang.mockRestore() - await unwedge(brain) - expect(brain.pendingEmbedCount()).toBe(0) - }) - - it('NO-ARG: waitForIndexed() waits on the pending-embed drain (every projection at the head)', async () => { - const brain = await memBrain() - await brain.add({ - data: 'a deferred capture that the bare barrier must cover', - type: NounType.Document, - deferEmbedding: true, - metadata: {} - }) - expect(brain.pendingEmbedCount()).toBeGreaterThanOrEqual(1) - - await brain.waitForIndexed() - - expect( - brain.pendingEmbedCount(), - 'the bare barrier drained the only asynchronous projection' - ).toBe(0) - }) - - it('SYNCHRONOUS LEGS: metadata/graph/aggregation resolve immediately — even while the semantic backlog is wedged', async () => { - const brain = await memBrain() - - // Quiet brain first: all three legs resolve on a brain with no backlog. - await brain.add({ data: 'quiet row', type: NounType.Document, metadata: { q: 1 } }) - await brain.awaitPendingEmbeds() - await brain.waitForIndexed('metadata') - await brain.waitForIndexed('graph') - await brain.waitForIndexed('aggregation') - - // The stronger pin: these projections update inside the write path today, - // so their leg resolves immediately BY DESIGN — independent of a wedged - // semantic backlog. (If any of them incorrectly delegated to the embed - // drain, this test would hang.) - const hang = vi - .spyOn(brain, 'embed') - .mockImplementation(() => new Promise(() => {})) - await brain.add({ - data: 'wedged deferred row', - type: NounType.Document, - deferEmbedding: true, - metadata: {} - }) - expect(brain.pendingEmbedCount()).toBe(1) - - await brain.waitForIndexed('metadata') - await brain.waitForIndexed('graph') - await brain.waitForIndexed('aggregation') - - hang.mockRestore() - await unwedge(brain) - }) - - it('GAUGES: getIndexStatus().projections carries the per-leg shape, and the compat field agrees', async () => { - const brain = await memBrain() - await brain.add({ data: 'gauge row', type: NounType.Document, metadata: { g: 1 } }) - await brain.awaitPendingEmbeds() - - const status = await brain.getIndexStatus() - expect(status.projections).toEqual({ - semantic: { pendingEmbeds: 0 }, - metadata: { synchronous: true }, - graph: { synchronous: true }, - aggregation: { pendingBackfills: 0, pendingCatchUps: 0 } - }) - // Compat: the existing top-level gauge stays and agrees. - expect(status.pendingEmbeds).toBe(0) - - // The semantic gauge is honest while a backlog exists. - const hang = vi - .spyOn(brain, 'embed') - .mockImplementation(() => new Promise(() => {})) - await brain.add({ - data: 'backlogged row', - type: NounType.Document, - deferEmbedding: true, - metadata: {} - }) - const busy = await brain.getIndexStatus() - expect(busy.projections.semantic.pendingEmbeds).toBeGreaterThanOrEqual(1) - expect(busy.pendingEmbeds).toBe(busy.projections.semantic.pendingEmbeds) - - hang.mockRestore() - await unwedge(brain) - }) - - it('GENERATION REFINEMENT: an empty backlog satisfies any generation immediately; a non-empty one falls back to the full drain', async () => { - const brain = await memBrain() - await brain.add({ data: 'generation row', type: NounType.Document, metadata: {} }) - await brain.awaitPendingEmbeds() - - // Empty backlog: the semantic watermark is at the head — >= any committed G. - await brain.waitForIndexed('semantic', { generation: 1 }) - - // Non-empty backlog: the conservative full drain (a superset of the - // requested wait, never a partial one). - await brain.add({ - data: 'second generation row', - type: NounType.Document, - deferEmbedding: true, - metadata: {} - }) - await brain.waitForIndexed('semantic', { generation: 1 }) - expect(brain.pendingEmbedCount(), 'the fallback is the full drain').toBe(0) - }) -}) diff --git a/tests/integration/watermark-adopt-reopen.test.ts b/tests/integration/watermark-adopt-reopen.test.ts deleted file mode 100644 index d0eb1182..00000000 --- a/tests/integration/watermark-adopt-reopen.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * @module tests/integration/watermark-adopt-reopen - * @description End-to-end LC1 watermark adoption: a clean flush+close stamps - * every projection at the committed generation; the reopen verdicts all read - * 'adopt' — a same-version reopen owes ZERO rebuild work, provably, via the - * stamps rather than via absence of complaint. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -describe('watermark stamps ride the flush fan-out', () => { - it('flush stamps all three projections at the committed generation; reopen adopts', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-wm-')) - dirs.push(dir) - let brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await brain.init() - brains.push(brain) - await brain.add({ data: 'stamped row', type: NounType.Document, metadata: { k: 1 } }) - await brain.flush() - - const committed = (brain as unknown as { - storage: { committedGeneration(): number } - }).storage.committedGeneration() - const mi = (brain as unknown as { metadataIndex: { watermark(): number | null } }).metadataIndex - expect(mi.watermark(), 'metadata stamp = committed').toBe(committed) - await brain.close() - brains.pop() - - brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await brain.init() - brains.push(brain) - const mi2 = (brain as unknown as { - metadataIndex: { watermarkVerdict(): string | null } - }).metadataIndex - expect(mi2.watermarkVerdict(), 'clean reopen adopts').toBe('adopt') - // And the brain serves. - expect((await brain.find({ where: { k: 1 }, limit: 5 })).length).toBe(1) - }, 60000) -}) diff --git a/tests/unit/aggregation/aggregation-provider-rebuild.test.ts b/tests/unit/aggregation/aggregation-provider-rebuild.test.ts deleted file mode 100644 index efd7b4bb..00000000 --- a/tests/unit/aggregation/aggregation-provider-rebuild.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -/** - * @module tests/unit/aggregation/aggregation-provider-rebuild - * @description Pins for SELF-ENGINE-LIFECYCLE-SPRINT asks (c) + (d): - * (c) the native provider's parallel `rebuildAggregate` — on the provider - * contract since 8.x but NEVER invoked (the JS walk streamed per-entity - * FFI calls instead) — is now the backfill walk's preferred door; - * (d) a write-path hook that cannot see its entity (before-image-less - * delete) flags an exact rescan LOUDLY instead of silently skipping the - * decrement (the skip let counts drift upward forever). - */ -import { describe, it, expect, vi } from 'vitest' -import { AggregationIndex } from '../../../src/aggregation/AggregationIndex.js' -import { NounType } from '../../../src/types/graphTypes.js' -import type { AggregationProvider, AggregateGroupState } from '../../../src/types/brainy.types.js' - -const DEF = { - name: 'by_subtype', - source: { type: NounType.Document }, - groupBy: ['system.subtype'] as string[], - metrics: { count: { op: 'count' as const } } -} - -/** Minimal in-memory storage double for the index's persistence surface. */ -function memStorage() { - const store = new Map() - return { - saveMetadata: async (k: string, v: unknown) => void store.set(k, v), - getMetadata: async (k: string) => store.get(k) ?? null - } as never -} - -function providerDouble(): AggregationProvider & { rebuildAggregate: ReturnType } { - return { - defineAggregate: vi.fn(), - removeAggregate: vi.fn(), - incrementalUpdate: vi.fn(() => []), - computeGroupKey: vi.fn(() => ({})), - rebuildAggregate: vi.fn((): Map => { - return new Map([ - [ - 'system.subtype=invoice', - { - groupKey: { 'system.subtype': 'invoice' }, - metrics: { count: { sum: 0, count: 2, min: Infinity, max: -Infinity, m2: 0 } } - } as AggregateGroupState - ] - ]) - }), - queryAggregate: vi.fn(() => []) - } as never -} - -describe('ask (c) — the native parallel rebuild is invoked, never dead code', () => { - it('rebuildWithProvider hands SOURCE-MATCHED entities to the provider once and swaps state in', () => { - const provider = providerDouble() - const index = new AggregationIndex(memStorage(), provider) - index.defineAggregate(DEF) - - expect(index.hasProviderRebuild()).toBe(true) - - const entities = [ - { type: NounType.Document, subtype: 'invoice', metadata: {} }, - { type: NounType.Document, subtype: 'invoice', metadata: {} }, - // Source-filter mismatch: a different noun type must be filtered OUT - // before the provider sees the batch. - { type: NounType.Person, subtype: 'invoice', metadata: {} } - ] - const handled = index.rebuildWithProvider(DEF.name, entities) - - expect(handled).toBe(true) - expect(provider.rebuildAggregate).toHaveBeenCalledTimes(1) - const [defArg, entArg] = provider.rebuildAggregate.mock.calls[0] - expect(defArg.name).toBe(DEF.name) - expect(entArg).toHaveLength(2) - - // The rebuilt state serves — and the aggregate is no longer pending. - expect(index.getPendingBackfills()).not.toContain(DEF.name) - }) - - it('returns false without a provider rebuild — the caller streams the JS walk', () => { - const index = new AggregationIndex(memStorage()) - index.defineAggregate(DEF) - expect(index.hasProviderRebuild()).toBe(false) - expect(index.rebuildWithProvider(DEF.name, [])).toBe(false) - }) -}) - -describe('ask (d) — the before-image-less delete is LOUD, never a silent skip', () => { - it('flagAllForRescan puts every defined aggregate back on the backfill list', () => { - const index = new AggregationIndex(memStorage()) - index.defineAggregate(DEF) - index.defineAggregate({ ...DEF, name: 'second' }) - // Simulate settled state: nothing pending. - for (const n of index.getPendingBackfills()) { - index.beginBackfill(n) - index.finishBackfill(n) - } - expect(index.getPendingBackfills()).toEqual([]) - - index.flagAllForRescan('delete of X carried no before-image metadata') - - expect(index.getPendingBackfills().sort()).toEqual(['by_subtype', 'second']) - }) -}) - -describe('reconcileEntity — the exact delta algebra at the catch-up boundary', () => { - it('before-only removes, after-only adds, both reconciles a group move', () => { - const index = new AggregationIndex(memStorage()) - index.defineAggregate(DEF) - for (const n of index.getPendingBackfills()) { - index.beginBackfill(n) - index.finishBackfill(n) - } - const doc = (subtype: string) => ({ type: NounType.Document, subtype, metadata: {} }) - - // Pre-window state, applied through the LIVE hooks (as adoption would - // have counted it): c and seed exist as drafts, x1 as an invoice. - index.onEntityAdded('c', doc('draft')) - index.onEntityAdded('seed', doc('draft')) - index.onEntityAdded('x1', doc('invoice')) - - // The window's reconciliation: two adds, one group move, one delete. - index.reconcileEntity(DEF.name, 'a', null, doc('invoice')) - index.reconcileEntity(DEF.name, 'b', null, doc('invoice')) - index.reconcileEntity(DEF.name, 'c', doc('draft'), doc('invoice')) - index.reconcileEntity(DEF.name, 'seed', doc('draft'), null) - - const rows = index.queryAggregate({ name: DEF.name }) - const count = (st: string) => - Number(rows.find(r => r.groupKey['system.subtype'] === st)?.metrics.count ?? 0) - expect(count('invoice')).toBe(4) // x1 + a + b + moved c - expect(count('draft')).toBe(0) // c moved out, seed deleted - }) -}) diff --git a/tests/unit/brainy/lazy-notready-honor.test.ts b/tests/unit/brainy/lazy-notready-honor.test.ts deleted file mode 100644 index 4cfc6857..00000000 --- a/tests/unit/brainy/lazy-notready-honor.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * @module tests/unit/brainy/lazy-notready-honor - * @description THE SILENT-EMPTY TRAP pin (found during a fleet adoption, - * SELF-ENGINE-PAIR-STANDARD): under `disableAutoRebuild: true`, the lazy - * first-query path (`ensureIndexesLoaded`) assessed ONLY the vector index's - * readiness — a native METADATA provider reporting not-ready (its strand - * report) never blocked the completion latch, so the promised lazy rebuild - * never fired and every `find()` silently returned `[]` on a populated - * store (measured: 52 entities durable-but-unqueryable, first query - * 0ms/0 rows). The law: a not-ready report from ANY provider falls through - * to the rebuild — never a silent empty. - * - * White-box provider-double pattern per tests/unit/brainy/migration-deference. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' -import { NounType } from '../../../src/types/graphTypes.js' -import { createTestConfig } from '../../helpers/test-factory.js' - -interface BrainInternals { - index: { size(): number } - metadataIndex: { isReady?: () => boolean } - lazyRebuildCompleted: boolean - ensureIndexesLoaded(): Promise - rebuildIndexesIfNeeded(force?: boolean): Promise -} - -const brains: Brainy[] = [] - -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - vi.restoreAllMocks() -}) - -async function warmLazyBrain(): Promise<{ brain: Brainy; internals: BrainInternals }> { - const brain = new Brainy(createTestConfig({ disableAutoRebuild: true })) - await brain.init() - brains.push(brain) - for (let i = 0; i < 3; i++) { - await brain.add({ data: `row ${i}`, type: NounType.Document, metadata: { i } }) - } - const internals = brain as unknown as BrainInternals - internals.lazyRebuildCompleted = false // simulate the cold first query - return { brain, internals } -} - -describe('lazy path honors EVERY provider’s not-ready report', () => { - it('a not-ready METADATA provider blocks the completion latch and fires the rebuild', async () => { - const { internals } = await warmLazyBrain() - - // The trap's shape: vector side looks fine (populated), metadata - // provider says NOT ready — the old gate latched complete here. - ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => false - const rebuildSpy = vi - .spyOn(internals, 'rebuildIndexesIfNeeded') - .mockResolvedValue(undefined) - - await internals.ensureIndexesLoaded() - - expect(rebuildSpy, 'not-ready metadata provider must fire the lazy rebuild').toHaveBeenCalledWith(true) - }) - - it('control: all providers ready/unknown+populated → latch completes, no rebuild', async () => { - const { internals } = await warmLazyBrain() - ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => true - const rebuildSpy = vi - .spyOn(internals, 'rebuildIndexesIfNeeded') - .mockResolvedValue(undefined) - - await internals.ensureIndexesLoaded() - - expect(rebuildSpy).not.toHaveBeenCalled() - expect(internals.lazyRebuildCompleted).toBe(true) - }) -}) diff --git a/tests/unit/brainy/persistence-policy.test.ts b/tests/unit/brainy/persistence-policy.test.ts deleted file mode 100644 index 92bb4e3c..00000000 --- a/tests/unit/brainy/persistence-policy.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -/** - * @module tests/unit/brainy/persistence-policy - * @description THE ENGINE-OWNED FLUSH CADENCE pins (A4, - * SELF-ENGINE-LIFECYCLE-SPRINT, David-directed: callers NEVER call flush() - * in hot paths). The production disease: 829 caller-scheduled per-write - * flushes convoying into 45–66 second write walls — cadence hand-rolled a - * layer above the only layer that can see dirty state and IO pressure. - * - * Pinned here: (1) the write-count trigger fires a BACKGROUND flush without - * any caller flush(); (2) the idle trigger; (3) `'manual'` restores - * caller-owned cadence exactly; (4) THE ACK LAW — a write acknowledges - * without awaiting any background flush, even one that never resolves. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' -import { NounType } from '../../../src/types/graphTypes.js' - -const brains: Brainy[] = [] - -async function mk(persistence?: { - policy?: 'auto' | 'manual' - flushEveryWrites?: number - flushIntervalMs?: number - flushOnIdleMs?: number -}): Promise { - const b = new Brainy({ - storage: { type: 'memory' }, - requireSubtype: false, - ...(persistence && { persistence }) - }) - await b.init() - brains.push(b) - return b -} - -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - vi.restoreAllMocks() -}) - -describe('persistence policy — the engine owns its flush cadence', () => { - it('write-count trigger: N committed writes fire ONE background flush, no caller flush()', async () => { - const brain = await mk({ flushEveryWrites: 5, flushOnIdleMs: 60_000, flushIntervalMs: 600_000 }) - const flushSpy = vi.spyOn(brain, 'flush') - - for (let i = 0; i < 5; i++) { - await brain.add({ data: `w${i}`, type: NounType.Document, metadata: { i } }) - } - - await vi.waitFor(() => expect(flushSpy).toHaveBeenCalled(), { timeout: 5000 }) - // Single-flight: the threshold crossing kicks exactly one. - expect(flushSpy.mock.calls.length).toBe(1) - }) - - it('idle trigger: a quiet store with dirty writes flushes itself', async () => { - const brain = await mk({ flushEveryWrites: 10_000, flushIntervalMs: 600_000, flushOnIdleMs: 60 }) - const flushSpy = vi.spyOn(brain, 'flush') - - await brain.add({ data: 'lone write', type: NounType.Document, metadata: {} }) - - await vi.waitFor(() => expect(flushSpy).toHaveBeenCalled(), { timeout: 5000 }) - }) - - it('idle debounce under load: slow writes never fire a flush per inter-write gap', async () => { - // The contended-disk amplifier: writes slower than the idle window make - // every gap look idle — without the spacing floor this fired a full - // flush per write (measured 15 background flushes in 100 contended adds - // on a production-shaped box). The floor (min(interval, 10×idle)) caps - // idle fires; deferred, never dropped. - const brain = await mk({ flushEveryWrites: 10_000, flushIntervalMs: 600_000, flushOnIdleMs: 50 }) - const flushSpy = vi.spyOn(brain, 'flush') - - // Six writes spaced wider than the idle window (50ms) with the whole - // span inside ~one floor window (500ms): the old behavior fires ~an - // idle flush per gap (≈6); the debounced behavior fires at most two - // (one immediate boot-window fire + one at the floor boundary). - for (let i = 0; i < 6; i++) { - await brain.add({ data: `slow ${i}`, type: NounType.Document, metadata: {} }) - await new Promise((r) => setTimeout(r, 70)) - } - expect(flushSpy.mock.calls.length, 'no flush-per-gap amplifier').toBeLessThanOrEqual(2) - - // Deferred, never dropped: the dirty writes still persist once the - // floor elapses on the now-quiet store. - await vi.waitFor(() => expect(flushSpy).toHaveBeenCalled(), { timeout: 5000 }) - }) - - it("'manual' policy: the engine NEVER flushes on its own", async () => { - const brain = await mk({ policy: 'manual', flushEveryWrites: 2, flushOnIdleMs: 30 }) - const flushSpy = vi.spyOn(brain, 'flush') - - for (let i = 0; i < 6; i++) { - await brain.add({ data: `m${i}`, type: NounType.Document, metadata: { i } }) - } - await new Promise((r) => setTimeout(r, 150)) - - expect(flushSpy).not.toHaveBeenCalled() - }) - - it('THE ACK LAW: writes acknowledge without awaiting the background flush — even a hung one', async () => { - const brain = await mk({ flushEveryWrites: 2, flushOnIdleMs: 60_000, flushIntervalMs: 600_000 }) - // A flush that NEVER resolves: if any write ack awaited it, the test - // would time out. (The engine's background flight must be fire-and-log.) - vi.spyOn(brain, 'flush').mockImplementation(() => new Promise(() => {})) - - for (let i = 0; i < 6; i++) { - const id = await brain.add({ data: `a${i}`, type: NounType.Document, metadata: { i } }) - expect(id).toBeTruthy() - } - // All six writes acked while the "flush" hangs forever. - const rows = await brain.find({ type: NounType.Document, limit: 10 }) - expect(rows.length).toBe(6) - - // Un-hang before afterEach close(): restore the method AND drop the - // never-resolving in-flight promise (close() awaits the flight — with a - // real flush that is correct; here it is the test's own artifact). - vi.restoreAllMocks() - ;(brain as unknown as { _persistBackgroundFlight: Promise | null })._persistBackgroundFlight = - null - }) -}) diff --git a/tests/unit/db/bounded-chains.test.ts b/tests/unit/db/bounded-chains.test.ts index d356277d..034bc663 100644 --- a/tests/unit/db/bounded-chains.test.ts +++ b/tests/unit/db/bounded-chains.test.ts @@ -477,17 +477,14 @@ describe('materializeAtGeneration — bounded & deadlock-free (GA #33)', () => { const store = (brain as any).generationStore const N = 400 - // Relative, not absolute: under the adopt-at-open default the open-time - // baseline backfill takes a generation of its own, so the first add is - // NOT generation 1 — pin the deep generation to the first add's commit. - let deepGen = 0 for (let i = 0; i < N; i++) { await brain.add({ data: `doc ${i}`, type: NounType.Document, subtype: 'note', metadata: { i }, vector: VEC }) - if (i === 0) deepGen = brain.generation() } const R = brain.generation() // ≈ N (each add is its own generation) expect(R).toBeGreaterThanOrEqual(N) + const deepGen = 1 + // Count getDelta invocations during the materialize. const realGetDelta = store.getDelta.bind(store) let getDeltaCalls = 0 @@ -512,8 +509,7 @@ describe('materializeAtGeneration — bounded & deadlock-free (GA #33)', () => { expect(getDeltaCalls).toBeLessThan(R * 5) expect(getDeltaCalls).toBeLessThan(N * N) // the regression guard - // The materialized brain at the first add's generation holds exactly the - // one user entity that existed. + // The materialized at-gen-1 brain holds exactly the one entity that existed. const atGen1 = await handle.find({ limit: N + 10 }) expect(atGen1.length).toBe(1) await handle.close() diff --git a/tests/unit/db/fact-log-group-sync.test.ts b/tests/unit/db/fact-log-group-sync.test.ts deleted file mode 100644 index 401aa3e4..00000000 --- a/tests/unit/db/fact-log-group-sync.test.ts +++ /dev/null @@ -1,270 +0,0 @@ -/** - * @module tests/unit/db/fact-log-group-sync - * @description Group commit on the fact log — the covering guarantee behind - * durable-at-ack: concurrent callers of ensureSynced() share ONE covering - * fsync (running + queued slots), a caller appending during a running sync - * joins a sync that STARTS after its append (never the possibly-stale running - * one), a solo writer syncs immediately, and at the brain level an at-ack - * ack resolving means the write's fact is on disk. - * - * The final pin holds the at-ack durability contract END TO END: an acked - * write's fact survives a crash-shaped reopen. This was a `.fails` known - * gap (FactLog.open() truncated every fact beyond the committed watermark, - * which only advances at the pending-tier flush) — CURED by the 10.0.0 - * adopt-at-open fleet default: a fresh brain stores the log-authority - * artifact at open, and under 'log' authority recovery REPLAYS intact - * facts above the manifest instead of truncating them. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { Brainy } from '../../../src/index.js' -import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' -import { - FactLog, - storageSupportsFactLog, - type CommitFact, - type FactLogStorage -} from '../../../src/db/factLog.js' - -const UUID = (n: number): string => - `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` - -const fact = (generation: number): CommitFact => ({ - generation, - timestamp: 1_700_000_000_000 + generation, - ops: [ - { - kind: 'noun', - id: UUID(generation), - record: { metadata: { noun: 'document', title: `doc ${generation}` }, vector: { v: [1, 2] } } - } - ] -}) - -/** Scan every fact from a FRESH reader log over the same directory. */ -async function readBack(dir: string, committedHead: number): Promise { - const storage: any = new FileSystemStorage(dir) - await storage.init() - const reader = new FactLog(storage as FactLogStorage) - await reader.open(committedHead) - const facts: CommitFact[] = [] - const scan = reader.scanFacts() - for await (const batch of scan.batches()) facts.push(...batch.facts) - return facts -} - -describe('fact log group commit — the covering fsync', () => { - let dir: string - let storage: any - let log: FactLog - - beforeEach(async () => { - dir = mkdtempSync(join(tmpdir(), 'brainy-group-sync-')) - storage = new FileSystemStorage(dir) - await storage.init() - expect(storageSupportsFactLog(storage)).toBe(true) - log = new FactLog(storage as FactLogStorage) - await log.open(0) - }) - - afterEach(() => { - rmSync(dir, { recursive: true, force: true }) - }) - - it('many concurrent ensureSynced() callers share one covering fsync — every caller resolves, batching happened', async () => { - for (let g = 1; g <= 10; g++) await log.append(fact(g)) - - // Count REAL fsync batches at the storage boundary, with a small delay so - // the concurrent callers genuinely overlap the running sync. - let fsyncBatches = 0 - const origSync = storage.syncRawObjects.bind(storage) - storage.syncRawObjects = async (paths: string[]) => { - fsyncBatches++ - await new Promise((r) => setTimeout(r, 15)) - return origSync(paths) - } - - const callers = Array.from({ length: 10 }, () => log.ensureSynced()) - await Promise.all(callers) // every caller resolves — no lost writer - - expect(fsyncBatches, 'callers shared a covering fsync').toBeLessThan(10) - expect(fsyncBatches).toBeGreaterThanOrEqual(1) - - // Durable: a fresh reader over the same directory sees all 10 facts. - const facts = await readBack(dir, 10) - expect(facts.map((f) => f.generation)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - }) - - it('an append during a RUNNING sync is covered by a sync that starts after it — never the stale running one', async () => { - for (let g = 1; g <= 3; g++) await log.append(fact(g)) - - // Gate the FIRST fsync so a sync is provably in flight. - let fsyncBatches = 0 - let releaseGate!: () => void - const gate = new Promise((r) => { - releaseGate = r - }) - let gated = true - const origSync = storage.syncRawObjects.bind(storage) - storage.syncRawObjects = async (paths: string[]) => { - fsyncBatches++ - if (gated) { - gated = false - await gate - } - return origSync(paths) - } - - const p1 = log.ensureSynced() // sync A: snapshots gens 1..3, blocks in fsync - await new Promise((r) => setTimeout(r, 10)) - expect(fsyncBatches, 'sync A is in flight').toBe(1) - - await log.append(fact(4)) // lands AFTER sync A snapshotted - let p2Resolved = false - const p2 = log.ensureSynced().then(() => { - p2Resolved = true - }) - - // The covering guarantee: p2 must NOT resolve off the running sync (it - // may have snapshotted before the append) — it waits for the queued one. - await new Promise((r) => setTimeout(r, 25)) - expect(p2Resolved, 'p2 never joins the possibly-stale running sync').toBe(false) - - releaseGate() - await p1 - await p2 - expect(p2Resolved).toBe(true) - expect(fsyncBatches, 'the queued covering sync ran after the running one').toBe(2) - - // The late append is durable once p2 resolved. - const facts = await readBack(dir, 4) - expect(facts.map((f) => f.generation)).toEqual([1, 2, 3, 4]) - }) - - it('a solo writer syncs immediately — one fsync, and a dirty-free ensureSynced adds none', async () => { - // Count only covering syncs: the first append itself fsyncs the tail - // manifest (the manifest-first flip), so instrument AFTER it. - await log.append(fact(1)) - let fsyncBatches = 0 - const origSync = storage.syncRawObjects.bind(storage) - storage.syncRawObjects = async (paths: string[]) => { - fsyncBatches++ - return origSync(paths) - } - - await log.ensureSynced() - expect(fsyncBatches).toBe(1) - - // Nothing new appended: the covering sync finds nothing dirty. - await log.ensureSynced() - expect(fsyncBatches).toBe(1) - }) -}) - -describe('durable-at-ack through the brain (group commit end-to-end)', () => { - const dirs: string[] = [] - const brains: any[] = [] - - const openBrain = async (dir?: string): Promise<{ brain: any; dir: string }> => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - const d = dir ?? mkdtempSync(join(tmpdir(), 'brainy-at-ack-')) - if (!dir) dirs.push(d) - const brain: any = new Brainy({ - storage: { type: 'filesystem', path: d }, - requireSubtype: false, - silent: true, - dimensions: 384 - }) - brains.push(brain) - await brain.init() - return { brain, dir: d } - } - - afterEach(async () => { - for (const b of brains.splice(0)) await b.close?.().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) - }) - - it('at-ack: N concurrent add() acks all resolve, every ack was covered by a log sync, and every fact is on disk after reopen', async () => { - const { brain, dir } = await openBrain() - // The 10.0.0 fleet default already adopted log authority at open, so - // the brain is at-ack; the white-box engage stays so this pin holds the - // durability MACHINERY itself independent of the open-time posture. - brain.generationStore.setLogDurability('at-ack') - - const factLog = brain.generationStore.getFactLog() - expect(factLog).not.toBeNull() - let syncs = 0 - const origSync = factLog.sync.bind(factLog) - factLog.sync = async () => { - syncs++ - return origSync() - } - - const ids: string[] = await Promise.all( - Array.from({ length: 10 }, (_, i) => - brain.add({ data: `concurrent write ${i}`, type: 'document', metadata: { i } }) - ) - ) - expect(new Set(ids).size, 'every ack resolved with a distinct id').toBe(10) - // Honest pin: single-op acks serialize under the commit mutex (append + - // covering sync run inside it), so concurrent add() acks do not currently - // share one fsync — cross-writer batching is the FactLog-layer property - // pinned above. What must hold here: at least one covering sync ran, and - // no ack resolved without the machinery engaged. - expect(syncs).toBeGreaterThanOrEqual(1) - expect(syncs).toBeLessThanOrEqual(10) - - await brain.close() - const { brain: reopened } = await openBrain(dir) - const scan = reopened.scanFacts() - expect(scan).not.toBeNull() - const liveFactIds = new Set() - for await (const batch of scan!.batches()) { - for (const f of batch.facts) { - for (const op of f.ops) if (op.kind === 'noun' && op.record !== null) liveFactIds.add(op.id) - } - } - for (const id of ids) { - expect(liveFactIds.has(id), `fact for acked write ${id} survives reopen`).toBe(true) - } - }) - - // THE AT-ACK CONTRACT, HELD (was a `.fails` known gap): an acked write's - // fact survives a crash-shaped reopen. Fixed by the 10.0.0 adopt-at-open - // fleet default — this brain adopted LOG authority at open (artifact - // stored, durable-at-ack live), and under 'log' authority FactLog - // recovery REPLAYS intact facts above the committed watermark at the next - // open instead of truncating them back. Durable-at-ack now survives the - // very crash it exists for. - it('at-ack CONTRACT: acked facts survive a crash-shaped reopen (no flush ever ran)', async () => { - const { brain, dir } = await openBrain() - expect(brain.logAuthority().authority, 'the fleet default adopted at open').toBe('log') - expect(brain.generationStore.logDurability).toBe('at-ack') - // Crash simulation: the pending-tier durability flush never happens - // (every trigger routes through flushPendingSingleOps), and the brain is - // abandoned without close() — exactly the power-loss shape at-ack is for. - brain.generationStore.flushPendingSingleOps = async () => {} - - const ids: string[] = [] - for (let i = 0; i < 5; i++) { - ids.push(await brain.add({ data: `acked write ${i}`, type: 'document', metadata: { i } })) - } - - // No flush, no close — reopen the directory as a new session. - const { brain: reopened } = await openBrain(dir) - const scan = reopened.scanFacts() - expect(scan).not.toBeNull() - const liveFactIds = new Set() - for await (const batch of scan!.batches()) { - for (const f of batch.facts) { - for (const op of f.ops) if (op.kind === 'noun' && op.record !== null) liveFactIds.add(op.id) - } - } - for (const id of ids) { - expect(liveFactIds.has(id), `acked fact ${id} survives the crash-shaped reopen`).toBe(true) - } - }) -}) diff --git a/tests/unit/db/factLogFormat.test.ts b/tests/unit/db/factLogFormat.test.ts deleted file mode 100644 index 0c507b41..00000000 --- a/tests/unit/db/factLogFormat.test.ts +++ /dev/null @@ -1,808 +0,0 @@ -/** - * @module tests/unit/db/factLogFormat - * @description Fact-log format v2 (record envelope + sector seals) pinned at - * the byte level: every record type round-trips field-exact (bigint ints, - * bin16 uuids, float-exact vectors), headers read v1 AND v2, unknown record - * types/versions refuse loudly with the typed error, the reserved crypto - * envelope (cipherFlag/keyId — plaintext-only this release) refuses anything - * nonzero/non-nil with the same typed error, genesis width mismatches - * refuse naming both widths, sealed groups align to the sector size with - * invisible pads, vector refs are writer-enforced single-hop, and torn tails - * truncate to the intact prefix at EVERY byte offset. This module is the - * reference implementation of a two-implementation contract — golden byte - * vectors here are frozen; a change that breaks them is a format change. - */ -import { describe, it, expect } from 'vitest' -import { encode, decode } from '@msgpack/msgpack' -import { - encodeFactV2, - decodeFact, - decodeGroupV2, - encodeSegmentHeaderV2, - parseSegmentHeader, - sealGroup, - framePayload, - encodePadFrame, - minPadFrameBytes, - UnknownLogRecordError, - GenesisWidthMismatchError, - LOG_RECORD_TYPES, - LOG_RECORD_VERSION, - LOG_RECORD_CIPHER_PLAINTEXT, - FACT_LOG_FORMAT_V1, - FACT_LOG_FORMAT_V2, - SEGMENT_HEADER_BYTES, - DEFAULT_SEAL_SIZE, - type CommitFactV2, - type LogRecord, - type VectorRef -} from '../../../src/db/factLogFormat.js' - -const UUID = (n: number): string => - `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` -const HASH_A = 'ab'.repeat(32) -const HASH_B = '0123456789abcdef'.repeat(4) - -/** uuid string → bin16 (test-local mirror of the wire helper). */ -const uuidBytes = (id: string): Uint8Array => { - const hex = id.replace(/-/g, '') - const bytes = new Uint8Array(16) - for (let i = 0; i < 16; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16) - return bytes -} - -const hex = (bytes: Uint8Array): string => Buffer.from(bytes).toString('hex') - -/** Encode → strip frame → decode; the standard round-trip. */ -const roundTrip = ( - fact: CommitFactV2, - encOpts?: Parameters[1], - decOpts?: { expectedIdSpaceWidth?: 32 | 64 } -): CommitFactV2 => decodeFact(framePayload(encodeFactV2(fact, encOpts)), 2, decOpts) - -/** A single-record fact around `record`, canonical shape for strict equality. */ -const factOf = (generation: number, record: LogRecord): CommitFactV2 => ({ - generation, - timestamp: 1_700_000_000_000 + generation, - records: [record] -}) - -/** - * Build a fact frame of EXACTLY `totalBytes` (projection.note binary filler), - * for engineering precise seal-boundary scenarios. - */ -function frameOfExactly(totalBytes: number, generation: number): Uint8Array { - let fillerLength = Math.max(0, totalBytes - 60) - for (let i = 0; i < 12; i++) { - const frame = encodeFactV2({ - generation, - timestamp: 1, - records: [{ type: 'projection.note', note: { fill: new Uint8Array(fillerLength) } }] - }) - const diff = totalBytes - frame.length - if (diff === 0) return frame - fillerLength += diff - if (fillerLength < 0) throw new Error(`no frame of ${totalBytes} bytes is constructible`) - } - throw new Error('frame sizing did not converge') -} - -describe('fact-log format v2 — record round-trips (field-exact)', () => { - it('noun.afterImage: bin16 uuid, u64-as-bigint beyond 2^53, metadata, inline vector', () => { - const fact = factOf(1, { - type: 'noun.afterImage', - id: UUID(1), - entityInt: (1n << 60n) + 3n, // provably beyond Number territory - metadata: { - noun: 'document', - title: 'doc 1', - nested: { tags: ['a', 'b'], score: 0.25 }, - big: Number.MAX_SAFE_INTEGER, - negative: -42, - flag: true, - missing: null - }, - vectorLeg: [0.1, -2.5, 3, 1e-7] - }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('noun.tombstone: body-less removal', () => { - const fact = factOf(2, { type: 'noun.tombstone', id: UUID(2) }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('verb.afterImage: both endpoints, three u64 handles, verb name', () => { - const fact = factOf(3, { - type: 'verb.afterImage', - id: UUID(3), - verbInt: 18_446_744_073_709_551_615n, // u64 max - metadata: { verb: 'contains', weight: 0.5 }, - vectorLeg: null, - verb: 'contains', - sourceId: UUID(31), - sourceInt: 7n, - targetId: UUID(32), - targetInt: (1n << 53n) + 1n - }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('verb.tombstone: body-less removal', () => { - const fact = factOf(4, { type: 'verb.tombstone', id: UUID(4) }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('batch.meta: one metadata map per fact', () => { - const fact = factOf(5, { type: 'batch.meta', meta: { source: 'import', count: 12 } }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('embed.pending: id + enqueue time', () => { - const fact = factOf(6, { type: 'embed.pending', id: UUID(6), enqueuedAt: 1_700_000_000_777 }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('embed.landed: inline vector, float-exact', () => { - const fact = factOf(7, { - type: 'embed.landed', - id: UUID(7), - vector: [0.30000000000000004, -1.5, 2 ** 31 + 0.5] - }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('blob.manifest: bin32 hash, size, mimeType, both refOps', () => { - const add = factOf(8, { - type: 'blob.manifest', - hash: HASH_A, - size: 1_048_576, - mimeType: 'image/png', - refOp: 'add' - }) - expect(roundTrip(add)).toStrictEqual(add) - const release = factOf(9, { - type: 'blob.manifest', - hash: HASH_B, - size: 0, - mimeType: 'application/octet-stream', - refOp: 'release' - }) - expect(roundTrip(release)).toStrictEqual(release) - }) - - it('projection.note: opaque map rides untouched', () => { - const fact = factOf(10, { - type: 'projection.note', - note: { consumer: 'reserved', payload: { depth: [1, 2, 3] } } - }) - expect(roundTrip(fact)).toStrictEqual(fact) - }) - - it('bootstrap.baseline: kind flag, metadata, vector leg — both kinds', () => { - const noun = factOf(11, { - type: 'bootstrap.baseline', - id: UUID(11), - kind: 'noun', - metadata: { noun: 'person' }, - vectorLeg: [1, 2, 3] - }) - expect(roundTrip(noun)).toStrictEqual(noun) - const verb = factOf(12, { - type: 'bootstrap.baseline', - id: UUID(12), - kind: 'verb', - metadata: null, - vectorLeg: null - }) - expect(roundTrip(verb)).toStrictEqual(verb) - }) - - it('log.genesis: width, brainId, createdAt — both widths', () => { - for (const idSpaceWidth of [32, 64] as const) { - const fact = factOf(1, { - type: 'log.genesis', - idSpaceWidth, - brainId: UUID(999), - createdAt: 1_700_000_000_000 - }) - expect(roundTrip(fact, undefined, { expectedIdSpaceWidth: idSpaceWidth })).toStrictEqual(fact) - } - }) - - it('a combined fact: genesis-first, all record types, fact meta, duplicate blobHashes', () => { - const fact: CommitFactV2 = { - generation: 1, - timestamp: 1_700_000_000_001, - records: [ - { type: 'log.genesis', idSpaceWidth: 64, brainId: UUID(999), createdAt: 1_699_999_999_999 }, - { type: 'noun.afterImage', id: UUID(1), entityInt: 1n, metadata: { a: 1 }, vectorLeg: [0.5] }, - { type: 'noun.tombstone', id: UUID(2) }, - { - type: 'verb.afterImage', - id: UUID(3), - verbInt: 3n, - metadata: null, - vectorLeg: null, - verb: 'relatedTo', - sourceId: UUID(31), - sourceInt: 1n, - targetId: UUID(32), - targetInt: 2n - }, - { type: 'verb.tombstone', id: UUID(4) }, - { type: 'batch.meta', meta: { origin: 'unit' } }, - { type: 'embed.pending', id: UUID(6), enqueuedAt: 5 }, - { type: 'embed.landed', id: UUID(7), vector: [0.1] }, - { type: 'blob.manifest', hash: HASH_A, size: 9, mimeType: 'text/plain', refOp: 'add' }, - { type: 'projection.note', note: {} }, - { type: 'bootstrap.baseline', id: UUID(11), kind: 'noun', metadata: null, vectorLeg: null } - ], - meta: { source: 'unit' }, - blobHashes: [HASH_A, HASH_A] // multiset — duplicates preserved - } - expect(roundTrip(fact, undefined, { expectedIdSpaceWidth: 64 })).toStrictEqual(fact) - }) -}) - -describe('fact-log format v2 — golden byte vectors (frozen contract)', () => { - it('v2 segment header bytes are pinned', () => { - expect(hex(encodeSegmentHeaderV2(7, 4096))).toBe( - '4246414354530000020000000700000000000000001000000000000000000000' - ) - }) - - it('a noun.tombstone frame is pinned byte-for-byte', () => { - const frame = encodeFactV2({ - generation: 3, - timestamp: 1_700_000_000_123, - records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }] - }) - expect(hex(frame)).toBe( - '2d00000048e4d43695cf0000000000000003cf0000018bcfe5687b9195020100c0' + - 'c41000000000000040008000000000000042c0c0' - ) - }) - - it('u64 registry fields ride as fixed 8-byte msgpack uint64 (0xcf)', () => { - const payload = framePayload( - encodeFactV2(factOf(1, { type: 'embed.pending', id: UUID(1), enqueuedAt: 2 })) - ) - // positions 0 and 1 (generation, timestamp) and enqueuedAt are all 0xcf - expect(payload[1]).toBe(0xcf) - expect(payload[10]).toBe(0xcf) - }) -}) - -describe('fact-log format v2 — segment headers (v1 AND v2)', () => { - const v1Header = (): Uint8Array => { - const header = new Uint8Array(SEGMENT_HEADER_BYTES) - header.set(new Uint8Array([0x42, 0x46, 0x41, 0x43, 0x54, 0x53, 0x00, 0x00]), 0) - const view = new DataView(header.buffer) - view.setUint32(8, FACT_LOG_FORMAT_V1, true) - view.setBigUint64(12, 42n, true) - return header - } - - it('a v2 header round-trips with its sealSize', () => { - const header = encodeSegmentHeaderV2(123_456, 512) - expect(header.length).toBe(SEGMENT_HEADER_BYTES) - expect(parseSegmentHeader(header)).toStrictEqual({ - formatVersion: FACT_LOG_FORMAT_V2, - firstGeneration: 123_456, - sealSize: 512 - }) - // default sealSize - expect(parseSegmentHeader(encodeSegmentHeaderV2(1)).sealSize).toBe(DEFAULT_SEAL_SIZE) - }) - - it('a v1 header parses: version 1, sealSize absent (undefined)', () => { - const parsed = parseSegmentHeader(v1Header()) - expect(parsed).toStrictEqual({ formatVersion: FACT_LOG_FORMAT_V1, firstGeneration: 42 }) - expect(parsed.sealSize).toBeUndefined() - }) - - it('corrupted magic throws', () => { - const header = encodeSegmentHeaderV2(1) - header[0] = 0x58 - expect(() => parseSegmentHeader(header)).toThrow(/bad magic/) - }) - - it('non-zero reserved bytes throw — v1 (offset 20+) and v2 (offset 22+)', () => { - const v1 = v1Header() - v1[21] = 1 - expect(() => parseSegmentHeader(v1)).toThrow(/non-zero reserved/) - - const v2 = encodeSegmentHeaderV2(1, 4096) - v2[25] = 1 - expect(() => parseSegmentHeader(v2)).toThrow(/non-zero reserved/) - }) - - it('the v2 sealSize bytes are NOT reserved bytes in v2 (but ARE in v1)', () => { - // sealSize 512 puts a non-zero byte at offset 21 — legal in v2 only. - const v2 = encodeSegmentHeaderV2(1, 512) - expect(parseSegmentHeader(v2).sealSize).toBe(512) - const v1 = v1Header() - v1[20] = 0x00 - v1[21] = 0x02 // same bytes a v2 sealSize=512 would carry - expect(() => parseSegmentHeader(v1)).toThrow(/non-zero reserved/) - }) - - it('an unknown header version and a short buffer throw', () => { - const header = encodeSegmentHeaderV2(1) - new DataView(header.buffer).setUint32(8, 3, true) - expect(() => parseSegmentHeader(header)).toThrow(/formatVersion 3/) - expect(() => parseSegmentHeader(header.subarray(0, 31))).toThrow(/32 bytes/) - }) - - it('header writer refuses out-of-range inputs', () => { - expect(() => encodeSegmentHeaderV2(-1)).toThrow(/non-negative/) - expect(() => encodeSegmentHeaderV2(1, 32)).toThrow(/sealSize/) - expect(() => encodeSegmentHeaderV2(1, 65_536)).toThrow(/sealSize/) - }) -}) - -describe('fact-log format v2 — decoder law (typed refusals, never skip)', () => { - it('unknown record type 12 throws UnknownLogRecordError naming type 12', () => { - const payload = encode([1, 1, [[12, 1]], null, null]) - expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError) - try { - decodeFact(payload, 2) - expect.unreachable('decode must throw') - } catch (error) { - const typed = error as UnknownLogRecordError - expect(typed).toBeInstanceOf(UnknownLogRecordError) - expect(typed.recordType).toBe(12) - expect(typed.recordVersion).toBe(1) - expect(typed.message).toMatch(/type 12/) - expect(typed.message).toMatch(/newer reader/) - } - }) - - it('recordVersion 2 on a known type throws the same class naming the version', () => { - const payload = encode([1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 2, new Uint8Array(16)]], null, null]) - try { - decodeFact(payload, 2) - expect.unreachable('decode must throw') - } catch (error) { - const typed = error as UnknownLogRecordError - expect(typed).toBeInstanceOf(UnknownLogRecordError) - expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE) - expect(typed.recordVersion).toBe(2) - expect(typed.message).toMatch(/version 2/) - expect(typed.message).toMatch(/newer reader/) - } - }) - - it('a fact mixing known and unknown records still refuses (no partial reads)', () => { - const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))] - const payload = encode([1, 1, [known, [200, 1]], null, null]) - expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError) - }) - - it('a nonzero cipherFlag refuses with the typed error — encrypted records need a newer reader', () => { - const payload = encode( - [1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 1, null, uuidBytes(UUID(1))]], null, null] - ) - try { - decodeFact(payload, 2) - expect.unreachable('decode must throw') - } catch (error) { - const typed = error as UnknownLogRecordError - expect(typed).toBeInstanceOf(UnknownLogRecordError) - expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE) - expect(typed.recordVersion).toBe(1) - expect(typed.message).toMatch(/cipherFlag 1/) - expect(typed.message).toMatch(/encrypted records need a newer reader/) - } - }) - - it('a non-nil keyId refuses the same way, even with cipherFlag 0', () => { - const payload = encode( - [ - 1, - 1, - [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, uuidBytes(UUID(9)), uuidBytes(UUID(1))]], - null, - null - ] - ) - expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError) - expect(() => decodeFact(payload, 2)).toThrow(/encrypted records need a newer reader/) - }) - - it('the encoder always writes the plaintext envelope: cipherFlag 0, keyId nil', () => { - const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) }))) - const raw = decode(payload) as unknown[] - const record = (raw[2] as unknown[][])[0] - expect(record[2]).toBe(LOG_RECORD_CIPHER_PLAINTEXT) - expect(record[3]).toBeNull() - expect(LOG_RECORD_CIPHER_PLAINTEXT).toBe(0) - }) - - it('an unknown segment format version has no decode path', () => { - const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) }))) - expect(() => decodeFact(payload, 3)).toThrow(/reads 1 and 2/) - }) -}) - -describe('fact-log format v2 — log.genesis width law', () => { - const genesisFact = (width: 32 | 64): CommitFactV2 => - factOf(1, { type: 'log.genesis', idSpaceWidth: width, brainId: UUID(9), createdAt: 1 }) - - it('expectedWidth 32 vs a 64-width genesis refuses, naming both widths', () => { - const payload = framePayload(encodeFactV2(genesisFact(64))) - expect(() => decodeFact(payload, 2, { expectedIdSpaceWidth: 32 })).toThrow( - GenesisWidthMismatchError - ) - try { - decodeFact(payload, 2, { expectedIdSpaceWidth: 32 }) - expect.unreachable('decode must throw') - } catch (error) { - const typed = error as GenesisWidthMismatchError - expect(typed.expectedWidth).toBe(32) - expect(typed.actualWidth).toBe(64) - expect(typed.message).toMatch(/32-bit/) - expect(typed.message).toMatch(/64-bit/) - } - }) - - it('a matching width (and no expectation at all) decodes cleanly', () => { - const payload = framePayload(encodeFactV2(genesisFact(64))) - expect(decodeFact(payload, 2, { expectedIdSpaceWidth: 64 }).records[0]).toMatchObject({ - idSpaceWidth: 64 - }) - expect(decodeFact(payload, 2).records[0]).toMatchObject({ idSpaceWidth: 64 }) - }) - - it('genesis anywhere but record 0 refuses — encode AND decode', () => { - const late: CommitFactV2 = { - generation: 1, - timestamp: 1, - records: [ - { type: 'noun.tombstone', id: UUID(1) }, - { type: 'log.genesis', idSpaceWidth: 64, brainId: UUID(9), createdAt: 1 } - ] - } - expect(() => encodeFactV2(late)).toThrow(/first record/) - const crafted = encode([ - 1, - 1, - [ - [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))], - [LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 64, uuidBytes(UUID(9)), 1] - ], - null, - null - ]) - expect(() => decodeFact(crafted, 2)).toThrow(/first record/) - }) - - it('an invalid genesis width on the wire is malformed, not a mismatch', () => { - const crafted = encode( - [1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 48, uuidBytes(UUID(9)), 1]], null, null] - ) - expect(() => decodeFact(crafted, 2)).toThrow(/32 or 64/) - }) -}) - -describe('fact-log format v2 — vector legs (single-hop law)', () => { - it('inline vectors round-trip float-exact', () => { - const vector = [0.1 + 0.2, -0.0000001, 3.141592653589793, 2 ** 40 + 0.25] - const fact = factOf(1, { - type: 'noun.afterImage', - id: UUID(1), - entityInt: 1n, - metadata: null, - vectorLeg: vector - }) - const decoded = roundTrip(fact) - expect((decoded.records[0] as { vectorLeg: number[] }).vectorLeg).toStrictEqual(vector) - }) - - it('a ref round-trips when the validator vouches for the target generation', () => { - const fact = factOf(6, { - type: 'noun.afterImage', - id: UUID(1), - entityInt: 1n, - metadata: null, - vectorLeg: { sameAsGeneration: 5 } - }) - const viaSet = roundTrip(fact, { inlineVectorGenerations: new Set([5]) }) - expect((viaSet.records[0] as { vectorLeg: VectorRef }).vectorLeg).toStrictEqual({ - sameAsGeneration: 5 - }) - const viaCallback = roundTrip(fact, { inlineVectorGenerations: (g) => g === 5 }) - expect(viaCallback).toStrictEqual(fact) - }) - - it('the encoder REFUSES a ref the validator rejects', () => { - const fact = factOf(6, { - type: 'noun.afterImage', - id: UUID(1), - entityInt: 1n, - metadata: null, - vectorLeg: { sameAsGeneration: 5 } - }) - expect(() => encodeFactV2(fact, { inlineVectorGenerations: new Set([4]) })).toThrow( - /single-hop/ - ) - expect(() => encodeFactV2(fact, { inlineVectorGenerations: () => false })).toThrow( - /generation 5/ - ) - }) - - it('the encoder REFUSES a ref when no validator was provided at all', () => { - const fact = factOf(6, { - type: 'noun.afterImage', - id: UUID(1), - entityInt: 1n, - metadata: null, - vectorLeg: { sameAsGeneration: 5 } - }) - expect(() => encodeFactV2(fact)).toThrow(/unverifiable ref/) - }) - - it('embed.landed is inline-only: encode refuses non-arrays, decode refuses wire refs', () => { - const bad = factOf(7, { - type: 'embed.landed', - id: UUID(7), - vector: null as unknown as number[] - }) - expect(() => encodeFactV2(bad)).toThrow(/INLINE/) - const craftedRef = encode( - [1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, 0, null, uuidBytes(UUID(7)), ['ref', 5]]], null, null] - ) - expect(() => decodeFact(craftedRef, 2)).toThrow(/INLINE/) - }) -}) - -describe('fact-log format v2 — sector seals', () => { - const facts = [1, 2, 3].map((g) => - factOf(g, { - type: 'noun.afterImage', - id: UUID(g), - entityInt: BigInt(g), - metadata: { title: `doc ${g}` }, - vectorLeg: [g + 0.5] - }) - ) - const frames = facts.map((f) => encodeFactV2(f)) - - it('sealGroup output is sector-aligned and decodes to exactly the input facts', () => { - const sealed = sealGroup(frames, 4096) - expect(sealed.length % 4096).toBe(0) - const { facts: decoded, validBytes } = decodeGroupV2(sealed) - expect(decoded).toStrictEqual(facts) // pads invisible - expect(validBytes).toBe(sealed.length) - }) - - it('an already-aligned group gets NO pad (byte-identical passthrough)', () => { - const exact = frameOfExactly(4096, 1) - const sealed = sealGroup([exact], 4096) - expect(sealed.length).toBe(4096) - expect(Buffer.compare(Buffer.from(sealed), Buffer.from(exact))).toBe(0) - expect(decodeGroupV2(sealed).facts).toHaveLength(1) - }) - - it('a normal gap gets ONE exact-fit pad frame', () => { - const sealed = sealGroup([frameOfExactly(2000, 1), frameOfExactly(1996, 2)], 4096) // gap 100 - expect(sealed.length).toBe(4096) - expect(decodeGroupV2(sealed).facts.map((f) => f.generation)).toEqual([1, 2]) - }) - - it('a gap too small for any frame (the <12-byte remainder and friends) pads through one extra sector', () => { - for (const gap of [1, 8, 11, 16, 32]) { - const sealed = sealGroup([frameOfExactly(4096 - gap, 1)], 4096) - expect(sealed.length % 4096).toBe(0) - expect(sealed.length).toBe(8192) // gap + one full sector, still aligned - const { facts: decoded, validBytes } = decodeGroupV2(sealed) - expect(decoded.map((f) => f.generation)).toEqual([1]) - expect(validBytes).toBe(8192) - } - // the smallest constructible pad frame fits exactly — no overshoot at 33 - const sealed33 = sealGroup([frameOfExactly(4096 - 33, 1)], 4096) - expect(sealed33.length).toBe(4096) - expect(decodeGroupV2(sealed33).facts.map((f) => f.generation)).toEqual([1]) - }) - - it('seals honor a custom sealSize (device-probed sizes are the caller business)', () => { - const sealed = sealGroup(frames, 512) - expect(sealed.length % 512).toBe(0) - expect(decodeGroupV2(sealed).facts).toStrictEqual(facts) - }) - - it('pad frame bytes are pinned (golden vector, sealSize 64)', () => { - const tomb = encodeFactV2({ - generation: 3, - timestamp: 1_700_000_000_123, - records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }] - }) - const sealed = sealGroup([tomb], 64) // 53 bytes → gap 11 → overshoot → 75-byte pad - expect(sealed.length).toBe(128) - expect(hex(sealed.subarray(tomb.length))).toBe( - // frame prefix + [0, 0, [[0, 1, bin8(40 zero bytes)]], nil, nil] - '4300000088b4c8fa95cf0000000000000000cf000000000000000091930001c428' + - '0'.repeat(80) + - 'c0c0' - ) - }) - - it('encodePadFrame builds exact-size pads for streaming writers; refuses sub-minimum sizes', () => { - // Pads are envelope-exempt (skipped wholesale), so the smallest pad frame - // is byte-stable across the crypto-envelope change. - expect(minPadFrameBytes()).toBe(33) - for (const size of [minPadFrameBytes(), 64, 4096]) { - const pad = encodePadFrame(size) - expect(pad.length).toBe(size) - const { facts: decoded, validBytes } = decodeGroupV2(pad) - expect(decoded).toEqual([]) // invisible to readers - expect(validBytes).toBe(size) - } - expect(() => encodePadFrame(minPadFrameBytes() - 1)).toThrow(/at least/) - }) - - it('sealGroup refuses garbage: empty groups, malformed frames, bad seal sizes', () => { - expect(() => sealGroup([], 4096)).toThrow(/at least one frame/) - expect(() => sealGroup([new Uint8Array([1, 2, 3])], 4096)).toThrow(/not a well-formed frame/) - const corrupted = encodeFactV2(facts[0]) - corrupted[corrupted.length - 1] ^= 0xff - expect(() => sealGroup([corrupted], 4096)).toThrow(/not a well-formed frame/) - expect(() => sealGroup(frames, 32)).toThrow(/sealSize/) - }) -}) - -describe('fact-log format v2 — torn-tail discipline', () => { - it('truncating a sealed group at EVERY byte offset of the tail yields the intact prefix, never an uncontrolled throw', () => { - const frames = [frameOfExactly(600, 1), frameOfExactly(700, 2), frameOfExactly(800, 3)] - const sealed = sealGroup(frames, 4096) - expect(sealed.length).toBe(4096) - const f3End = 600 + 700 + 800 - - for (let cut = 600 + 700; cut < sealed.length; cut++) { - const { facts: decoded, validBytes } = decodeGroupV2(sealed.subarray(0, cut)) - const expected = cut < f3End ? [1, 2] : [1, 2, 3] - expect(decoded.map((f) => f.generation)).toEqual(expected) - expect(validBytes).toBe(cut < f3End ? 600 + 700 : f3End) - } - }) - - it('a flipped payload byte (not just truncation) also terminates the walk at the damage', () => { - const frames = [frameOfExactly(600, 1), frameOfExactly(700, 2)] - const sealed = sealGroup(frames, 4096) - const damaged = sealed.slice() - damaged[600 + 100] ^= 0xff // inside frame 2's payload - const { facts: decoded, validBytes } = decodeGroupV2(damaged) - expect(decoded.map((f) => f.generation)).toEqual([1]) - expect(validBytes).toBe(600) - }) -}) - -describe('fact-log format v2 — writer refusals (loud, never silent)', () => { - const tombstone = (g: number): CommitFactV2 => factOf(g, { type: 'noun.tombstone', id: UUID(g) }) - - it('accepts empty records (an all-deduped batch is a real generation); refuses generation 0 and a second batch.meta', () => { - // Contract change with the live cutover: v1 always encoded op-less - // commits (a batch whose relates dedupe away still mints a generation); - // v2 must not fork commit semantics — empty records round-trip. - const empty = decodeFact(framePayload(encodeFactV2({ generation: 1, timestamp: 1, records: [] })), 2) - expect(empty.records).toEqual([]) - expect(() => encodeFactV2({ ...tombstone(1), generation: 0 })).toThrow(/positive integer/) - expect(() => - encodeFactV2({ - generation: 1, - timestamp: 1, - records: [ - { type: 'batch.meta', meta: { a: 1 } }, - { type: 'batch.meta', meta: { b: 2 } } - ] - }) - ).toThrow(/at most one batch.meta/) - }) - - it('refuses pad records — filler belongs to sealGroup, not to writers', () => { - const fact = { - generation: 1, - timestamp: 1, - records: [{ type: 'pad' } as unknown as LogRecord] - } - expect(() => encodeFactV2(fact)).toThrow(/cannot encode record type pad/) - }) - - it('refuses malformed field values: non-uuid ids, bad hashes, out-of-range u64s', () => { - expect(() => - encodeFactV2(factOf(1, { type: 'noun.tombstone', id: 'not-a-uuid' })) - ).toThrow(/not a uuid/) - expect(() => - encodeFactV2( - factOf(1, { type: 'blob.manifest', hash: 'abc', size: 1, mimeType: 'x', refOp: 'add' }) - ) - ).toThrow(/64 hex chars/) - expect(() => - encodeFactV2( - factOf(1, { - type: 'noun.afterImage', - id: UUID(1), - entityInt: -1n, - metadata: null, - vectorLeg: null - }) - ) - ).toThrow(/u64 range/) - expect(() => - encodeFactV2( - factOf(1, { - type: 'noun.afterImage', - id: UUID(1), - entityInt: 1n << 64n, - metadata: null, - vectorLeg: null - }) - ) - ).toThrow(/u64 range/) - }) -}) - -describe('fact-log format — the v1 decode path stays readable forever', () => { - it('decodeFact(payload, 1) reads the v1 ops shape (positional, bin16, tombstones)', () => { - // Crafted exactly as the v1 writer frames facts: default msgpack, ops at - // position 2 as [kind u8, id bin16, [metadata, vector] | nil]. - const payload = encode([ - 4, - 1_700_000_000_004, - [ - [0, uuidBytes(UUID(41)), [{ noun: 'document', title: 'doc 41' }, { v: [1, 2] }]], - [1, uuidBytes(UUID(42)), null] // verb tombstone - ], - { source: 'v1' }, - ['abc123'] - ]) - const fact = decodeFact(payload, 1) - expect(fact).toStrictEqual({ - generation: 4, - timestamp: 1_700_000_000_004, - ops: [ - { - kind: 'noun', - id: UUID(41), - record: { metadata: { noun: 'document', title: 'doc 41' }, vector: { v: [1, 2] } } - }, - { kind: 'verb', id: UUID(42), record: null } - ], - meta: { source: 'v1' }, - blobHashes: ['abc123'] - }) - }) -}) - -describe('fact-log format v2 — frame envelope helper', () => { - it('framePayload verifies exact length and crc32c', () => { - const frame = encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })) - expect(() => framePayload(frame)).not.toThrow() - - const shortFrame = frame.subarray(0, frame.length - 1) - expect(() => framePayload(shortFrame)).toThrow(/declares/) - - const corrupted = frame.slice() - corrupted[corrupted.length - 1] ^= 0xff - expect(() => framePayload(corrupted)).toThrow(/crc32c/) - }) - - it('the record-type registry and version constants are the frozen wire codes', () => { - expect(LOG_RECORD_TYPES).toStrictEqual({ - PAD: 0, - NOUN_AFTER_IMAGE: 1, - NOUN_TOMBSTONE: 2, - VERB_AFTER_IMAGE: 3, - VERB_TOMBSTONE: 4, - BATCH_META: 5, - EMBED_PENDING: 6, - EMBED_LANDED: 7, - BLOB_MANIFEST: 8, - PROJECTION_NOTE: 9, - BOOTSTRAP_BASELINE: 10, - LOG_GENESIS: 11 - }) - expect(LOG_RECORD_VERSION).toBe(1) - }) -}) diff --git a/tests/unit/db/fault-injection-shim.test.ts b/tests/unit/db/fault-injection-shim.test.ts deleted file mode 100644 index a6d4109e..00000000 --- a/tests/unit/db/fault-injection-shim.test.ts +++ /dev/null @@ -1,231 +0,0 @@ -/** - * @module tests/unit/db/fault-injection-shim - * @description The fault-injection storage wrapper proven in isolation: a - * torn write persists a decodable prefix (the crash shape durability tests - * replay), a dropped sync is observable (armed → the inner adapter never sees - * it; journaled), a failed append throws without writing a byte, knobs are - * one-shot, and unarmed operation is a transparent passthrough. The full - * commit-path fault matrix lives with the log's ack work — this file proves - * the SHIM itself. - */ -import { describe, it, expect, beforeEach } from 'vitest' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { - FactLog, - storageSupportsFactLog, - type CommitFact, - type FactLogStorage -} from '../../../src/db/factLog.js' -import { - FaultInjectionStorage, - FaultInjectedError -} from '../../../src/db/faultInjectionStorage.js' -import { - encodeFactV2, - encodeSegmentHeaderV2, - decodeGroupV2, - parseSegmentHeader, - SEGMENT_HEADER_BYTES, - type CommitFactV2 -} from '../../../src/db/factLogFormat.js' - -const UUID = (n: number): string => - `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` - -const factV2 = (generation: number): CommitFactV2 => ({ - generation, - timestamp: 1_700_000_000_000 + generation, - records: [{ type: 'noun.tombstone', id: UUID(generation) }] -}) - -const factV1 = (generation: number): CommitFact => ({ - generation, - timestamp: 1_700_000_000_000 + generation, - ops: [ - { - kind: 'noun', - id: UUID(generation), - record: { metadata: { noun: 'document' }, vector: null } - } - ] -}) - -describe('fault-injection storage wrapper', () => { - let inner: FactLogStorage & { syncRawObjects: (paths: string[]) => Promise } - let shim: FaultInjectionStorage - let innerSyncCalls: string[][] - - beforeEach(async () => { - const mem: any = new MemoryStorage() - await mem.init() - innerSyncCalls = [] - const realSync = mem.syncRawObjects.bind(mem) - mem.syncRawObjects = async (paths: string[]) => { - innerSyncCalls.push([...paths]) - return realSync(paths) - } - inner = mem - shim = new FaultInjectionStorage(inner) - }) - - it('satisfies the fact-log storage surface (drop-in wrapper)', () => { - expect(storageSupportsFactLog(shim)).toBe(true) - }) - - it('unarmed, every operation is a transparent passthrough', async () => { - await shim.writeRawBytes('seg', new Uint8Array([1, 2, 3])) - await shim.appendRawBytes('seg', new Uint8Array([4, 5])) - expect(Array.from((await shim.readRawBytes('seg'))!)).toEqual([1, 2, 3, 4, 5]) - expect(await shim.rawByteSize('seg')).toBe(5) - expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1, 2, 3, 4, 5]) - - await shim.writeRawObject('obj.json', { a: 1 }) - expect(await shim.readRawObject('obj.json')).toEqual({ a: 1 }) - await shim.deleteRawObject('obj.json') - expect(await shim.readRawObject('obj.json')).toBeNull() - - await shim.syncRawObjects(['seg']) - expect(innerSyncCalls).toEqual([['seg']]) - expect(shim.injectedFaults).toEqual([]) - }) - - describe('tearWriteAtByte — a torn write produces a decodable-prefix segment', () => { - it('persists only the first N bytes of the next append; the prefix decodes intact', async () => { - const path = 'facts/seg-test.bfl' - const frame1 = encodeFactV2(factV2(1)) - const frame2 = encodeFactV2(factV2(2)) - - await shim.appendRawBytes(path, encodeSegmentHeaderV2(1, 4096)) - await shim.appendRawBytes(path, frame1) - shim.tearWriteAtByte(frame2.length - 5) // crash 5 bytes before the frame lands - await shim.appendRawBytes(path, frame2) // reports success — the tear is silent - - const bytes = (await inner.readRawBytes(path))! - expect(bytes.length).toBe(SEGMENT_HEADER_BYTES + frame1.length + frame2.length - 5) - - // The "crash": reopen from storage and read what actually survived. - const header = parseSegmentHeader(bytes) - expect(header).toStrictEqual({ formatVersion: 2, firstGeneration: 1, sealSize: 4096 }) - const { facts, validBytes } = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES)) - expect(facts.map((f) => f.generation)).toEqual([1]) // fact 2's torn frame is invisible - expect(validBytes).toBe(frame1.length) - - expect(shim.injectedFaults).toEqual([ - { - kind: 'torn-write', - path, - requestedBytes: frame2.length, - writtenBytes: frame2.length - 5 - } - ]) - }) - - it('a tear inside the frame prefix (first bytes) leaves the earlier facts intact too', async () => { - const path = 'facts/seg-prefix.bfl' - const frame1 = encodeFactV2(factV2(1)) - await shim.appendRawBytes(path, encodeSegmentHeaderV2(1, 4096)) - await shim.appendRawBytes(path, frame1) - shim.tearWriteAtByte(3) - await shim.appendRawBytes(path, encodeFactV2(factV2(2))) - - const bytes = (await inner.readRawBytes(path))! - const { facts } = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES)) - expect(facts.map((f) => f.generation)).toEqual([1]) - }) - - it('a tear at byte 0 writes nothing at all', async () => { - shim.tearWriteAtByte(0) - await shim.appendRawBytes('empty.bfl', new Uint8Array([1, 2, 3])) - expect(await inner.readRawBytes('empty.bfl')).toBeNull() - expect(shim.injectedFaults[0]).toMatchObject({ kind: 'torn-write', writtenBytes: 0 }) - }) - - it('is one-shot: the append after the torn one lands whole', async () => { - shim.tearWriteAtByte(1) - await shim.appendRawBytes('seg', new Uint8Array([1, 2, 3, 4])) - await shim.appendRawBytes('seg', new Uint8Array([5, 6])) - expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1, 5, 6]) - }) - - it('refuses a negative tear offset', () => { - expect(() => shim.tearWriteAtByte(-1)).toThrow(/non-negative/) - }) - }) - - describe('dropNextSync — a dropped sync is observable', () => { - it('the armed sync never reaches the inner adapter and is journaled', async () => { - shim.dropNextSync() - await shim.syncRawObjects(['a.bfl', 'b.bfl']) - expect(innerSyncCalls).toEqual([]) // the device never saw it - expect(shim.injectedFaults).toEqual([{ kind: 'dropped-sync', paths: ['a.bfl', 'b.bfl'] }]) - }) - - it('is one-shot: the following sync passes through', async () => { - shim.dropNextSync() - await shim.syncRawObjects(['x']) - await shim.syncRawObjects(['y']) - expect(innerSyncCalls).toEqual([['y']]) - }) - }) - - describe('failNextAppend — a failed append throws without writing a byte', () => { - it('throws the typed error, writes nothing, and journals the fault', async () => { - await shim.appendRawBytes('seg', new Uint8Array([1])) - shim.failNextAppend() - await expect(shim.appendRawBytes('seg', new Uint8Array([2, 3]))).rejects.toThrow( - FaultInjectedError - ) - expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1]) // untouched - expect(shim.injectedFaults).toEqual([{ kind: 'failed-append', path: 'seg' }]) - // one-shot: the next append succeeds - await shim.appendRawBytes('seg', new Uint8Array([4])) - expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([1, 4]) - }) - - it('carries the operation and path for programmatic assertions', async () => { - shim.failNextAppend() - try { - await shim.appendRawBytes('some/path.bfl', new Uint8Array([1])) - expect.unreachable('append must throw') - } catch (error) { - const typed = error as FaultInjectedError - expect(typed).toBeInstanceOf(FaultInjectedError) - expect(typed.operation).toBe('append') - expect(typed.path).toBe('some/path.bfl') - } - }) - - it('wins over a simultaneously-armed tear; the tear stays pending for the next append', async () => { - shim.failNextAppend() - shim.tearWriteAtByte(2) - await expect(shim.appendRawBytes('seg', new Uint8Array([1, 2, 3]))).rejects.toThrow( - FaultInjectedError - ) - expect(await inner.readRawBytes('seg')).toBeNull() - await shim.appendRawBytes('seg', new Uint8Array([9, 8, 7])) - expect(Array.from((await inner.readRawBytes('seg'))!)).toEqual([9, 8]) // torn at 2 - expect(shim.injectedFaults.map((f) => f.kind)).toEqual(['failed-append', 'torn-write']) - }) - }) - - describe('composed with the real fact log (v1 surface)', () => { - it('a torn append is truncated away on reopen — the log heals to the intact prefix', async () => { - const log = new FactLog(shim) - await log.open(0) - await log.append(factV1(1)) - await log.sync() - - shim.tearWriteAtByte(10) // fact 2's frame lands 10 bytes long — torn - await log.append(factV1(2)) - await log.sync() - - // The crash: abandon the instance, reopen from what storage actually holds. - const reopened = new FactLog(inner) - await reopened.open(2) // generation 2 committed elsewhere — but its fact is torn - expect(reopened.headGeneration()).toBe(1) - const all: CommitFact[] = [] - for await (const batch of reopened.scanFacts().batches()) all.push(...batch.facts) - expect(all.map((f) => f.generation)).toEqual([1]) - }) - }) -}) diff --git a/tests/unit/db/log-authority-oracle-verbs.test.ts b/tests/unit/db/log-authority-oracle-verbs.test.ts deleted file mode 100644 index 68da1867..00000000 --- a/tests/unit/db/log-authority-oracle-verbs.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -/** - * @module tests/unit/db/log-authority-oracle-verbs - * @description The verification oracle's VERB legs — module-level pins with - * doubles (the brain-level wiring rides the owner's call site): - * 1. Wired verb legs diff verbs exactly like nouns (pre-log / state-differs / - * tombstone-vs-present / log-live-absent). - * 2. UNWIRED verb legs = an HONEST PARTIAL verdict: verbsChecked stays 0 — - * the oracle never claims scope it did not scan. - */ -import { describe, it, expect } from 'vitest' -import { runLogCompletenessOracle, recordDigest } from '../../../src/db/logAuthority.js' -import type { FactScanHandle } from '../../../src/db/factLog.js' - -type Op = { kind: 'noun' | 'verb'; id: string; record: { metadata: unknown; vector: unknown } | null } - -function scanOf(facts: Array<{ generation: number; ops: Op[] }>): () => FactScanHandle | null { - return () => - ({ - batches: async function* () { - yield { facts: facts.map((f) => ({ ...f, timestamp: 0 })) } - } - }) as unknown as FactScanHandle -} - -function pagedList(rows: string[]) { - return async ({ pagination }: { pagination: { limit: number; offset?: number } }) => { - const start = pagination.offset ?? 0 - const items = rows.slice(start, start + pagination.limit).map((id) => ({ id })) - return { items, hasMore: start + pagination.limit < rows.length } - } -} - -const rec = (v: number) => ({ metadata: { v }, vector: null }) - -describe('oracle verb legs', () => { - it('wired: verbs diff by digest — clean log goes green over nouns AND verbs', async () => { - const report = await runLogCompletenessOracle({ - storage: { getNouns: pagedList(['n1']) } as never, - scanFacts: scanOf([ - { generation: 1, ops: [{ kind: 'noun', id: 'n1', record: rec(1) }] }, - { generation: 2, ops: [{ kind: 'verb', id: 'v1', record: rec(7) }] } - ]), - canonicalNounDigest: async () => recordDigest(rec(1)), - factRecordDigest: recordDigest, - canonicalVerbDigest: async () => recordDigest(rec(7)), - getVerbs: pagedList(['v1']) - }) - expect(report.verdict).toBe('green') - expect(report.nounsChecked).toBe(1) - expect(report.verbsChecked).toBe(1) - expect(report.matched).toBe(2) - }) - - it('wired: every verb divergence class is NAMED', async () => { - const report = await runLogCompletenessOracle({ - storage: { getNouns: pagedList([]) } as never, - scanFacts: scanOf([ - { - generation: 1, - ops: [ - { kind: 'verb', id: 'v-differs', record: rec(1) }, - { kind: 'verb', id: 'v-tomb', record: null }, - { kind: 'verb', id: 'v-orphan', record: rec(3) } - ] - } - ]), - canonicalNounDigest: async () => null, - factRecordDigest: recordDigest, - canonicalVerbDigest: async (id) => - id === 'v-differs' ? recordDigest(rec(999)) : id === 'v-tomb' ? recordDigest(rec(2)) : null, - // canonical enumerates: v-differs (drifted), v-tomb (log says deleted), - // v-prelog (never logged); v-orphan is log-live but canonical-absent. - getVerbs: pagedList(['v-differs', 'v-tomb', 'v-prelog']) - }) - expect(report.verdict).toBe('red') - const by = (id: string) => report.mismatches.find((m) => m.id === id) - expect(by('v-differs')).toMatchObject({ kind: 'verb', reason: 'state-differs' }) - expect(by('v-tomb')).toMatchObject({ kind: 'verb', reason: 'log-tombstone-canonical-present' }) - expect(by('v-prelog')).toMatchObject({ kind: 'verb', reason: 'pre-log-record' }) - expect(by('v-orphan')).toMatchObject({ kind: 'verb', reason: 'log-live-canonical-absent' }) - }) - - it('unwired: verbsChecked stays 0 — honest partial scope, never a silent claim', async () => { - const report = await runLogCompletenessOracle({ - storage: { getNouns: pagedList(['n1']) } as never, - scanFacts: scanOf([ - { generation: 1, ops: [{ kind: 'noun', id: 'n1', record: rec(1) }] }, - { generation: 2, ops: [{ kind: 'verb', id: 'v1', record: rec(7) }] } - ]), - canonicalNounDigest: async () => recordDigest(rec(1)), - factRecordDigest: recordDigest - }) - expect(report.verbsChecked).toBe(0) - expect(report.nounsChecked).toBe(1) - }) -}) diff --git a/tests/unit/db/torn-open-guards.test.ts b/tests/unit/db/torn-open-guards.test.ts deleted file mode 100644 index 77c1b8b4..00000000 --- a/tests/unit/db/torn-open-guards.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -/** - * @module tests/unit/db/torn-open-guards - * @description Power-cut throw-site cures (brainy-alone fault-injection - * findings, both release-gating): - * 1. A torn generation manifest/counter (NaN/garbage where a generation - * belongs) DISCARDS with narration and re-derives — never a RangeError - * killing the open. - * 2. A manifest-listed-but-unloadable column segment QUARANTINES at - * discovery with narration; the field serves its remaining segments - * DEGRADED — never a raw throw killing every query on the field. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { mkdtempSync, rmSync, readdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { gzipSync } from 'node:zlib' -import { Brainy } from '../../../src/index.js' -import { NounType } from '../../../src/types/graphTypes.js' - -const dirs: string[] = [] -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) - for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) -}) - -async function open(dir: string): Promise { - const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) - await b.init() - brains.push(b) - return b -} - -describe('torn-open guards', () => { - it('a torn generation manifest (NaN) opens with narrated discard — never a RangeError', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-torn-gen-')) - dirs.push(dir) - let brain = await open(dir) - const id = await brain.add({ data: 'survivor row', type: NounType.Document, metadata: { k: 1 } }) - await brain.flush() - await brain.close() - brains.pop() - - // The power-cut shape: the manifest's generation field is garbage. - const sys = join(dir, '_system') - const manifestPath = ['manifest.json', 'manifest.json.gz'] - .map((f) => join(sys, f)) - .find((p) => existsSync(p))! - const torn = { version: 1, generation: 'NaN-garbage', committedAt: 'x', horizon: null } - if (manifestPath.endsWith('.gz')) writeFileSync(manifestPath, gzipSync(JSON.stringify(torn))) - else writeFileSync(manifestPath, JSON.stringify(torn)) - - // Open MUST succeed (narrated discard + recovery re-derivation), and the - // durable row must still serve (log-authority replay recovers it). - brain = await open(dir) - expect((await brain.get(id))!.data).toContain('survivor row') - // Writes continue with a sane monotonic generation. - await brain.add({ data: 'post-recovery', type: NounType.Document, metadata: { k: 2 } }) - expect(Number.isSafeInteger(brain.generation())).toBe(true) - }, 120000) - - it('a torn column segment quarantines at discovery; the field serves remaining segments degraded — never a raw throw', async () => { - const dir = mkdtempSync(join(tmpdir(), 'brainy-torn-seg-')) - dirs.push(dir) - let brain = await open(dir) - for (let i = 0; i < 6; i++) { - await brain.add({ data: `row ${i}`, type: NounType.Document, metadata: { bucket: i % 2 } }) - } - await brain.flush() - await brain.close() - brains.pop() - - // Tear ONE column segment's bytes on disk (manifest keeps listing it) — - // the QUERIED field's own segment, so the quarantine path provably - // engages. Column segments live under the raw-blob root: - // `/_blobs/_column_index//L-.bin`. - const segDir = join(dir, '_blobs', '_column_index', 'bucket') - let tornOne = false - if (existsSync(segDir)) { - for (const f of readdirSync(segDir, { withFileTypes: true })) { - if (!f.isDirectory() && /^L\d+-.*\.bin$/.test(f.name)) { - writeFileSync(join(segDir, f.name), Buffer.from([0x00, 0x01, 0x02])) // garbage - tornOne = true - break - } - } - } - expect(tornOne, 'found a segment file to tear (layout probe)').toBe(true) - - // Queries on the field MUST NOT throw — degraded-announced service. - brain = await open(dir) - const rows = await brain.find({ where: { bucket: 0 }, limit: 10 }) - expect(Array.isArray(rows), 'query survives the torn segment').toBe(true) - // Full completeness is NOT asserted (the torn segment's rows may be - // absent — that is the documented degraded contract until heal). - }, 120000) -}) diff --git a/tests/unit/graph/graph-adjacency-watermark.test.ts b/tests/unit/graph/graph-adjacency-watermark.test.ts deleted file mode 100644 index 8298277e..00000000 --- a/tests/unit/graph/graph-adjacency-watermark.test.ts +++ /dev/null @@ -1,213 +0,0 @@ -/** - * @module tests/unit/graph/graph-adjacency-watermark - * @description Watermark-stamp pins for the graph-adjacency projection. - * - * THE LAW under test: the persisted adjacency artifact (the two verb-id LSM - * trees' SSTables + manifests) carries a stamp asserting "this state - * reflects every committed generation ≤ W and nothing above W" — written - * AFTER both trees' flushes complete — and init() computes the three-way - * verdict: stamped==committed → 'adopt' · stampedcommitted OR unstamped → 'rescan', LOUDLY. - * - * The verdict is COMPUTED AND EXPOSED only — cold-load recovery and rebuild - * triggers are unchanged. - */ -import { describe, it, expect, vi, afterEach } from 'vitest' -import { v4 as uuidv4 } from 'uuid' -import { - GraphAdjacencyIndex, - GRAPH_ADJACENCY_STAMP_KEY -} from '../../../src/graph/graphAdjacencyIndex.js' -import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { VerbType } from '../../../src/types/graphTypes.js' -import type { GraphVerb } from '../../../src/coreTypes.js' -import { prodLog } from '../../../src/utils/logger.js' - -function makeVerb(id: string, sourceId: string, targetId: string): GraphVerb { - return { - id, - sourceId, - targetId, - vector: [], - type: VerbType.RelatedTo, - verb: VerbType.RelatedTo - } -} - -async function makeStorage(committed: number | null): Promise { - const storage = new MemoryStorage() - await storage.init() - if (committed !== null) { - vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed) - } - return storage -} - -function setCommitted(storage: MemoryStorage, committed: number): void { - vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed) -} - -/** Session 1: index verbs, optionally stamp, flush + close — the artifact. */ -async function writeArtifact(storage: MemoryStorage, stamp: number | null): Promise { - const idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' }) - await idMapper.init() - const index = new GraphAdjacencyIndex(storage, {}, idMapper) - const a = uuidv4() - const b = uuidv4() - const aInt = BigInt(idMapper.getOrAssign(a)) - const bInt = BigInt(idMapper.getOrAssign(b)) - await index.addVerb(makeVerb(uuidv4(), a, b), aInt, bInt, 1n) - if (stamp !== null) index.stampWatermark(stamp) - await index.flush() - await index.close() -} - -/** Session 2: reopen on the same storage via the cold-load path. */ -async function reopen(storage: MemoryStorage): Promise { - const index = new GraphAdjacencyIndex(storage) - await index.init() - return index -} - -afterEach(() => { - vi.restoreAllMocks() -}) - -describe('graph adjacency index — watermark stamp + three-way load verdict', () => { - it("save-with-stamp then reopen at the same committed generation → 'adopt'", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('adopt') - expect(index.watermark()).toBe(5) - expect(index.watermarkGap()).toBeNull() - await index.close() - }) - - it("stamp BEHIND the committed generation → 'catchup' with the exact gap reported", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - - setCommitted(storage, 11) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('catchup') - expect(index.watermark()).toBe(5) - expect(index.watermarkGap()).toEqual({ from: 5, to: 11 }) - await index.close() - }) - - it("stamp ABOVE the committed generation → 'rescan', said out loud", async () => { - const storage = await makeStorage(9) - await writeArtifact(storage, 9) - - setCommitted(storage, 4) - - const warnSpy = vi.spyOn(prodLog, 'warn') - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('rescan') - expect(index.watermarkGap()).toBeNull() - const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n') - expect(said).toContain('RESCAN') - expect(said).toContain('ABOVE') - await index.close() - }) - - it("legacy unstamped artifact on a stamped store → 'rescan', LOUD — never a silent adopt", async () => { - const storage = await makeStorage(3) - await writeArtifact(storage, null) // pre-stamp adjacency: SSTables, no stamp - - expect(await storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY)).toBeNull() - - const warnSpy = vi.spyOn(prodLog, 'warn') - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('rescan') - expect(index.watermark()).toBeNull() - const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n') - expect(said).toContain('RESCAN') - expect(said).toContain('unstamped') - await index.close() - }) - - it("a store with no committed-generation capability keeps pre-stamp behavior → 'adopt'", async () => { - const storage = await makeStorage(null) - await writeArtifact(storage, null) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('adopt') - expect(index.watermark()).toBeNull() - await index.close() - }) - - it('STAMP-AFTER-DATA: the stamp is the last saveMetadata of the flush, after both trees’ SSTable + manifest writes', async () => { - const storage = await makeStorage(2) - const idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' }) - await idMapper.init() - const index = new GraphAdjacencyIndex(storage, {}, idMapper) - const a = uuidv4() - const b = uuidv4() - await index.addVerb( - makeVerb(uuidv4(), a, b), - BigInt(idMapper.getOrAssign(a)), - BigInt(idMapper.getOrAssign(b)), - 1n - ) - - const keys: string[] = [] - const originalSave = storage.saveMetadata.bind(storage) - vi.spyOn(storage, 'saveMetadata').mockImplementation(async (id, metadata) => { - keys.push(id) - return originalSave(id, metadata) - }) - - index.stampWatermark(2) - await index.flush() - - const stampAt = keys.indexOf(GRAPH_ADJACENCY_STAMP_KEY) - expect(stampAt, 'stamp record was written').toBeGreaterThanOrEqual(0) - expect(stampAt, 'stamp is the FINAL metadata write of the flush').toBe(keys.length - 1) - // Both trees flushed durable bytes before the stamp landed. - expect( - keys.slice(0, stampAt).some(k => k.startsWith('graph-lsm-verbs-source')), - 'verbs-by-source tree wrote before the stamp' - ).toBe(true) - expect( - keys.slice(0, stampAt).some(k => k.startsWith('graph-lsm-verbs-target')), - 'verbs-by-target tree wrote before the stamp' - ).toBe(true) - - const record = (await storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY)) as { - watermark: number - formatVersion: number - stampedAt: number - } - expect(record.watermark).toBe(2) - expect(record.formatVersion).toBe(1) - expect(typeof record.stampedAt).toBe('number') - - await index.close() - }) - - it('a pending stamp also lands on the close() shutdown path, after the final tree flushes', async () => { - const storage = await makeStorage(6) - const idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' }) - await idMapper.init() - const index = new GraphAdjacencyIndex(storage, {}, idMapper) - const a = uuidv4() - const b = uuidv4() - await index.addVerb( - makeVerb(uuidv4(), a, b), - BigInt(idMapper.getOrAssign(a)), - BigInt(idMapper.getOrAssign(b)), - 1n - ) - - index.stampWatermark(6) - await index.close() // no explicit flush — close() flushes, then stamps - - const record = (await storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY)) as { watermark: number } - expect(record?.watermark).toBe(6) - }) -}) diff --git a/tests/unit/hnsw/hnsw-watermark.test.ts b/tests/unit/hnsw/hnsw-watermark.test.ts deleted file mode 100644 index bf8b8510..00000000 --- a/tests/unit/hnsw/hnsw-watermark.test.ts +++ /dev/null @@ -1,200 +0,0 @@ -/** - * @module tests/unit/hnsw/hnsw-watermark - * @description Watermark-stamp pins for the JS HNSW vector projection. - * - * THE LAW under test: the persisted HNSW artifact (per-node records + the - * entryPoint/maxLevel system record) carries a stamp asserting "this state - * reflects every committed generation ≤ W and nothing above W" — written - * AFTER every byte it certifies is durable — and rebuild() computes the - * three-way verdict: stamped==committed → 'adopt' · stampedcommitted OR unstamped → 'rescan', - * LOUDLY. Vector-bearing stamps carry the model identity this module can - * honestly assert: dimensions only (no embedding-model id is reachable from - * the index module). - * - * The verdict is COMPUTED AND EXPOSED only — no rebuild trigger changed. - */ -import { describe, it, expect, vi, afterEach } from 'vitest' -import { v4 as uuidv4 } from 'uuid' -import { JsHnswVectorIndex, HNSW_INDEX_STAMP_KEY } from '../../../src/hnsw/hnswIndex.js' -import { euclideanDistance } from '../../../src/utils/index.js' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { prodLog } from '../../../src/utils/logger.js' - -const DIM = 8 - -function randomVector(dim: number): number[] { - return Array.from({ length: dim }, () => Math.random() * 2 - 1) -} - -async function makeStorage(committed: number | null): Promise { - const storage = new MemoryStorage() - await storage.init() - if (committed !== null) { - vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed) - } - return storage -} - -function setCommitted(storage: MemoryStorage, committed: number): void { - vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed) -} - -function makeIndex(storage: MemoryStorage): JsHnswVectorIndex { - return new JsHnswVectorIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { useParallelization: false, storage, persistMode: 'deferred' } - ) -} - -/** Session 1: insert nodes, optionally stamp, flush — the durable artifact. */ -async function writeArtifact(storage: MemoryStorage, stamp: number | null): Promise { - const index = makeIndex(storage) - for (let i = 0; i < 3; i++) { - await index.addItem({ id: uuidv4(), vector: randomVector(DIM) }) - } - if (stamp !== null) index.stampWatermark(stamp) - await index.flush() -} - -/** Session 2: reopen on the same storage via the load path (rebuild). */ -async function reopen(storage: MemoryStorage): Promise { - const index = makeIndex(storage) - await index.rebuild() - return index -} - -afterEach(() => { - vi.restoreAllMocks() -}) - -describe('JS HNSW index — watermark stamp + three-way load verdict', () => { - it("save-with-stamp then reopen at the same committed generation → 'adopt'", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('adopt') - expect(index.watermark()).toBe(5) - expect(index.watermarkGap()).toBeNull() - }) - - it("stamp BEHIND the committed generation → 'catchup' with the exact gap reported", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - - setCommitted(storage, 9) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('catchup') - expect(index.watermark()).toBe(5) - expect(index.watermarkGap()).toEqual({ from: 5, to: 9 }) - }) - - it("stamp ABOVE the committed generation → 'rescan', said out loud", async () => { - const storage = await makeStorage(9) - await writeArtifact(storage, 9) - - setCommitted(storage, 4) - - const warnSpy = vi.spyOn(prodLog, 'warn') - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('rescan') - expect(index.watermarkGap()).toBeNull() - const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n') - expect(said).toContain('RESCAN') - expect(said).toContain('ABOVE') - }) - - it("legacy unstamped artifact on a stamped store → 'rescan', LOUD — never a silent adopt", async () => { - const storage = await makeStorage(3) - await writeArtifact(storage, null) // pre-stamp index: data flushed, no stamp - - expect(await storage.getMetadata(HNSW_INDEX_STAMP_KEY)).toBeNull() - - const warnSpy = vi.spyOn(prodLog, 'warn') - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('rescan') - expect(index.watermark()).toBeNull() - const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n') - expect(said).toContain('RESCAN') - expect(said).toContain('unstamped') - }) - - it("a store with no committed-generation capability keeps pre-stamp behavior → 'adopt'", async () => { - const storage = await makeStorage(null) - await writeArtifact(storage, null) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('adopt') - expect(index.watermark()).toBeNull() - }) - - it('STAMP-AFTER-DATA: the stamp lands after every node record and the system record', async () => { - const storage = await makeStorage(2) - const index = makeIndex(storage) - for (let i = 0; i < 3; i++) { - await index.addItem({ id: uuidv4(), vector: randomVector(DIM) }) - } - - // One shared op log across all three write surfaces pins global order. - const ops: string[] = [] - const origNode = storage.saveVectorIndexData.bind(storage) - vi.spyOn(storage, 'saveVectorIndexData').mockImplementation(async (id, data) => { - ops.push(`node:${id}`) - return origNode(id, data) - }) - const origSystem = storage.saveHNSWSystem.bind(storage) - vi.spyOn(storage, 'saveHNSWSystem').mockImplementation(async data => { - ops.push('system') - return origSystem(data) - }) - const origMeta = storage.saveMetadata.bind(storage) - vi.spyOn(storage, 'saveMetadata').mockImplementation(async (id, metadata) => { - ops.push(`meta:${id}`) - return origMeta(id, metadata) - }) - - index.stampWatermark(2) - await index.flush() - - const stampAt = ops.indexOf(`meta:${HNSW_INDEX_STAMP_KEY}`) - expect(stampAt, 'stamp record was written').toBeGreaterThanOrEqual(0) - expect(stampAt, 'stamp is the FINAL write of the flush').toBe(ops.length - 1) - expect(ops.filter(o => o.startsWith('node:')).length).toBeGreaterThan(0) - expect(ops.indexOf('system')).toBeLessThan(stampAt) - }) - - it('the stamp record carries {watermark, formatVersion, stampedAt} + modelIdentity (dims only)', async () => { - const storage = await makeStorage(7) - await writeArtifact(storage, 7) - - const record = (await storage.getMetadata(HNSW_INDEX_STAMP_KEY)) as { - watermark: number - formatVersion: number - stampedAt: number - modelIdentity: { embedModelId?: string; dimensions: number | null } - } - expect(record.watermark).toBe(7) - expect(record.formatVersion).toBe(1) - expect(typeof record.stampedAt).toBe('number') - // The JS index never sees the embedder — dimensions are the only vector- - // space identity it can honestly assert. - expect(record.modelIdentity).toEqual({ dimensions: DIM }) - }) - - it('a pending stamp still lands when nothing is dirty (already-durable bytes, stamp-after-data trivially holds)', async () => { - const storage = await makeStorage(4) - const index = makeIndex(storage) - await index.addItem({ id: uuidv4(), vector: randomVector(DIM) }) - await index.flush() // data durable, no stamp yet - - index.stampWatermark(4) - await index.flush() // nothing dirty — the stamp must still be written - - const record = (await storage.getMetadata(HNSW_INDEX_STAMP_KEY)) as { watermark: number } - expect(record?.watermark).toBe(4) - expect(index.watermark()).toBe(4) - }) -}) diff --git a/tests/unit/hnsw/update-item-atomic.test.ts b/tests/unit/hnsw/update-item-atomic.test.ts deleted file mode 100644 index f8798949..00000000 --- a/tests/unit/hnsw/update-item-atomic.test.ts +++ /dev/null @@ -1,366 +0,0 @@ -/** - * @module tests/unit/hnsw/update-item-atomic - * @description Guard for the atomic vector-index update: a row must NEVER be - * absent from vector search during an update. The historical update path - * staged a remove followed by an add as two separately-awaited transaction - * operations — between them the row was in NEITHER index (dark to semantic - * recall while perfectly visible to metadata reads; observed as seconds-long - * flicker in a production deployment). The structural cure verified here: - * - * 1. `JsHnswVectorIndex.updateItem` — same vector (element-wise) is a pure - * no-op (the production flicker shape: a type-only update re-indexing an - * UNCHANGED vector); a changed vector swaps in place, the node never - * leaving the map (white-box probe at the first internal step after the - * synchronous swap), including when the node IS the entry point. - * 2. `ReplaceInVectorIndexOperation` — one transaction leg that prefers the - * provider's in-place `updateItem`, with a remove+add-ADJACENT fallback - * for providers that have not shipped it; rollback restores the declared - * before-vector on both branches. - * 3. The brain's update path — with the JS index carrying `updateItem`, - * `removeItem` is never called during `brain.update()`, for the - * type-only shape AND for a genuine vector change. - */ -import { describe, it, expect, vi } from 'vitest' -import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js' -import { ReplaceInVectorIndexOperation } from '../../../src/transaction/operations/IndexOperations.js' -import type { VectorIndexProvider } from '../../../src/plugin.js' -import type { Vector, VectorDocument } from '../../../src/coreTypes.js' -import { euclideanDistance } from '../../../src/utils/index.js' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { Brainy } from '../../../src/brainy' -import { createAddParams, createTestConfig } from '../../helpers/test-factory' - -const DIM = 8 - -function seededRand(seed: number): () => number { - let s = seed >>> 0 - return () => { - s = (s + 0x6d2b79f5) | 0 - let t = Math.imul(s ^ (s >>> 15), 1 | s) - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t - return ((t ^ (t >>> 14)) >>> 0) / 4294967296 - } -} - -/** A deterministic vector pointing in a pseudo-random direction (well-connected graph). */ -function vec(idx: number): number[] { - const rand = seededRand(idx + 1) - return Array.from({ length: DIM }, () => rand() * 2 - 1) -} - -type Noun = { id: string; vector: number[]; connections: Map>; level: number } - -function nounsOf(index: JsHnswVectorIndex): Map { - return (index as unknown as { nouns: Map }).nouns -} - -/** Flatten a reverse index to sorted `target|level|source` triples. */ -function triplesFromIncoming(inc: Map>>): string[] { - const out: string[] = [] - for (const [target, byLevel] of inc) { - for (const [level, sources] of byLevel) { - for (const source of sources) out.push(`${target}|${level}|${source}`) - } - } - return out.sort() -} - -/** Derive the ground-truth reverse index directly from the live forward adjacency. */ -function triplesFromAdjacency(nouns: Map): string[] { - const out: string[] = [] - for (const [nodeId, node] of nouns) { - for (const [level, targets] of node.connections) { - for (const target of targets) out.push(`${target}|${level}|${nodeId}`) - } - } - return out.sort() -} - -function assertReverseIndexConsistent(index: JsHnswVectorIndex): void { - const live = ( - index as unknown as { ensureIncoming: () => Map>> } - ).ensureIncoming() - expect(triplesFromIncoming(live)).toEqual(triplesFromAdjacency(nounsOf(index))) -} - -function assertNoSelfLoops(index: JsHnswVectorIndex, id: string): void { - const node = nounsOf(index).get(id)! - for (const [level, targets] of node.connections) { - expect(targets.has(id), `self-loop at level ${level}`).toBe(false) - } -} - -function makeIndex(M = 16): JsHnswVectorIndex { - return new JsHnswVectorIndex( - { M, efConstruction: 200, efSearch: 64, ml: 16 }, - euclideanDistance, - { useParallelization: false, storage: new MemoryStorage() } - ) -} - -async function fillIndex(index: JsHnswVectorIndex, count: number): Promise { - for (let i = 0; i < count; i++) { - await index.addItem({ id: `n-${i}`, vector: vec(i) }) - } -} - -describe('JsHnswVectorIndex.updateItem — atomic in-place vector update', () => { - it('same vector (element-wise equal, fresh array) is a pure no-op: no remove, no relink, still searchable', async () => { - const index = makeIndex() - await fillIndex(index, 30) - - const target = 'n-7' - const sameVector = [...vec(7)] // fresh array, identical elements - - const before = await index.search(vec(7), 1) - expect(before[0][0]).toBe(target) - - const removeSpy = vi.spyOn(index, 'removeItem') - const nodeBefore = nounsOf(index).get(target)! - const connectionsBefore = nodeBefore.connections // reference — a relink replaces it - - await index.updateItem({ id: target, vector: sameVector }) - - expect(removeSpy).not.toHaveBeenCalled() - expect(index.size()).toBe(30) - // No relink happened: the connections map is the SAME object, untouched. - expect(nounsOf(index).get(target)!.connections).toBe(connectionsBefore) - - const after = await index.search(vec(7), 1) - expect(after[0][0]).toBe(target) - expect(after[0][1]).toBeCloseTo(0, 10) - - removeSpy.mockRestore() - }) - - it('changed vector: node never leaves the map (probe fires after the synchronous swap), removeItem never called, findable by the NEW vector', async () => { - const index = makeIndex() - await fillIndex(index, 40) - - const target = 'n-5' - const newVector = vec(500) - - // White-box probe: ensureIncoming is the FIRST internal step of the unlink - // walk, i.e. the first thing updateItem does after the synchronous vector - // swap. At that instant the node must (a) still be in the map and (b) - // already carry the NEW vector — the visibility-atomic ordering. - const inner = index as unknown as { - nouns: Map - ensureIncoming: () => Map>> - } - const origEnsure = inner.ensureIncoming.bind(index) - let probed = false - let presentDuring = false - let swappedFirst = false - ;(index as any).ensureIncoming = function () { - if (!probed) { - probed = true - presentDuring = inner.nouns.has(target) - swappedFirst = inner.nouns.get(target)?.vector === newVector - } - return origEnsure() - } - - const removeSpy = vi.spyOn(index, 'removeItem') - await index.updateItem({ id: target, vector: newVector }) - delete (index as any).ensureIncoming // restore the prototype method - - expect(probed).toBe(true) - expect(presentDuring).toBe(true) - expect(swappedFirst).toBe(true) - expect(removeSpy).not.toHaveBeenCalled() - expect(index.size()).toBe(40) - expect(nounsOf(index).has(target)).toBe(true) - - // Findable by search with the NEW vector, at distance ~0. - const got = await index.search(newVector, 1) - expect(got[0][0]).toBe(target) - expect(got[0][1]).toBeCloseTo(0, 10) - - // The relink left the graph bookkeeping exactly consistent. - assertNoSelfLoops(index, target) - assertReverseIndexConsistent(index) - - removeSpy.mockRestore() - }) - - it('keeps the node at its existing level (never releveled by an update)', async () => { - const index = makeIndex() - await fillIndex(index, 30) - - const target = 'n-3' - const levelBefore = nounsOf(index).get(target)!.level - - await index.updateItem({ id: target, vector: vec(600) }) - - expect(nounsOf(index).get(target)!.level).toBe(levelBefore) - expect(index.getMaxLevel()).toBeGreaterThanOrEqual(levelBefore) - }) - - it('updating the ENTRY POINT in place keeps it valid — entry id and maxLevel unchanged, graph never stranded', async () => { - const index = makeIndex() - await fillIndex(index, 40) - - const entryId = index.getEntryPointId()! - const maxLevelBefore = index.getMaxLevel() - const newVector = vec(700) - - await index.updateItem({ id: entryId, vector: newVector }) - - // Entry-point bookkeeping must not regress. - expect(index.getEntryPointId()).toBe(entryId) - expect(index.getMaxLevel()).toBe(maxLevelBefore) - expect(index.size()).toBe(40) - - // The entry point itself is findable by its new vector... - const gotEntry = await index.search(newVector, 1) - expect(gotEntry[0][0]).toBe(entryId) - - // ...and the REST of the graph is still reachable through it (a stranded, - // edgeless entry point would make every other node invisible). - const otherId = [...nounsOf(index).keys()].find((id) => id !== entryId)! - const otherIdx = Number(otherId.slice(2)) - const gotOther = await index.search(vec(otherIdx), 1) - expect(gotOther[0][0]).toBe(otherId) - - assertNoSelfLoops(index, entryId) - assertReverseIndexConsistent(index) - }) - - it('absent id delegates to addItem (plain insert)', async () => { - const index = makeIndex() - await fillIndex(index, 10) - - await index.updateItem({ id: 'fresh', vector: vec(900) }) - - expect(index.size()).toBe(11) - const got = await index.search(vec(900), 1) - expect(got[0][0]).toBe('fresh') - }) -}) - -describe('ReplaceInVectorIndexOperation — one atomic transaction leg', () => { - it('uses the provider updateItem path and rolls back to the old vector in place', async () => { - const index = makeIndex() - await fillIndex(index, 30) - - const target = 'n-9' - const oldVector = vec(9) - const newVector = vec(800) - - const removeSpy = vi.spyOn(index, 'removeItem') - const op = new ReplaceInVectorIndexOperation(index, target, oldVector, newVector) - expect(op.name).toBe('ReplaceInVectorIndex(hnsw-js)') - - const rollback = await op.execute() - expect(removeSpy).not.toHaveBeenCalled() - expect((await index.search(newVector, 1))[0][0]).toBe(target) - - await rollback() - expect(removeSpy).not.toHaveBeenCalled() - expect(index.size()).toBe(30) - - // Old vector restored, element-wise, and searchable again. - const restored = nounsOf(index).get(target)!.vector - expect(restored.length).toBe(oldVector.length) - for (let i = 0; i < oldVector.length; i++) { - expect(restored[i]).toBe(oldVector[i]) - } - const back = await index.search(oldVector, 1) - expect(back[0][0]).toBe(target) - expect(back[0][1]).toBeCloseTo(0, 10) - - removeSpy.mockRestore() - }) - - it('falls back to remove+add ADJACENT within the single op for a provider without updateItem, and rolls back the same way', async () => { - // A provider that has not shipped updateItem — the temporary seam: the - // pair stays adjacent inside ONE op (no other transaction operation can - // interleave), until the provider ships its own in-place updateItem. - const calls: string[] = [] - const store = new Map() - const legacyProvider = { - name: 'legacy-native', - addItem: async (item: VectorDocument) => { - calls.push(`add:${item.id}`) - store.set(item.id, item.vector) - return item.id - }, - removeItem: async (id: string) => { - calls.push(`remove:${id}`) - return store.delete(id) - }, - search: async () => [], - size: () => store.size, - clear: () => store.clear(), - rebuild: async () => {}, - flush: async () => 0, - getPersistMode: () => 'immediate' as const - } as unknown as VectorIndexProvider - - store.set('x', [1, 0]) - const op = new ReplaceInVectorIndexOperation(legacyProvider, 'x', [1, 0], [0, 1]) - - const rollback = await op.execute() - expect(calls).toEqual(['remove:x', 'add:x']) - expect(store.get('x')).toEqual([0, 1]) - - await rollback() - expect(calls).toEqual(['remove:x', 'add:x', 'remove:x', 'add:x']) - expect(store.get('x')).toEqual([1, 0]) - }) -}) - -describe('brain.update() — the update path stages ONE atomic vector-index leg', () => { - it('a type-only update (unchanged vector — the production flicker shape) never calls removeItem on the vector index', async () => { - const brain = new Brainy(createTestConfig()) - await brain.init() - try { - const id = await brain.add( - createAddParams({ data: 'atomic flicker guard entity', type: 'thing' }) - ) - - const index = (brain as unknown as { index: JsHnswVectorIndex }).index - const removeSpy = vi.spyOn(index, 'removeItem') - const sizeBefore = index.size() - - await brain.update({ id, type: 'document' }) - - expect(removeSpy).not.toHaveBeenCalled() - expect(index.size()).toBe(sizeBefore) - - const updated = await brain.get(id) - expect(updated).not.toBeNull() - expect(updated!.type).toBe('document') - - removeSpy.mockRestore() - } finally { - await brain.close() - } - }) - - it('a genuine vector change on update also never calls removeItem (in-place replace)', async () => { - const brain = new Brainy(createTestConfig()) - await brain.init() - try { - const id = await brain.add( - createAddParams({ data: 'vector change stays visible', type: 'thing' }) - ) - const existing = await brain.get(id, { includeVectors: true }) - // Same dimensionality, guaranteed-different content. - const changed = existing!.vector.map((x: number, i: number) => (i === 0 ? x + 0.25 : x)) - - const index = (brain as unknown as { index: JsHnswVectorIndex }).index - const removeSpy = vi.spyOn(index, 'removeItem') - - await brain.update({ id, vector: changed }) - - expect(removeSpy).not.toHaveBeenCalled() - expect(nounsOf(index).has(id)).toBe(true) - - removeSpy.mockRestore() - } finally { - await brain.close() - } - }) -}) diff --git a/tests/unit/indexes/columnStore/segment-load-fault.test.ts b/tests/unit/indexes/columnStore/segment-load-fault.test.ts index 9ef4ba13..deb0868f 100644 --- a/tests/unit/indexes/columnStore/segment-load-fault.test.ts +++ b/tests/unit/indexes/columnStore/segment-load-fault.test.ts @@ -5,22 +5,19 @@ * doing so dropped every entity in that segment out of `filter`/`rangeQuery`/ * `sortTopK` with no error, so a corrupt index looked like a merely short result. * - * The three failure classes and their required behaviour (torn-segment - * QUARANTINE contract — a raw throw at query time killed every query on the - * field forever; a silent skip hid the loss; quarantine is the middle): + * The three failure classes and their required behaviour: * - a real storage IO fault (EIO) PROPAGATES verbatim — a present-but-unreadable * segment is not "absent", so it must not read as an empty result; - * - a manifest-listed segment with undecodable bytes is QUARANTINED at - * discovery: the query serves the field's remaining segments degraded and - * `quarantinedSegments()` reports the torn segment (loud once, counted - * always, healable); - * - a manifest-listed segment with NO bytes (gone on disk) quarantines the - * same way. + * - a manifest-listed segment with undecodable bytes throws `ColumnSegmentLoadError`; + * - a manifest-listed segment with NO bytes (gone on disk) throws `ColumnSegmentLoadError`. * Only genuine absence stays benign: querying a field that has no manifest at all * returns empty (nothing was ever written for it) — that is not a fault. */ import { describe, it, expect, beforeEach } from 'vitest' -import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js' +import { + ColumnStore, + ColumnSegmentLoadError +} from '../../../../src/indexes/columnStore/ColumnStore.js' import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' @@ -83,44 +80,30 @@ describe('ColumnStore segment-load faults surface loudly, absence stays benign ( return s } - it('propagates a storage IO fault verbatim — not [] and not a quarantine (a present-but-unreadable segment is not torn)', async () => { + it('propagates a storage IO fault verbatim — not [] and not a ColumnSegmentLoadError', async () => { storage.faultMode = 'io' const store = await reopen() await expect(store.filter('createdAt', 300)).rejects.toMatchObject({ code: 'EIO' }) - // An IO fault is NOT quarantined — the segment may be fine once the disk - // recovers; only torn/absent bytes enter the ledger. - expect(store.quarantinedSegments('createdAt')).toEqual([]) await store.close() }) - it('QUARANTINES an undecodable manifest-listed segment at discovery — the query serves degraded, the ledger names the tear', async () => { + it('throws ColumnSegmentLoadError when a manifest-listed segment is undecodable', async () => { storage.faultMode = 'corrupt' const store = await reopen() - // Degraded-announced serve: the field's only segment is torn, so the - // result is empty — but the query completes instead of throwing. - const sorted = await store.sortTopK('createdAt', 'desc', 10) - expect(sorted).toEqual([]) - const ledger = store.quarantinedSegments('createdAt') - expect(ledger).toHaveLength(1) - expect(ledger[0].error).toMatch(/decode failed/) - expect(ledger[0].hits).toBeGreaterThanOrEqual(1) - // Subsequent queries keep serving (skip + count), never a throw. - const hitsBefore = ledger[0].hits - await expect(store.filter('createdAt', 300)).resolves.toBeDefined() - expect(store.quarantinedSegments('createdAt')[0].hits).toBeGreaterThan(hitsBefore) + await expect( + store.sortTopK('createdAt', 'desc', 10) + ).rejects.toBeInstanceOf(ColumnSegmentLoadError) await store.close() }) - it('QUARANTINES a manifest-listed segment with no loadable bytes — degraded serve, ledger entry, never a throw', async () => { + it('throws ColumnSegmentLoadError when a manifest-listed segment has no loadable bytes', async () => { storage.faultMode = 'missing' const store = await reopen() - const bitmap = await store.rangeQuery('createdAt', 100, 500) - expect(bitmap.size).toBe(0) - const ledger = store.quarantinedSegments('createdAt') - expect(ledger).toHaveLength(1) - expect(ledger[0].error).toMatch(/no loadable bytes/) + await expect( + store.rangeQuery('createdAt', 100, 500) + ).rejects.toBeInstanceOf(ColumnSegmentLoadError) await store.close() }) diff --git a/tests/unit/plugin-version-coupling.test.ts b/tests/unit/plugin-version-coupling.test.ts index ffcc2a88..00b236aa 100644 --- a/tests/unit/plugin-version-coupling.test.ts +++ b/tests/unit/plugin-version-coupling.test.ts @@ -63,9 +63,7 @@ describe('getBrainyVersion() — synchronously correct on first call', () => { expect(v).toBe(PACKAGE_VERSION) expect(v).not.toBe('3.14.0') expect(v).not.toBe('0.0.0') // the unknown-read sentinel must not surface in a real install - // Deliberately major-agnostic: the equality with PACKAGE_VERSION above already - // proves the sync read; this shape pin only guards against sentinel garbage. - expect(v).toMatch(/^\d+\.\d+\.\d+/) + expect(v.startsWith('8.')).toBe(true) }) }) @@ -97,16 +95,13 @@ describe('version coupling at init() — no silent fallback', () => { await brain.close() }) - it('does NOT throw for a realistic version-matched caret range on a COLD init', async () => { + it('does NOT throw for a realistic cor 3.x range (^8.0.0) on a COLD init', async () => { // The actual regression: loadPlugins() is the first init step and makes the - // first getBrainyVersion() call, so a stale sync default ('3.14.0') would - // reject a correctly-matched native provider declaring the real caret range — - // it fails ^ just as it failed ^8, so the regression intent is - // preserved while the range stays major-agnostic. A fresh brain registering a - // `^.0.0` plugin must init cleanly. - const major = PACKAGE_VERSION.split('.')[0] + // first getBrainyVersion() call, so a stale sync default would reject a + // correctly-matched native provider declaring the real 8.x range. A fresh + // brain registering a `^8.0.0` plugin must init cleanly. const brain = memBrain() - brain.use(fakePlugin('@fake/cor-3x', { brainyRange: `^${major}.0.0` })) + brain.use(fakePlugin('@fake/cor-3x', { brainyRange: '^8.0.0' })) await expect(brain.init()).resolves.toBeUndefined() await brain.close() }) diff --git a/tests/unit/plugin/provider-generation.test.ts b/tests/unit/plugin/provider-generation.test.ts deleted file mode 100644 index 6295b6f1..00000000 --- a/tests/unit/plugin/provider-generation.test.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Generation threading to the metadata-index and vector-index provider write - * surfaces — the counterpart of the graph pins in - * tests/unit/transaction/graphIndexOperations-generation.test.ts. - * - * The provider contract gained an optional trailing `generation?: bigint` on - * `MetadataIndexProvider.addToIndex`/`removeFromIndex`, - * `VectorIndexProvider.addItem`/`removeItem` (+ the feature-detected - * `updateItem`), and the id-mapper's `getOrAssign`/`remove`. A native provider - * with per-record delta logs stamps its durable records with it — so the value - * arriving MUST be the real commit generation (nonzero, monotonic), never a - * fabricated 0 and never absent on the coordinator's write paths. - * - * Two layers of pins: - * 1. End-to-end: provider doubles registered via the plugin system capture - * the generation argument during brain.add()/update()/remove() and it - * must equal the committed watermark (`brain.now().generation`). - * 2. Operation layer: execute-time (not construction-time) resolution, and - * one shared generation across an op's forward + rollback halves. - */ -import { describe, it, expect, afterEach } from 'vitest' -import { Brainy, NounType } from '../../../src/index.js' -import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js' -import { - AddToVectorIndexOperation, - RemoveFromVectorIndexOperation, - ReplaceInVectorIndexOperation, - AddToMetadataIndexOperation, - RemoveFromMetadataIndexOperation -} from '../../../src/transaction/operations/IndexOperations.js' -import type { VectorIndexProvider } from '../../../src/plugin.js' - -const V = () => Array.from({ length: 384 }, () => Math.random()) - -type Captured = { method: string; id: string; generation: bigint | undefined } - -const brains: Brainy[] = [] -afterEach(async () => { - for (const b of brains.splice(0)) await b.close().catch(() => {}) -}) - -/** Metadata manager subclass that records the generation of every write. */ -function makeCapturingMetadataFactory(calls: Captured[]) { - return (storage: any) => { - class CapturingManager extends MetadataIndexManager { - async addToIndex(id: string, entityOrMetadata: any, skipFlush = false, deferWrites = false, generation?: bigint): Promise { - calls.push({ method: 'addToIndex', id, generation }) - return super.addToIndex(id, entityOrMetadata, skipFlush, deferWrites, generation) - } - async removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise { - calls.push({ method: 'removeFromIndex', id, generation }) - return super.removeFromIndex(id, metadata, generation) - } - } - return new CapturingManager(storage) - } -} - -/** Minimal vector-index double capturing the generation of every write. */ -function makeCapturingVectorFactory(calls: Captured[]) { - return () => { - const items = new Map() - const double: VectorIndexProvider & { updateItem(item: { id: string; vector: number[] }, generation?: bigint): Promise } = { - name: 'capture-double', - async addItem(item, generation) { - calls.push({ method: 'addItem', id: item.id, generation }) - items.set(item.id, item.vector as number[]) - return item.id - }, - async removeItem(id, generation) { - calls.push({ method: 'removeItem', id, generation }) - return items.delete(id) - }, - async updateItem(item, generation) { - calls.push({ method: 'updateItem', id: item.id, generation }) - items.set(item.id, item.vector) - }, - async search() { return [] }, - size: () => items.size, - clear: () => { items.clear() }, - async rebuild() {}, - async flush() { return 0 }, - getPersistMode: () => 'deferred' as const - } - return double - } -} - -async function makeBrain(plugin: any): Promise { - const brain = new Brainy({ - storage: { type: 'memory' }, - requireSubtype: false, - silent: true, - plugins: [] - }) - brain.use(plugin) - await brain.init() - brains.push(brain) - return brain -} - -describe('Metadata-index provider — real commit generation on every write (end-to-end)', () => { - it('add()/update()/remove() pass the nonzero, monotonic commit generation to addToIndex/removeFromIndex', async () => { - const calls: Captured[] = [] - const brain = await makeBrain({ - name: 'capture-metadata', - activate: async (ctx: any) => { - ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls)) - return true - } - }) - - const id = await brain.add({ data: 'one', type: NounType.Concept, metadata: { k: 'a' }, vector: V() }) - const addCall = calls.find((c) => c.method === 'addToIndex' && c.id === id) - expect(addCall).toBeDefined() - expect(typeof addCall!.generation).toBe('bigint') - expect(addCall!.generation!).toBeGreaterThan(0n) - // Committed watermark after a single-op write IS this write's generation. - expect(addCall!.generation!).toBe(BigInt(brain.now().generation)) - - calls.length = 0 - await brain.update({ id, metadata: { k: 'b' } }) - const updRemove = calls.find((c) => c.method === 'removeFromIndex' && c.id === id) - const updAdd = calls.find((c) => c.method === 'addToIndex' && c.id === id) - expect(updRemove?.generation).toBeDefined() - expect(updAdd?.generation).toBeDefined() - // One commit → the remove-old + add-new legs share one watermark. - expect(updAdd!.generation!).toBe(updRemove!.generation!) - expect(updAdd!.generation!).toBe(BigInt(brain.now().generation)) - const updateGen = updAdd!.generation! - expect(updateGen).toBeGreaterThan(0n) - - calls.length = 0 - await brain.remove(id) - const rmCall = calls.find((c) => c.method === 'removeFromIndex' && c.id === id) - expect(rmCall?.generation).toBeDefined() - expect(rmCall!.generation!).toBeGreaterThan(updateGen) // monotonic - expect(rmCall!.generation!).toBe(BigInt(brain.now().generation)) - }) - - it('transact() adds stamp the batch receipt generation', async () => { - const calls: Captured[] = [] - const brain = await makeBrain({ - name: 'capture-metadata-tx', - activate: async (ctx: any) => { - ctx.registerProvider('metadataIndex', makeCapturingMetadataFactory(calls)) - return true - } - }) - - // Bootstrap honesty: init-time infrastructure writes (the VFS root) are - // applied WITHOUT a generation — the provider must receive undefined, - // never a fabricated 0. - for (const c of calls) expect(c.generation).toBeUndefined() - calls.length = 0 - - const db = await brain.transact([ - { op: 'add', data: 'tx-one', type: NounType.Concept, vector: V() }, - { op: 'add', data: 'tx-two', type: NounType.Concept, vector: V() } - ] as any) - - const receiptGen = BigInt(db.receipt!.generation) - const addGens = calls.filter((c) => c.method === 'addToIndex').map((c) => c.generation) - expect(addGens.length).toBeGreaterThanOrEqual(2) - for (const g of addGens) expect(g).toBe(receiptGen) - }) -}) - -describe('Vector-index provider — real commit generation on every write (end-to-end)', () => { - it('add()/update()/remove() pass the nonzero commit generation to addItem/updateItem/removeItem', async () => { - const calls: Captured[] = [] - const brain = await makeBrain({ - name: 'capture-vector', - activate: async (ctx: any) => { - ctx.registerProvider('vector', makeCapturingVectorFactory(calls)) - return true - } - }) - - const id = await brain.add({ data: 'vec', type: NounType.Concept, vector: V() }) - const addCall = calls.find((c) => c.method === 'addItem' && c.id === id) - expect(addCall).toBeDefined() - expect(typeof addCall!.generation).toBe('bigint') - expect(addCall!.generation!).toBeGreaterThan(0n) - expect(addCall!.generation!).toBe(BigInt(brain.now().generation)) - - calls.length = 0 - await brain.update({ id, vector: V() }) - const updCall = calls.find((c) => c.method === 'updateItem' && c.id === id) - expect(updCall?.generation).toBeDefined() - expect(updCall!.generation!).toBeGreaterThan(addCall!.generation!) // monotonic - expect(updCall!.generation!).toBe(BigInt(brain.now().generation)) - - calls.length = 0 - await brain.remove(id) - const rmCall = calls.find((c) => c.method === 'removeItem' && c.id === id) - expect(rmCall?.generation).toBeDefined() - expect(rmCall!.generation!).toBeGreaterThan(updCall!.generation!) - expect(rmCall!.generation!).toBe(BigInt(brain.now().generation)) - }) -}) - -describe('Index operations — generation threading (operation layer)', () => { - function makeVectorSpy() { - const calls: Array<{ method: string; generation: bigint | undefined }> = [] - const index = { - name: 'spy', - async addItem(_item: any, generation?: bigint) { calls.push({ method: 'addItem', generation }); return 'x' }, - async removeItem(_id: string, generation?: bigint) { calls.push({ method: 'removeItem', generation }); return true }, - async updateItem(_item: any, generation?: bigint) { calls.push({ method: 'updateItem', generation }) } - } as unknown as VectorIndexProvider - return { index, calls } - } - - it('vector add/remove/replace resolve the thunk at EXECUTE time and reuse one generation for rollback', async () => { - const { index, calls } = makeVectorSpy() - let current = 1n - const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2], () => current) - current = 42n // assigned after construction, read at execute - const rollback = await op.execute() - expect(calls[0]).toEqual({ method: 'addItem', generation: 42n }) - current = 77n // rollback must NOT re-read — one watermark per round trip - await rollback() - expect(calls[1]).toEqual({ method: 'removeItem', generation: 42n }) - - calls.length = 0 - const rm = new RemoveFromVectorIndexOperation(index, 'id-1', [1, 2], () => 7n) - const rb2 = await rm.execute() - await rb2() - expect(calls).toEqual([ - { method: 'removeItem', generation: 7n }, - { method: 'addItem', generation: 7n } - ]) - - calls.length = 0 - const rep = new ReplaceInVectorIndexOperation(index, 'id-1', [1, 2], [3, 4], () => 9n) - const rb3 = await rep.execute() - await rb3() - expect(calls).toEqual([ - { method: 'updateItem', generation: 9n }, - { method: 'updateItem', generation: 9n } - ]) - }) - - it('metadata add/remove pass the resolved generation through both halves', async () => { - const calls: Array<{ method: string; generation: bigint | undefined }> = [] - const manager = { - async addToIndex(_id: string, _e: any, _s?: boolean, _d?: boolean, generation?: bigint) { - calls.push({ method: 'addToIndex', generation }) - }, - async removeFromIndex(_id: string, _m?: any, generation?: bigint) { - calls.push({ method: 'removeFromIndex', generation }) - } - } as unknown as MetadataIndexManager - - const add = new AddToMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 11n) - const rb = await add.execute() - await rb() - const rm = new RemoveFromMetadataIndexOperation(manager, 'id-1', { type: 'x' }, () => 12n) - const rb2 = await rm.execute() - await rb2() - expect(calls).toEqual([ - { method: 'addToIndex', generation: 11n }, - { method: 'removeFromIndex', generation: 11n }, - { method: 'removeFromIndex', generation: 12n }, - { method: 'addToIndex', generation: 12n } - ]) - }) - - it('omitted thunk (legacy caller) → provider receives undefined, never a fabricated 0', async () => { - const { index, calls } = makeVectorSpy() - const op = new AddToVectorIndexOperation(index, 'id-1', [1, 2]) - await op.execute() - expect(calls[0]).toEqual({ method: 'addItem', generation: undefined }) - }) -}) diff --git a/tests/unit/reprojection/reprojection-engine.test.ts b/tests/unit/reprojection/reprojection-engine.test.ts deleted file mode 100644 index 58d10b44..00000000 --- a/tests/unit/reprojection/reprojection-engine.test.ts +++ /dev/null @@ -1,590 +0,0 @@ -/** - * @module tests/unit/reprojection/reprojection-engine - * @description Spec-by-example for the pure-TS reprojection engine — the - * frozen contract mirrored from the native twin (a shared conformance suite - * runs against both, so the shapes pinned here are load-bearing): - * - * (a) register + advance folds a scripted source to caught-up with exact - * watermark/applied counts and adapter-owned stamping; - * (b) budget exhaustion answers mid-stream and a second advance RESUMES from - * the watermark — never a refold; - * (c) a door bump mid-advance preempts within one installment — pinned by - * MECHANISM (no further applyBatch after the bumping step), with only a - * generous wall-clock sanity bound; - * (d) advanceAll round-robins families at batch granularity — no starvation; - * (e) swap builds beside (the old adapter serves throughout), flips - * atomically at parity, refuses a concurrent swap with a typed error; - * (f) quarantine: a typed poison fact is skipped + ledgered, narration - * doubles, a NON-typed throw aborts loudly; - * (g) discard() lands on the LOSING adapter after a swap. - */ -import { describe, it, expect, vi, afterEach } from 'vitest' -import { - ReprojectionEngine, - DoorSignal, - ProjectionApplyError, - SwapInFlightError, - MAX_INSTALLMENT_MS, - type ProjectionAdapter, - type FactSource -} from '../../../src/reprojection/reprojectionEngine.js' -import { FactLogSource } from '../../../src/reprojection/factLogSource.js' -import type { CommitFact } from '../../../src/db/factLog.js' -import { prodLog } from '../../../src/utils/logger.js' - -/** Build one committed fact for a generation. */ -function fact(generation: number): CommitFact { - return { - generation, - timestamp: 1_700_000_000_000 + generation, - ops: [ - { - kind: 'noun', - id: `id-${generation}`, - record: { metadata: { n: generation }, vector: null } - } - ] - } -} - -/** A scripted FactSource over a (possibly mutable) list of generations. */ -function scriptedSource(gens: () => number[]): FactSource { - return { - async scan(from: number, limit: number): Promise { - return gens() - .filter((g) => g > from) - .sort((x, y) => x - y) - .slice(0, limit) - .map(fact) - } - } -} - -/** - * A recording in-memory adapter: stamps after data (the watermark advances - * only after a successful apply), applies idempotently (a Map keyed by - * generation), and can be scripted to poison (typed) or hard-fail (untyped) - * specific generations, or to run a hook inside applyBatch. - */ -class RecordingAdapter implements ProjectionAdapter { - readonly family: string - /** Generations per applyBatch call, in call order (empty arrays included). */ - readonly batches: number[][] = [] - /** The upTo passed to each applyBatch call, in call order. */ - readonly upTos: number[] = [] - /** Latest state per generation — idempotent under at-least-once delivery. */ - readonly state = new Map() - /** Generations that throw a typed ProjectionApplyError. */ - readonly poison = new Set() - /** Generations that throw a plain (untyped) Error. */ - readonly hardFail = new Set() - /** Runs inside applyBatch after validation, before the stamp. */ - onApply?: (gens: number[]) => void | Promise - discarded = 0 - private wm: number | null - - constructor(family: string, watermark: number | null = null) { - this.family = family - this.wm = watermark - } - - watermark(): number | null { - return this.wm - } - - async applyBatch(facts: CommitFact[], upTo: number): Promise { - for (const [i, f] of facts.entries()) { - if (this.hardFail.has(f.generation)) { - throw new Error(`disk exploded at generation ${f.generation}`) - } - if (this.poison.has(f.generation)) { - throw new ProjectionApplyError({ - generation: f.generation, - recordIndex: i, - cause: new Error(`unfoldable payload at ${f.generation}`) - }) - } - } - for (const f of facts) this.state.set(f.generation, f.ops) - const gens = facts.map((f) => f.generation) - this.batches.push(gens) - this.upTos.push(upTo) - if (this.onApply) await this.onApply(gens) - this.wm = upTo // stamp-after-data - } - - async discard(): Promise { - this.discarded++ - } -} - -const range = (from: number, to: number): number[] => - Array.from({ length: to - from + 1 }, (_, i) => from + i) - -afterEach(() => { - vi.restoreAllMocks() -}) - -describe('reprojection engine — (a) register + advance to caught-up', () => { - it('folds a scripted source in order, adapter-stamped, with exact counts', async () => { - const source = scriptedSource(() => range(1, 7)) - const engine = new ReprojectionEngine({ source, batchSize: 3 }) - const adapter = new RecordingAdapter('a') - engine.register(adapter) - - const result = await engine.advance('a', { budgetMs: 10_000 }) - - expect(result.status).toBe('caught-up') - expect(result.watermark).toBe(7) - expect(result.applied).toBe(7) - // Batch shape and the upTo handed to the adapter's own stamp. - expect(adapter.batches).toEqual([[1, 2, 3], [4, 5, 6], [7]]) - expect(adapter.upTos).toEqual([3, 6, 7]) - // The watermark is the ADAPTER's stamp — the engine never wrote one. - expect(adapter.watermark()).toBe(7) - expect(engine.getAdapter('a')).toBe(adapter) - }) - - it('honors upTo as an inclusive cap and answers caught-up at the cap', async () => { - const source = scriptedSource(() => range(1, 9)) - const engine = new ReprojectionEngine({ source, batchSize: 3 }) - const adapter = new RecordingAdapter('a') - engine.register(adapter) - - const result = await engine.advance('a', { budgetMs: 10_000, upTo: 5 }) - - expect(result.status).toBe('caught-up') - expect(result.watermark).toBe(5) - expect(result.applied).toBe(5) - expect(adapter.batches.flat()).toEqual([1, 2, 3, 4, 5]) - }) - - it('a caught-up family answers immediately with zero applied', async () => { - const source = scriptedSource(() => range(1, 4)) - const engine = new ReprojectionEngine({ source, batchSize: 10 }) - const adapter = new RecordingAdapter('a', 4) // already stamped to the head - engine.register(adapter) - - const result = await engine.advance('a', { budgetMs: 10_000 }) - - expect(result).toEqual({ status: 'caught-up', watermark: 4, applied: 0 }) - expect(adapter.batches).toEqual([]) - }) - - it('refuses duplicate registration and unregistered families loudly', async () => { - const engine = new ReprojectionEngine({ source: scriptedSource(() => []) }) - engine.register(new RecordingAdapter('a')) - expect(() => engine.register(new RecordingAdapter('a'))).toThrow(/already registered/) - await expect(engine.advance('ghost', { budgetMs: 0 })).rejects.toThrow(/not registered/) - }) -}) - -describe('reprojection engine — (b) budget exhaustion resumes, never refolds', () => { - it('returns budget-exhausted mid-stream; the next advance resumes from the watermark', async () => { - const source = scriptedSource(() => range(1, 10)) - const engine = new ReprojectionEngine({ source, batchSize: 2 }) - const adapter = new RecordingAdapter('b') - engine.register(adapter) - - // Zero budget: exactly ONE step of guaranteed progress, then the answer. - const first = await engine.advance('b', { budgetMs: 0 }) - expect(first.status).toBe('budget-exhausted') - expect(first.watermark).toBe(2) - expect(first.applied).toBe(2) - expect(adapter.batches).toEqual([[1, 2]]) - - // The second advance RESUMES from the stamp — its first batch starts at 3. - const second = await engine.advance('b', { budgetMs: 10_000 }) - expect(second.status).toBe('caught-up') - expect(second.watermark).toBe(10) - expect(second.applied).toBe(8) - expect(adapter.batches[1]).toEqual([3, 4]) - // No refold: every generation delivered exactly once across both calls. - expect(adapter.batches.flat()).toEqual(range(1, 10)) - }) -}) - -describe('reprojection engine — (c) door bump preempts within one installment', () => { - it('a bump during a step yields preempted at that step boundary — no further applyBatch', async () => { - const source = scriptedSource(() => range(1, 12)) - const engine = new ReprojectionEngine({ source, batchSize: 2 }) - const adapter = new RecordingAdapter('c') - adapter.onApply = (gens) => { - if (gens[0] === 3) engine.doorSignal.bump() // door traffic mid-second-batch - } - engine.register(adapter) - - const started = Date.now() - const result = await engine.advance('c', { budgetMs: 60_000 }) - const elapsed = Date.now() - started - - expect(result.status).toBe('preempted') - expect(result.watermark).toBe(4) - expect(result.applied).toBe(4) - // THE MECHANISM PIN: the batch that observed the bump was the LAST batch — - // preemption landed at the very next boundary, not after more work. - expect(adapter.batches).toEqual([[1, 2], [3, 4]]) - // Generous wall-clock sanity only (the pin above carries the contract): - // two tiny batches plus one installment boundary sit far under 5s. - expect(elapsed).toBeLessThan(5_000) - expect(MAX_INSTALLMENT_MS).toBe(50) - - // Resuming folds the rest — preemption lost nothing. - const resumed = await engine.advance('c', { budgetMs: 60_000 }) - expect(resumed.status).toBe('caught-up') - expect(resumed.watermark).toBe(12) - expect(adapter.batches.flat()).toEqual(range(1, 12)) - }) - - it('bumps are edge-triggered per advance: a stale bump never preempts', async () => { - const source = scriptedSource(() => range(1, 4)) - const doorSignal = new DoorSignal() - const engine = new ReprojectionEngine({ source, doorSignal, batchSize: 2 }) - const adapter = new RecordingAdapter('c2') - engine.register(adapter) - - doorSignal.bump() // BEFORE the advance — belongs to earlier traffic - const result = await engine.advance('c2', { budgetMs: 10_000 }) - expect(result.status).toBe('caught-up') - expect(result.watermark).toBe(4) - }) -}) - -describe('reprojection engine — (d) advanceAll round-robin fairness', () => { - it('a one-batch family is served on the first round despite a huge backlog next to it', async () => { - const source = scriptedSource(() => range(1, 40)) - const engine = new ReprojectionEngine({ source, batchSize: 5 }) - const callOrder: string[] = [] - const big = new RecordingAdapter('big') // 8 batches behind - const small = new RecordingAdapter('small', 35) // 1 batch behind - big.onApply = () => { - callOrder.push('big') - } - small.onApply = () => { - callOrder.push('small') - } - engine.register(big) - engine.register(small) - - const results = await engine.advanceAll({ budgetMs: 10_000 }) - - expect(results.big).toEqual({ status: 'caught-up', watermark: 40, applied: 40 }) - expect(results.small).toEqual({ status: 'caught-up', watermark: 40, applied: 5 }) - // Fairness pin: 'small' folded its single batch on round ONE — it never - // waited behind 'big''s backlog. - expect(callOrder[1]).toBe('small') - expect(callOrder.filter((f) => f === 'small')).toHaveLength(1) - }) - - it('two full-backlog families interleave strictly, one batch each per round', async () => { - const source = scriptedSource(() => range(1, 40)) - const engine = new ReprojectionEngine({ source, batchSize: 5 }) - const callOrder: string[] = [] - const first = new RecordingAdapter('first') - const second = new RecordingAdapter('second') - first.onApply = () => { - callOrder.push('first') - } - second.onApply = () => { - callOrder.push('second') - } - engine.register(first) - engine.register(second) - - const results = await engine.advanceAll({ budgetMs: 10_000 }) - - expect(results.first.status).toBe('caught-up') - expect(results.second.status).toBe('caught-up') - // 8 rounds × (first, second): strict alternation — neither ever ran twice - // while the other waited. - expect(callOrder).toHaveLength(16) - for (let i = 0; i < callOrder.length; i += 2) { - expect(callOrder.slice(i, i + 2)).toEqual(['first', 'second']) - } - }) - - it('budget exhaustion mid-round reports every unfinished family at its own watermark', async () => { - const source = scriptedSource(() => range(1, 40)) - const engine = new ReprojectionEngine({ source, batchSize: 5 }) - const a = new RecordingAdapter('a') - const b = new RecordingAdapter('b') - engine.register(a) - engine.register(b) - - const results = await engine.advanceAll({ budgetMs: 0 }) - - // Zero budget: the leading family gets its one guaranteed step, then the - // budget answer lands for everyone still mid-stream. - expect(results.a.status).toBe('budget-exhausted') - expect(results.b.status).toBe('budget-exhausted') - expect(results.a.applied + results.b.applied).toBeGreaterThanOrEqual(5) - // A later advanceAll resumes both to the head. - const finished = await engine.advanceAll({ budgetMs: 10_000 }) - expect(finished.a.status).toBe('caught-up') - expect(finished.b.status).toBe('caught-up') - expect(a.batches.flat()).toEqual(range(1, 40)) - expect(b.batches.flat()).toEqual(range(1, 40)) - }) -}) - -describe('reprojection engine — (e) swap: build-beside, atomic flip, single-flight', () => { - it('the old adapter serves at its own watermark throughout the build; the flip is atomic at parity', async () => { - const log = range(1, 20) - const source = scriptedSource(() => log) - const engine = new ReprojectionEngine({ source, batchSize: 4 }) - const oldAdapter = new RecordingAdapter('e') - engine.register(oldAdapter) - await engine.advance('e', { budgetMs: 10_000 }) - expect(oldAdapter.watermark()).toBe(20) - - // The log grows after the old adapter stamped — the build must reach the - // HEAD (24), not merely the old watermark (20), before the flip. - log.push(21, 22, 23, 24) - - const servingDuringBuild: Array<{ adapter: ProjectionAdapter | undefined; watermark: number | null }> = [] - let replacement!: RecordingAdapter - const result = await engine.swap('e', async () => { - replacement = new RecordingAdapter('e') - replacement.onApply = () => { - servingDuringBuild.push({ - adapter: engine.getAdapter('e'), - watermark: engine.getAdapter('e')!.watermark() - }) - } - return replacement - }) - - // Build-beside pin: EVERY mid-build observation saw the OLD adapter, - // still serving, still at its own stamp. - expect(servingDuringBuild.length).toBeGreaterThan(0) - for (const seen of servingDuringBuild) { - expect(seen.adapter).toBe(oldAdapter) - expect(seen.watermark).toBe(20) - } - // The flip: the registry now serves the replacement, at parity with head. - expect(engine.getAdapter('e')).toBe(replacement) - expect(result.watermark).toBe(24) - expect(result.applied).toBe(24) - expect(replacement.batches.flat()).toEqual(range(1, 24)) - }) - - it('a second concurrent swap on the same family refuses with the typed single-flight error', async () => { - const source = scriptedSource(() => range(1, 8)) - const engine = new ReprojectionEngine({ source, batchSize: 4 }) - engine.register(new RecordingAdapter('e2')) - - let release!: () => void - const gate = new Promise((resolve) => { - release = resolve - }) - const inFlight = engine.swap('e2', async () => { - const building = new RecordingAdapter('e2') - building.onApply = () => gate // the build parks mid-fold - return building - }) - - // While the first swap builds, a second one is refused — typed. - const refusal = await engine.swap('e2', async () => new RecordingAdapter('e2')).catch((e) => e) - expect(refusal).toBeInstanceOf(SwapInFlightError) - expect((refusal as SwapInFlightError).family).toBe('e2') - - release() - const done = await inFlight - expect(done.watermark).toBe(8) - // Single-flight released: a follow-up swap is admitted again. - const again = await engine.swap('e2', async () => new RecordingAdapter('e2')) - expect(again.watermark).toBe(8) - }) - - it('a failed build discards the partial replacement and leaves the old adapter serving', async () => { - const source = scriptedSource(() => range(1, 8)) - const engine = new ReprojectionEngine({ source, batchSize: 4 }) - const oldAdapter = new RecordingAdapter('e3') - engine.register(oldAdapter) - await engine.advance('e3', { budgetMs: 10_000 }) - - let failed!: RecordingAdapter - await expect( - engine.swap('e3', async () => { - failed = new RecordingAdapter('e3') - failed.hardFail.add(5) // an UNTYPED failure mid-build - return failed - }) - ).rejects.toThrow(/disk exploded/) - - expect(failed.discarded).toBe(1) // the partial build was cleaned up - expect(oldAdapter.discarded).toBe(0) - expect(engine.getAdapter('e3')).toBe(oldAdapter) // still serving, untouched - expect(oldAdapter.watermark()).toBe(8) - }) -}) - -describe('reprojection engine — (f) quarantine: the fourth answer class', () => { - it('a typed poison fact is skipped, ledgered, and the rest folds to quarantined', async () => { - const source = scriptedSource(() => range(1, 10)) - const engine = new ReprojectionEngine({ source, batchSize: 4 }) - const adapter = new RecordingAdapter('f') - adapter.poison.add(6) - engine.register(adapter) - - const result = await engine.advance('f', { budgetMs: 10_000 }) - - expect(result.status).toBe('quarantined') - expect(result.watermark).toBe(10) - expect(result.applied).toBe(9) // every generation but the poison - expect(adapter.batches.flat().sort((x, y) => x - y)).toEqual([1, 2, 3, 4, 5, 7, 8, 9, 10]) - expect(adapter.state.has(6)).toBe(false) - - const ledger = engine.quarantined('f') - expect(ledger).toHaveLength(1) - expect(ledger[0].generation).toBe(6) - expect(ledger[0].error).toBeInstanceOf(ProjectionApplyError) - expect(ledger[0].error.recordIndex).toBe(1) // 6 sat at index 1 of [5..8] - expect(typeof ledger[0].at).toBe('number') - }) - - it('narration doubles: warns on the 1st, 2nd, and 4th quarantine — not the 3rd', async () => { - const warnSpy = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) - const source = scriptedSource(() => range(1, 10)) - const engine = new ReprojectionEngine({ source, batchSize: 10 }) - const adapter = new RecordingAdapter('f2') - for (const g of [2, 4, 6, 8]) adapter.poison.add(g) - engine.register(adapter) - - const result = await engine.advance('f2', { budgetMs: 10_000 }) - - expect(result.status).toBe('quarantined') - expect(result.watermark).toBe(10) - expect(result.applied).toBe(6) - expect(engine.quarantined('f2').map((q) => q.generation)).toEqual([2, 4, 6, 8]) - const quarantineWarns = warnSpy.mock.calls.filter((c) => String(c[0]).includes('quarantined generation')) - // 4 entries, narrated at counts 1, 2, and 4 — the 3rd stayed quiet. - expect(quarantineWarns).toHaveLength(3) - expect(quarantineWarns.map((c) => String(c[0]))).toEqual([ - expect.stringContaining('(1 quarantined total)'), - expect.stringContaining('(2 quarantined total)'), - expect.stringContaining('(4 quarantined total)') - ]) - }) - - it('an all-poison window still advances the stamp via an empty applyBatch', async () => { - const source = scriptedSource(() => range(1, 3)) - const engine = new ReprojectionEngine({ source, batchSize: 3 }) - const adapter = new RecordingAdapter('f3') - for (const g of [1, 2, 3]) adapter.poison.add(g) - engine.register(adapter) - - const result = await engine.advance('f3', { budgetMs: 10_000 }) - - expect(result.status).toBe('quarantined') - expect(result.watermark).toBe(3) - expect(result.applied).toBe(0) - // The final call carried NO facts but a real upTo — the pure watermark - // advance past poison, stamped by the adapter itself. - expect(adapter.batches).toEqual([[]]) - expect(adapter.upTos).toEqual([3]) - expect(engine.quarantined('f3').map((q) => q.generation)).toEqual([1, 2, 3]) - }) - - it('a NON-typed throw aborts the advance loudly — unknown failure is never poison', async () => { - const source = scriptedSource(() => range(1, 8)) - const engine = new ReprojectionEngine({ source, batchSize: 4 }) - const adapter = new RecordingAdapter('f4') - adapter.hardFail.add(5) - engine.register(adapter) - - await expect(engine.advance('f4', { budgetMs: 10_000 })).rejects.toThrow(/disk exploded at generation 5/) - - expect(adapter.watermark()).toBe(4) // the clean first batch landed; nothing after - expect(engine.quarantined('f4')).toEqual([]) // no ledger entry for an unknown failure - }) - - it('an adapter re-condemning an already-quarantined generation is refused loudly', async () => { - const source = scriptedSource(() => range(1, 4)) - const engine = new ReprojectionEngine({ source, batchSize: 4 }) - // A misbehaving adapter: always blames generation 3, even once it is - // filtered out of its batches. - const adapter: ProjectionAdapter = { - family: 'f5', - watermark: () => null, - applyBatch: async () => { - throw new ProjectionApplyError({ generation: 3, cause: new Error('always 3') }) - }, - discard: async () => {} - } - engine.register(adapter) - - await expect(engine.advance('f5', { budgetMs: 10_000 })).rejects.toThrow(/ALREADY quarantined/) - expect(engine.quarantined('f5').map((q) => q.generation)).toEqual([3]) - }) - - it('an adapter that never stamps is refused loudly instead of spinning', async () => { - const source = scriptedSource(() => range(1, 4)) - const engine = new ReprojectionEngine({ source, batchSize: 2 }) - const adapter: ProjectionAdapter = { - family: 'f6', - watermark: () => null, // never advances - applyBatch: async () => {}, - discard: async () => {} - } - engine.register(adapter) - - await expect(engine.advance('f6', { budgetMs: 10_000 })).rejects.toThrow(/not stamping/) - }) -}) - -describe('reprojection engine — (g) discard lands on the losing adapter after a swap', () => { - it('the OLD adapter is discarded exactly once, after the flip; the winner is never discarded', async () => { - const source = scriptedSource(() => range(1, 6)) - const engine = new ReprojectionEngine({ source, batchSize: 3 }) - const losing = new RecordingAdapter('g') - engine.register(losing) - await engine.advance('g', { budgetMs: 10_000 }) - expect(losing.discarded).toBe(0) // serving adapters are never discarded - - let winner!: RecordingAdapter - await engine.swap('g', async () => { - winner = new RecordingAdapter('g') - winner.onApply = () => { - // Mid-build the loser still serves and is still intact. - expect(losing.discarded).toBe(0) - } - return winner - }) - - expect(losing.discarded).toBe(1) - expect(winner.discarded).toBe(0) - expect(engine.getAdapter('g')).toBe(winner) - }) -}) - -describe('FactLogSource — the production source enforces the window contract', () => { - it('delegates to the injected callback and passes clean windows through', async () => { - const calls: Array<[number, number]> = [] - const source = new FactLogSource(async (from, limit) => { - calls.push([from, limit]) - return range(from + 1, Math.min(from + limit, 5)).map(fact) - }) - const facts = await source.scan(2, 2) - expect(facts.map((f) => f.generation)).toEqual([3, 4]) - expect(calls).toEqual([[2, 2]]) - expect(await source.scan(5, 3)).toEqual([]) - }) - - it('refuses out-of-contract callbacks loudly: oversize, non-ascending, at-or-below from', async () => { - const oversize = new FactLogSource(async () => range(1, 5).map(fact)) - await expect(oversize.scan(0, 2)).rejects.toThrow(/contract violation/) - - const unsorted = new FactLogSource(async () => [fact(3), fact(2)]) - await expect(unsorted.scan(0, 10)).rejects.toThrow(/strictly ascending/) - - const stale = new FactLogSource(async () => [fact(2)]) - await expect(stale.scan(2, 10)).rejects.toThrow(/strictly ascending/) - }) - - it('validates its own window arguments', async () => { - const source = new FactLogSource(async () => []) - await expect(source.scan(-1, 5)).rejects.toThrow(/non-negative integer/) - await expect(source.scan(0, 0)).rejects.toThrow(/positive integer/) - }) -}) diff --git a/tests/unit/storage/torn-record-loud.test.ts b/tests/unit/storage/torn-record-loud.test.ts deleted file mode 100644 index d35f8b43..00000000 Binary files a/tests/unit/storage/torn-record-loud.test.ts and /dev/null differ diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index 4b078146..21f918f1 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -33,10 +33,6 @@ const MANUAL_ONLY = new Set([ // Conformance suites run as an explicit gate stage (both engines run them // by direct invocation), never swept into the unit/integration configs. 'tests/conformance/collider-fidelity.test.ts', - // Golden-log fold-conformance oracle: the two-implementation contract pin - // (byte + fold digests) — runs in the explicit conformance gate stage, - // same invocation family as the other conformance suites. - 'tests/conformance/golden-log-fold.test.ts', 'tests/api/performance-benchmarks.test.ts', 'tests/critical-neural-validation.test.ts', 'tests/critical-performance-benchmark.test.ts', diff --git a/tests/unit/utils/metadataIndex-nested-orderby.test.ts b/tests/unit/utils/metadataIndex-nested-orderby.test.ts deleted file mode 100644 index 55dab59b..00000000 --- a/tests/unit/utils/metadataIndex-nested-orderby.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** - * @module tests/unit/utils/metadataIndex-nested-orderby - * @description THE NESTED-FIELD ADDRESSING PIN for ordered reads (the - * field-addressing law, dotted-path clause). The defect this keeps dead: - * `orderBy` on a nested user metadata field (dotted path, e.g. - * `orderBy: 'profile.score'` over `metadata: { profile: { score: 7 } }`) - * silently returned insertion order — a no-op sort — because the sort - * path's value resolution read flat bag keys only. The law: a dotted user - * address is either SERVED CORRECTLY (the batched resolver walks inside - * the bag) or REFUSED with a typed UnresolvableFieldError — never a silent - * pass-through. Both spellings (`profile.score` / `metadata.profile.score`) - * are the same address; the filter side (`where: { 'profile.score': … }`) - * obeys the same law. - */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { Brainy, UnresolvableFieldError } from '../../../src/index.js' -import { NounType } from '../../../src/types/graphTypes.js' - -const ROWS = 30 - -describe('nested (dotted-path) user field orderBy — the field-addressing law', () => { - let brain: Brainy - /** id → nested score, for the rows that carry profile.score */ - const scoreById = new Map() - /** ids of the two rows WITHOUT a profile bag */ - let noProfileIds: string[] = [] - - beforeAll(async () => { - brain = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) - await brain.init() - for (let i = 0; i < ROWS; i++) { - // (i * 11) % 30 is a permutation of 0..29 (gcd(11,30)=1): every score - // distinct, insertion order maximally different from value order — a - // silent insertion-order pass-through cannot accidentally look sorted. - const score = (i * 11) % ROWS - const id = await brain.add({ - data: `row ${i}`, - type: NounType.Document, - metadata: { profile: { score }, plain: i } - }) - scoreById.set(id, score) - } - const a = await brain.add({ - data: 'no-profile a', - type: NounType.Document, - metadata: { plain: 1000 } - }) - const b = await brain.add({ - data: 'no-profile b', - type: NounType.Document, - metadata: { plain: 1001 } - }) - noProfileIds = [a, b].sort() - }, 120000) - - afterAll(async () => { - await brain.close().catch(() => {}) - }) - - /** Assert one complete ordered read against the sealed ordering contract. */ - function assertOrdered( - rows: Array<{ id: string }>, - order: 'asc' | 'desc', - label: string - ): void { - // Rows are NEVER dropped: all 30 scored + 2 profile-less rows come back. - expect(rows.length, `${label}: complete result`).toBe(ROWS + 2) - - // Missing-value rows sort LAST in BOTH directions, ties by id ascending. - const lastTwo = rows.slice(-2).map((r) => r.id) - expect(lastTwo, `${label}: missing-value rows LAST, id asc`).toEqual(noProfileIds) - - // The scored 30 are ordered by the NESTED value — the exact permutation, - // not insertion order. - const observed = rows.slice(0, ROWS).map((r) => scoreById.get(r.id)) - const wanted = [...scoreById.values()].sort((x, y) => - order === 'asc' ? x - y : y - x - ) - expect(observed, `${label}: nested values in ${order} order`).toEqual(wanted) - } - - it('orderBy: "profile.score" desc — served correctly, missing rows LAST (never a silent insertion-order no-op)', async () => { - const rows = await brain.find({ - type: NounType.Document, - orderBy: 'profile.score', - order: 'desc', - limit: 40 - }) - assertOrdered(rows, 'desc', 'bare dotted, desc') - }) - - it('orderBy: "profile.score" asc — same law in the other direction', async () => { - const rows = await brain.find({ - type: NounType.Document, - orderBy: 'profile.score', - order: 'asc', - limit: 40 - }) - assertOrdered(rows, 'asc', 'bare dotted, asc') - }) - - it('explicit spelling "metadata.profile.score" is the SAME address — identical result', async () => { - const bare = await brain.find({ - type: NounType.Document, - orderBy: 'profile.score', - order: 'desc', - limit: 40 - }) - const explicit = await brain.find({ - type: NounType.Document, - orderBy: 'metadata.profile.score', - order: 'desc', - limit: 40 - }) - assertOrdered(explicit, 'desc', 'metadata.-prefixed, desc') - expect( - explicit.map((r) => r.id), - 'both spellings resolve to the identical ordered id sequence' - ).toEqual(bare.map((r) => r.id)) - }) - - it('a dotted path carried by NO entity REFUSES with UnresolvableFieldError — never a silent insertion-order return', async () => { - await expect( - brain.find({ - type: NounType.Document, - orderBy: 'no.such.path', - order: 'desc', - limit: 40 - }) - ).rejects.toThrow(UnresolvableFieldError) - }) - - it('dotted where: { "profile.score": 7 } finds exactly the right row — the filter side of the same law', async () => { - const wantedId = [...scoreById.entries()].find(([, s]) => s === 7)![0] - const rows = await brain.find({ - type: NounType.Document, - where: { 'profile.score': 7 }, - limit: 40 - }) - expect(rows.map((r) => r.id)).toEqual([wantedId]) - }) -}) diff --git a/tests/unit/utils/metadataIndex-sort-callshape.test.ts b/tests/unit/utils/metadataIndex-sort-callshape.test.ts deleted file mode 100644 index ffe89566..00000000 --- a/tests/unit/utils/metadataIndex-sort-callshape.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -/** - * @module tests/unit/utils/metadataIndex-sort-callshape - * @description THE ASYMPTOTIC CALL-SHAPE PIN for ordered reads - * (BRAINY-PROD-LATENCY-TRIAD, David-approved plan Track A1). The defect it - * keeps dead: `getSortedIdsForFilter`'s value resolution did a SERIAL - * `storage.getNoun()` (the heavyweight VECTOR record) per filtered row — - * 62–98ms × 3,224 rows = the measured 199–317 SECOND production sort, with - * `topK` applied only after the full scan. These pins assert the SHAPE of - * the storage traffic, not wall-clock (latency-blind, so they hold on any - * machine): an ordered read performs ZERO per-row vector-record reads and - * resolves sort values through BATCHED metadata-record calls only. - */ -import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' -import { NounType } from '../../../src/types/graphTypes.js' - -const ROWS = 60 - -describe('ordered reads — the batched call-shape law (no per-row storage loops)', () => { - let brain: Brainy - let storage: { - getNoun: (id: string) => Promise - getNounMetadata: (id: string) => Promise - getNounMetadataBatch: (ids: string[]) => Promise> - } - - beforeAll(async () => { - brain = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) - await brain.init() - for (let i = 0; i < ROWS; i++) { - await brain.add({ - data: `row ${i}`, - type: NounType.Document, - metadata: { rank: (i * 7) % ROWS, plain: `p${i}` } - }) - } - storage = (brain as unknown as { storage: typeof storage }).storage - }, 120000) - - afterAll(async () => { - await brain.close().catch(() => {}) - }) - - it('user-field orderBy: zero vector-record reads, zero serial metadata reads — batch calls only', async () => { - const getNounSpy = vi.spyOn(storage, 'getNoun') - const singleReadSpy = vi.spyOn(storage, 'getNounMetadata') - const batchSpy = vi.spyOn(storage, 'getNounMetadataBatch') - - const rows = await brain.find({ - type: NounType.Document, - orderBy: 'rank', - order: 'desc', - limit: 10 - }) - expect(rows.length).toBe(10) - expect((rows[0].metadata as Record).rank).toBe(ROWS - 1) - - // THE PIN: the sort's value resolution never opens a vector record and - // never falls into a per-row metadata loop. (Result hydration after - // pagination is allowed to read; the SORT itself must be batch-only — - // hence the ceiling: strictly fewer single reads than sorted rows.) - expect(getNounSpy.mock.calls.length, 'per-row vector-record reads in an ordered read').toBe(0) - expect(batchSpy.mock.calls.length, 'the batch door was used').toBeGreaterThanOrEqual(1) - expect( - singleReadSpy.mock.calls.length, - 'serial per-row metadata reads (the 199s shape)' - ).toBeLessThan(ROWS / 2) - - vi.restoreAllMocks() - }) - - it('system.createdAt orderBy: exact values from batched records — the bucketed index is never a per-row disk excuse', async () => { - const getNounSpy = vi.spyOn(storage, 'getNoun') - const batchSpy = vi.spyOn(storage, 'getNounMetadataBatch') - - const rows = await brain.find({ - type: NounType.Document, - orderBy: 'system.createdAt', - order: 'asc', - limit: 15 - }) - expect(rows.length).toBe(15) - - expect(getNounSpy.mock.calls.length, 'per-row vector-record reads').toBe(0) - expect(batchSpy.mock.calls.length).toBeGreaterThanOrEqual(1) - - // Exactness: ascending createdAt must be non-decreasing with full - // millisecond precision (the old path sorted minute-BUCKETED values or - // paid a per-row disk read for exact ones — both are dead). Find results - // carry the timestamps on the nested full entity. - const stamps = rows.map( - (r) => ((r as unknown as { entity?: { createdAt?: number } }).entity?.createdAt ?? - (r as unknown as { createdAt?: number }).createdAt) as number - ) - for (let i = 1; i < stamps.length; i++) { - expect(stamps[i]).toBeGreaterThanOrEqual(stamps[i - 1]) - } - - vi.restoreAllMocks() - }) - - it('the ordering contract survives the batch path: missing values LAST both directions, ties by id asc, rows never dropped', async () => { - // Three rows lack `rank`? No — all carry it; add two rows WITHOUT it. - const a = await brain.add({ data: 'no-rank a', type: NounType.Document, metadata: { plain: 'x' } }) - const b = await brain.add({ data: 'no-rank b', type: NounType.Document, metadata: { plain: 'y' } }) - - for (const order of ['asc', 'desc'] as const) { - const rows = await brain.find({ - type: NounType.Document, - orderBy: 'rank', - order, - limit: ROWS + 10 - }) - expect(rows.length, `complete result (${order})`).toBe(ROWS + 2) - const lastTwo = rows.slice(-2).map((r) => r.id).sort() - expect(lastTwo, `missing-value rows sort LAST (${order})`).toEqual([a, b].sort()) - } - }) -}) diff --git a/tests/unit/utils/metadataIndex-watermark.test.ts b/tests/unit/utils/metadataIndex-watermark.test.ts deleted file mode 100644 index 6c195b35..00000000 --- a/tests/unit/utils/metadataIndex-watermark.test.ts +++ /dev/null @@ -1,171 +0,0 @@ -/** - * @module tests/unit/utils/metadataIndex-watermark - * @description Watermark-stamp pins for the metadata projection. - * - * THE LAW under test: every persisted projection artifact carries a stamp - * asserting "this state reflects every committed generation ≤ W and nothing - * above W, atomically" — written AFTER every byte it certifies is durable — - * and at load the owner computes the three-way verdict: - * stamped==committed → 'adopt' · stampedcommitted OR unstamped → 'rescan', LOUDLY. - * Same rule, same verdict names as the shipped aggregation machinery - * (AggregationIndex.stateAdoptionVerdict). - * - * The verdict is COMPUTED AND EXPOSED only — these pins assert no rebuild - * trigger changed; acting on 'catchup' lands with the coordinator's wiring. - */ -import { describe, it, expect, vi, afterEach } from 'vitest' -import { v4 as uuidv4 } from 'uuid' -import { - MetadataIndexManager, - METADATA_INDEX_STAMP_KEY -} from '../../../src/utils/metadataIndex.js' -import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' -import { prodLog } from '../../../src/utils/logger.js' - -/** Fresh storage with a controllable committed generation. */ -async function makeStorage(committed: number | null): Promise { - const storage = new MemoryStorage() - await storage.init() - if (committed !== null) { - vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed) - } - return storage -} - -/** Set (or reset) the mocked committed generation on an existing storage. */ -function setCommitted(storage: MemoryStorage, committed: number): void { - vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed) -} - -/** Session 1: index a field, optionally stamp, flush — the durable artifact. */ -async function writeArtifact( - storage: MemoryStorage, - stamp: number | null -): Promise { - const index = new MetadataIndexManager(storage) - await index.init() - await index.addToIndex(uuidv4(), { status: 'active', role: 'admin' }) - if (stamp !== null) index.stampWatermark(stamp) - await index.flush() -} - -/** Session 2: reopen on the same storage and return the loaded manager. */ -async function reopen(storage: MemoryStorage): Promise { - const index = new MetadataIndexManager(storage) - await index.init() - return index -} - -afterEach(() => { - vi.restoreAllMocks() -}) - -describe('metadata index — watermark stamp + three-way load verdict', () => { - it("save-with-stamp then reopen at the same committed generation → 'adopt', zero-work verdict", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('adopt') - expect(index.watermark()).toBe(5) - expect(index.watermarkGap()).toBeNull() - }) - - it("stamp BEHIND the committed generation → 'catchup' with the exact gap reported", async () => { - const storage = await makeStorage(5) - await writeArtifact(storage, 5) - - // Later commits landed after the last stamped flush (unclean exit shape). - setCommitted(storage, 8) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('catchup') - expect(index.watermark()).toBe(5) - expect(index.watermarkGap()).toEqual({ from: 5, to: 8 }) - }) - - it("stamp ABOVE the committed generation → 'rescan', said out loud", async () => { - const storage = await makeStorage(9) - await writeArtifact(storage, 9) - - // A truncated log on a copied store pulled the watermark back. - setCommitted(storage, 4) - - const warnSpy = vi.spyOn(prodLog, 'warn') - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('rescan') - expect(index.watermarkGap()).toBeNull() - const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n') - expect(said).toContain('RESCAN') - expect(said).toContain('ABOVE') - }) - - it("legacy unstamped artifact on a stamped store → 'rescan', LOUD — never a silent adopt", async () => { - const storage = await makeStorage(3) - await writeArtifact(storage, null) // pre-stamp brain: data flushed, no stamp - - expect(await storage.getMetadata(METADATA_INDEX_STAMP_KEY)).toBeNull() - - const warnSpy = vi.spyOn(prodLog, 'warn') - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('rescan') - expect(index.watermark()).toBeNull() - const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n') - expect(said).toContain('RESCAN') - expect(said).toContain('unstamped') - }) - - it("a store with no committed-generation capability keeps pre-stamp behavior → 'adopt'", async () => { - const storage = await makeStorage(null) // committedGeneration() → null - await writeArtifact(storage, null) - - const index = await reopen(storage) - expect(index.watermarkVerdict()).toBe('adopt') - expect(index.watermark()).toBeNull() - }) - - it('STAMP-AFTER-DATA: the stamp is the last saveMetadata of the flush, after registry and field indexes', async () => { - const storage = await makeStorage(2) - const index = new MetadataIndexManager(storage) - await index.init() - await index.addToIndex(uuidv4(), { status: 'active' }) - - const keys: string[] = [] - const originalSave = storage.saveMetadata.bind(storage) - vi.spyOn(storage, 'saveMetadata').mockImplementation(async (id, metadata) => { - keys.push(id) - return originalSave(id, metadata) - }) - - index.stampWatermark(2) - await index.flush() - - const stampAt = keys.indexOf(METADATA_INDEX_STAMP_KEY) - expect(stampAt, 'stamp record was written').toBeGreaterThanOrEqual(0) - expect(stampAt, 'stamp is the FINAL metadata write of the flush').toBe(keys.length - 1) - const registryAt = keys.indexOf('__metadata_field_registry__') - expect(registryAt, 'field registry written during this flush').toBeGreaterThanOrEqual(0) - expect(registryAt).toBeLessThan(stampAt) - - // The persisted stamp record carries the required shape. - const record = (await storage.getMetadata(METADATA_INDEX_STAMP_KEY)) as { - watermark: number - formatVersion: number - stampedAt: number - } - expect(record.watermark).toBe(2) - expect(record.formatVersion).toBe(1) - expect(typeof record.stampedAt).toBe('number') - }) - - it('a flush WITHOUT a pending stamp writes no stamp record (no phantom certification)', async () => { - const storage = await makeStorage(2) - const index = new MetadataIndexManager(storage) - await index.init() - await index.addToIndex(uuidv4(), { status: 'active' }) - await index.flush() - - expect(await storage.getMetadata(METADATA_INDEX_STAMP_KEY)).toBeNull() - }) -})