From a17540649723e4c001f535720b26fa6519aad977 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 10 Jul 2026 18:06:37 -0700
Subject: [PATCH 01/82] fix: transact forward references resolve graph endpoint
 ints at execute time
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

transact() promises atomic forward references — add an entity and relate to
it in one batch — but the planner resolved relationship endpoint ints at
PLAN time, before the batch's add operations had applied. Asking the id
mapper about an entity that does not exist yet made the (correctly strict)
native mapper throw in EntityIdMapper.getOrAssign, so
transact([{op:'add', id:X}, {op:'relate', to:X}]) failed on native
deployments; the permissive JS mapper masked the bug and silently leaked an
id assignment whenever a batch was later rejected by a commit precondition
(normal control flow since the conditional-commit CAS landed).

Make endpoint resolution lazy: AddToGraphIndexOperation and
RemoveFromGraphIndexOperation take VerbEndpointInts — an eager
{sourceInt, targetInt} or a thunk evaluated when the operation EXECUTES,
mirroring the constructor's already-lazy generationFn. The four
transact-planner sites (relate, its bidirectional reverse edge, remove's
relationship cascade, unrelate) pass thunks, so resolution happens inside
the commit after same-batch adds have applied; the seven single-operation
sites keep eager objects (their endpoints provably pre-exist — relate
validates both, unrelate/updateRelation/remove read the existing verb).
The remove operation's rollback captures the ints resolved at execute so
its re-add uses the same mappings.

Byproduct fix: a rejected batch no longer pollutes the id mapper — the
regression suite pins this (mapper has no assignment for the phantom
entity after a precondition-rejected forward-ref batch), alongside the
exact reported shape, both-endpoints-in-batch, bidirectional,
add+relate+remove in one batch, and the split-transact control
(tests/integration/transact-forward-ref-graph.test.ts).
---
 src/brainy.ts                                 |  42 +++---
 src/transaction/operations/IndexOperations.ts |  49 ++++--
 .../transact-forward-ref-graph.test.ts        | 140 ++++++++++++++++++
 3 files changed, 202 insertions(+), 29 deletions(-)
 create mode 100644 tests/integration/transact-forward-ref-graph.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index d54cfe1c..422cb2e6 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -3024,7 +3024,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           // rollback can re-add through the BigInt addVerb contract)
           const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
           tx.addOperation(
-            new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration)
+            new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration)
           )
           // Delete verb metadata
           tx.addOperation(
@@ -3734,7 +3734,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         // Operation 3: Add to graph index for O(1) lookups
         tx.addOperation(
           new AddToGraphIndexOperation(
-            this.graphIndex, verb, sourceInt, targetInt,
+            this.graphIndex, verb, { sourceInt, targetInt },
             this.graphWriteGeneration,
             (verbInt) => this.cacheVerbInt(verbInt, id)
           )
@@ -3772,7 +3772,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           // Operation 6: Add reverse relationship to graph index
           tx.addOperation(
             new AddToGraphIndexOperation(
-              this.graphIndex, reverseVerb, targetInt, sourceInt,
+              this.graphIndex, reverseVerb, { sourceInt: targetInt, targetInt: sourceInt },
               this.graphWriteGeneration,
               (verbInt) => this.cacheVerbInt(verbInt, reverseId)
             )
@@ -3854,7 +3854,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         if (verb && endpointInts) {
           tx.addOperation(
             new RemoveFromGraphIndexOperation(
-              this.graphIndex, verb, endpointInts.sourceInt, endpointInts.targetInt, this.graphWriteGeneration
+              this.graphIndex, verb, endpointInts, this.graphWriteGeneration
             )
           )
         }
@@ -4006,12 +4006,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       if (typeChanged && reindexInts) {
         tx.addOperation(
           new RemoveFromGraphIndexOperation(
-            this.graphIndex, existing, reindexInts.sourceInt, reindexInts.targetInt, this.graphWriteGeneration
+            this.graphIndex, existing, reindexInts, this.graphWriteGeneration
           )
         )
         tx.addOperation(
           new AddToGraphIndexOperation(
-            this.graphIndex, verbForIndex, reindexInts.sourceInt, reindexInts.targetInt,
+            this.graphIndex, verbForIndex, reindexInts,
             this.graphWriteGeneration,
             (verbInt) => this.cacheVerbInt(verbInt, params.id)
           )
@@ -6634,7 +6634,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
               for (const verb of allVerbs) {
                 const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
                 tx.addOperation(
-                  new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration)
+                  new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration)
                 )
                 tx.addOperation(
                   new DeleteVerbMetadataOperation(this.storage, verb.id)
@@ -9014,9 +9014,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
     plan.operations.push(new DeleteNounMetadataOperation(this.storage, id))
     for (const verb of cascade.values()) {
-      const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
       plan.operations.push(
-        new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration),
+        // Endpoint ints resolve at EXECUTE time — a cascade verb (or its
+        // endpoints) may have been created earlier in this same batch, so a
+        // plan-time resolution would ask the id mapper about entities that do
+        // not exist yet (the native mapper rightly refuses).
+        new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration),
         new DeleteVerbMetadataOperation(this.storage, verb.id)
       )
       plan.touchedVerbs.push(verb.id)
@@ -9163,8 +9166,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       data: params.data,
       createdAt: now
     }
-    const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
-
     plan.operations.push(
       new SaveVerbOperation(this.storage, {
         id,
@@ -9175,7 +9176,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         targetId: params.to
       }),
       new SaveVerbMetadataOperation(this.storage, id, verbMetadata),
-      new AddToGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration, (verbInt) =>
+      // Endpoint ints resolve at EXECUTE time — after any same-batch add of
+      // an endpoint has applied. Plan-time resolution was a consumer-reported
+      // native/JS parity bug: transact([add X, relate →X]) asked the native
+      // id mapper to assign an int for an entity that did not exist yet.
+      new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) =>
         this.cacheVerbInt(verbInt, id)
       )
     )
@@ -9203,9 +9208,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         ...verb,
         id: reverseId,
         sourceId: params.to,
-        targetId: params.from,
-        sourceInt: targetInt,
-        targetInt: sourceInt
+        targetId: params.from
+        // sourceInt/targetInt are mirrored onto this object when the graph
+        // operation resolves endpoints at execute time.
       }
       plan.operations.push(
         new SaveVerbOperation(this.storage, {
@@ -9217,7 +9222,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           targetId: params.from
         }),
         new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata),
-        new AddToGraphIndexOperation(this.graphIndex, reverseVerb, targetInt, sourceInt, this.graphWriteGeneration, (verbInt) =>
+        new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) =>
           this.cacheVerbInt(verbInt, reverseId)
         )
       )
@@ -9259,9 +9264,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       : (state.verbs.get(id) ?? (await this.storage.getVerb(id)))
 
     if (verb) {
-      const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
       plan.operations.push(
-        new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt, this.graphWriteGeneration)
+        // Endpoint ints resolve at EXECUTE time — the verb (or its endpoints)
+        // may have been created earlier in this same batch (forward refs).
+        new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration)
       )
     }
     plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id))
diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts
index f0bede05..97c39270 100644
--- a/src/transaction/operations/IndexOperations.ts
+++ b/src/transaction/operations/IndexOperations.ts
@@ -176,14 +176,36 @@ export class RemoveFromMetadataIndexOperation implements Operation {
  * generation is reused for the rollback removal, so an add and its undo
  * reference one watermark in a provider's per-generation edge chain.
  */
+/**
+ * A verb's interned endpoint ints — eager (already resolved), or a thunk
+ * evaluated when the operation EXECUTES. The lazy form exists for
+ * `transact()` forward references: a relate whose endpoint is added in the
+ * SAME batch cannot resolve ints at plan time (the entity does not exist yet
+ * — a strict native id mapper rightly refuses to assign, and even a permissive
+ * one would leak the assignment if the batch is rejected at precommit).
+ * Deferring to execute time resolves after the batch's add operations have
+ * applied, mirroring how `generationFn` is already evaluated lazily.
+ */
+export type VerbEndpointInts =
+  | { sourceInt: bigint; targetInt: bigint }
+  | (() => { sourceInt: bigint; targetInt: bigint })
+
+/** Resolve a {@link VerbEndpointInts} at execution time. */
+function resolveEndpointInts(
+  endpoints: VerbEndpointInts
+): { sourceInt: bigint; targetInt: bigint } {
+  return typeof endpoints === 'function' ? endpoints() : endpoints
+}
+
 export class AddToGraphIndexOperation implements Operation {
   readonly name = 'AddToGraphIndex'
 
   /**
    * @param index - The graph-index provider (JS baseline or native).
    * @param verb - The verb to index (`sourceInt`/`targetInt` mirrored on it).
-   * @param sourceInt - The source entity's interned int.
-   * @param targetInt - The target entity's interned int.
+   * @param endpointInts - The endpoints' interned ints — eager, or a thunk
+   *   evaluated at execute time (REQUIRED for transact forward references;
+   *   see {@link VerbEndpointInts}).
    * @param generationFn - Resolves the commit generation to stamp this edge at,
    *   evaluated when the operation executes (see class note).
    * @param onVerbInt - Optional hook invoked with the interned verb int
@@ -192,17 +214,18 @@ export class AddToGraphIndexOperation implements Operation {
   constructor(
     private readonly index: GraphIndexProvider,
     private readonly verb: GraphVerb,
-    private readonly sourceInt: bigint,
-    private readonly targetInt: bigint,
+    private readonly endpointInts: VerbEndpointInts,
     private readonly generationFn: () => bigint,
     private readonly onVerbInt?: (verbInt: bigint) => void
   ) {}
 
   async execute(): Promise<RollbackAction> {
     // Stamp this edge at the in-flight commit generation; reuse it for the
-    // rollback so add + undo reference the same watermark.
+    // rollback so add + undo reference the same watermark. Endpoint ints
+    // resolve HERE — after any same-batch adds have applied.
     const generation = this.generationFn()
-    const verbInt = await this.index.addVerb(this.verb, this.sourceInt, this.targetInt, generation)
+    const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts)
+    const verbInt = await this.index.addVerb(this.verb, sourceInt, targetInt, generation)
     this.onVerbInt?.(verbInt)
 
     // Return rollback action
@@ -231,28 +254,32 @@ export class RemoveFromGraphIndexOperation implements Operation {
   /**
    * @param index - The graph-index provider (JS baseline or native).
    * @param verb - The verb being removed (required for rollback re-add).
-   * @param sourceInt - The source entity's interned int (rollback re-add).
-   * @param targetInt - The target entity's interned int (rollback re-add).
+   * @param endpointInts - The endpoints' interned ints for the rollback
+   *   re-add — eager, or a thunk evaluated at execute time (required when the
+   *   verb or its endpoints were created in the SAME transact batch; see
+   *   {@link VerbEndpointInts}).
    * @param generationFn - Resolves the commit generation for this removal,
    *   evaluated when the operation executes.
    */
   constructor(
     private readonly index: GraphIndexProvider,
     private readonly verb: GraphVerb, // Required for rollback
-    private readonly sourceInt: bigint,
-    private readonly targetInt: bigint,
+    private readonly endpointInts: VerbEndpointInts,
     private readonly generationFn: () => bigint
   ) {}
 
   async execute(): Promise<RollbackAction> {
     // Resolve the removal generation once; reuse it for the rollback re-add.
+    // Endpoint ints resolve HERE (after any same-batch adds applied) and are
+    // captured for the rollback, whose re-add must use the same mappings.
     const generation = this.generationFn()
+    const { sourceInt, targetInt } = resolveEndpointInts(this.endpointInts)
     await this.index.removeVerb(this.verb.id, generation)
 
     // Return rollback action
     return async () => {
       // Re-add verb with original data
-      await this.index.addVerb(this.verb, this.sourceInt, this.targetInt, generation)
+      await this.index.addVerb(this.verb, sourceInt, targetInt, generation)
     }
   }
 }
diff --git a/tests/integration/transact-forward-ref-graph.test.ts b/tests/integration/transact-forward-ref-graph.test.ts
new file mode 100644
index 00000000..a9619424
--- /dev/null
+++ b/tests/integration/transact-forward-ref-graph.test.ts
@@ -0,0 +1,140 @@
+/**
+ * @module tests/integration/transact-forward-ref-graph
+ * @description Regression for a consumer-reported native/JS parity bug:
+ * `transact([{op:'add', id:X}, {op:'relate', to:X}])` — the platform's first
+ * atomic add+relate consumer — threw in `EntityIdMapper.getOrAssign` at PLAN
+ * time on the native id mapper (an entity added in the same batch does not
+ * exist when the planner runs, so a strict mapper rightly refuses to assign;
+ * the permissive JS mapper masked the bug — and silently leaked an int
+ * assignment whenever the batch was later rejected at precommit).
+ *
+ * Fix: graph-index operations resolve endpoint ints at EXECUTE time (a lazy
+ * thunk, mirroring the existing lazy `generationFn`), after the batch's add
+ * operations have applied. These tests pin every forward-ref shape on the JS
+ * path, including the JS-observable deferral proof: a REJECTED batch leaves
+ * the id mapper untouched (pre-fix, plan-time resolution assigned ints for
+ * entities that never came to exist). The native side is locked in by the
+ * accelerator's own gate running this same shape.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType, VerbType } from '../../src/types/graphTypes.js'
+
+let seq = 0
+const freshId = (): string =>
+  `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}`
+
+describe('transact() forward references into the graph index (native/JS parity)', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-fwd-ref-'))
+    brain = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      dimensions: 384,
+      silent: true
+    })
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('CASE 1 (the exact repro): add + relate-to-that-add in ONE transact', async () => {
+    const agent = await brain.add({ id: freshId(), data: 'agent probe', type: NounType.Person })
+    const thread = freshId()
+
+    const db = await brain.transact([
+      { op: 'add', id: thread, data: 'thread PROBE-ONE', type: NounType.Thing },
+      { op: 'relate', from: agent, to: thread, type: VerbType.RelatedTo }
+    ])
+
+    // The batch committed as one generation and the edge traverses.
+    expect(db.generation).toBeGreaterThan(0)
+    const related = await brain.related(agent)
+    expect(related.map((r: any) => r.to)).toContain(thread)
+  })
+
+  it('both endpoints created in-batch: add A + add B + relate A→B', async () => {
+    const a = freshId()
+    const b = freshId()
+    await brain.transact([
+      { op: 'add', id: a, data: 'node A', type: NounType.Thing },
+      { op: 'add', id: b, data: 'node B', type: NounType.Thing },
+      { op: 'relate', from: a, to: b, type: VerbType.Contains }
+    ])
+    expect((await brain.related(a)).map((r: any) => r.to)).toContain(b)
+  })
+
+  it('bidirectional relate to an in-batch add: both edges traverse', async () => {
+    const hub = await brain.add({ id: freshId(), data: 'hub', type: NounType.Thing })
+    const spoke = freshId()
+    await brain.transact([
+      { op: 'add', id: spoke, data: 'spoke', type: NounType.Thing },
+      { op: 'relate', from: hub, to: spoke, type: VerbType.RelatedTo, bidirectional: true }
+    ])
+    expect((await brain.related(hub)).map((r: any) => r.to)).toContain(spoke)
+    expect((await brain.related(spoke)).map((r: any) => r.to)).toContain(hub)
+  })
+
+  it('add + relate + remove in ONE transact: the cascade covers the in-batch verb', async () => {
+    const keeper = await brain.add({ id: freshId(), data: 'keeper', type: NounType.Thing })
+    const doomed = freshId()
+    await brain.transact([
+      { op: 'add', id: doomed, data: 'doomed', type: NounType.Thing },
+      { op: 'relate', from: keeper, to: doomed, type: VerbType.Contains },
+      { op: 'remove', id: doomed }
+    ])
+    expect(await brain.get(doomed)).toBeNull()
+    expect((await brain.related(keeper)).length).toBe(0)
+  })
+
+  it('CASE 2 control: the same two ops split across two transacts still work', async () => {
+    const agent = await brain.add({ id: freshId(), data: 'agent two', type: NounType.Person })
+    const thread = freshId()
+    await brain.transact([{ op: 'add', id: thread, data: 'thread two', type: NounType.Thing }])
+    await brain.transact([{ op: 'relate', from: agent, to: thread, type: VerbType.RelatedTo }])
+    expect((await brain.related(agent)).map((r: any) => r.to)).toContain(thread)
+  })
+
+  it('a REJECTED forward-ref batch leaves the id mapper untouched (the deferral proof)', async () => {
+    const existing = await brain.add({
+      id: freshId(),
+      data: 'cas anchor',
+      type: NounType.Thing
+    })
+    const never = freshId()
+
+    // Reject the batch via a stale per-op CAS — planning completes, the
+    // commit precondition throws, nothing applies.
+    await expect(
+      brain.transact([
+        { op: 'add', id: never, data: 'never exists', type: NounType.Thing },
+        { op: 'relate', from: existing, to: never, type: VerbType.RelatedTo },
+        { op: 'update', id: existing, metadata: { poke: 1 }, ifRev: 999 }
+      ])
+    ).rejects.toMatchObject({ name: 'RevisionConflictError' })
+
+    // Nothing applied…
+    expect(await brain.get(never)).toBeNull()
+    // …and the id mapper was never asked to assign for the phantom entity —
+    // pre-fix, plan-time resolution leaked an int here on every rejected batch.
+    const idMapper = brain['metadataIndex'].getIdMapper()
+    expect(idMapper.getInt(never)).toBeUndefined()
+
+    // The same shape then succeeds cleanly on retry.
+    await brain.transact([
+      { op: 'add', id: never, data: 'exists now', type: NounType.Thing },
+      { op: 'relate', from: existing, to: never, type: VerbType.RelatedTo }
+    ])
+    expect((await brain.related(existing)).map((r: any) => r.to)).toContain(never)
+  })
+})

From 708978210a760a679703f7eec73be8b041d250cf Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 10 Jul 2026 18:06:37 -0700
Subject: [PATCH 02/82] docs: RELEASES.md entry for 8.2.1 (transact forward-ref
 parity fix)

---
 RELEASES.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index 5f957292..8a2d52ac 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,27 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.1 — 2026-07-11 (transact forward references work on the native accelerator)
+
+Parity fix. `transact()` has always promised atomic forward references — `add` an entity and
+`relate` to it in one batch — but the transaction planner resolved relationship endpoint ids at
+**plan** time, before the batch's adds had applied. Asking the id mapper about an entity that
+doesn't exist yet made the accelerator's (correctly strict) native mapper throw in
+`EntityIdMapper.getOrAssign`, so `transact([{op:'add', id:X}, {op:'relate', to:X}])` failed on
+native deployments while the permissive JS mapper masked the bug — and silently leaked an id
+assignment whenever a batch was later rejected by a commit precondition.
+
+Endpoint resolution is now **lazy** — evaluated when the graph operation executes inside the
+commit, after the batch's adds have applied (the same lazy pattern the operation's generation
+stamp already used). Fixed across all transact-planned graph operations: relate (including the
+bidirectional reverse edge), remove's relationship cascade, and unrelate. Single-operation writes
+are unchanged. Also fixed as a byproduct: a rejected batch no longer pollutes the id mapper.
+
+Regression suite covers the exact reported shape, both-endpoints-in-batch, bidirectional,
+add+relate+remove in one batch, and the mapper-cleanliness proof on rejected batches
+(`tests/integration/transact-forward-ref-graph.test.ts`). No API changes; no accelerator version
+pairing required.
+
 ## v8.2.0 — 2026-07-10 (temporal VFS — file content joins the immutability model)
 
 Time travel now covers Virtual Filesystem **content**. Previously the temporal model had a hole

From 62a449d38b19b5146b1d198ce48ea9eba9f1ec6b Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 10 Jul 2026 18:08:09 -0700
Subject: [PATCH 03/82] test: update graph-index operation constructors to the
 VerbEndpointInts signature

---
 .../transaction/graphIndexOperations-generation.test.ts  | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/unit/transaction/graphIndexOperations-generation.test.ts b/tests/unit/transaction/graphIndexOperations-generation.test.ts
index 46e7dde2..9dc5c90c 100644
--- a/tests/unit/transaction/graphIndexOperations-generation.test.ts
+++ b/tests/unit/transaction/graphIndexOperations-generation.test.ts
@@ -55,7 +55,7 @@ describe('Graph index operations — generation threading', () => {
   it('AddToGraphIndexOperation stamps addVerb (and its rollback removeVerb) at the resolved generation', async () => {
     const { provider, calls } = makeSpyProvider()
     const verb = makeVerb('verb-1')
-    const op = new AddToGraphIndexOperation(provider, verb, 10n, 20n, () => 7n)
+    const op = new AddToGraphIndexOperation(provider, verb, { sourceInt: 10n, targetInt: 20n }, () => 7n)
 
     const rollback = await op.execute()
     expect(calls).toEqual([{ method: 'addVerb', generation: 7n, verbId: 'verb-1' }])
@@ -67,7 +67,7 @@ describe('Graph index operations — generation threading', () => {
   it('RemoveFromGraphIndexOperation stamps removeVerb (and its rollback addVerb) at the resolved generation', async () => {
     const { provider, calls } = makeSpyProvider()
     const verb = makeVerb('verb-2')
-    const op = new RemoveFromGraphIndexOperation(provider, verb, 10n, 20n, () => 12n)
+    const op = new RemoveFromGraphIndexOperation(provider, verb, { sourceInt: 10n, targetInt: 20n }, () => 12n)
 
     const rollback = await op.execute()
     expect(calls).toEqual([{ method: 'removeVerb', generation: 12n, verbId: 'verb-2' }])
@@ -83,7 +83,7 @@ describe('Graph index operations — generation threading', () => {
     // at its execute-time value.
     const { provider, calls } = makeSpyProvider()
     let current = 1n
-    const op = new AddToGraphIndexOperation(provider, makeVerb('verb-3'), 10n, 20n, () => current)
+    const op = new AddToGraphIndexOperation(provider, makeVerb('verb-3'), { sourceInt: 10n, targetInt: 20n }, () => current)
 
     current = 42n // store assigns the batch generation after the op is built
     await op.execute()
@@ -97,8 +97,7 @@ describe('Graph index operations — generation threading', () => {
     const op = new AddToGraphIndexOperation(
       provider,
       makeVerb('verb-4'),
-      10n,
-      20n,
+      { sourceInt: 10n, targetInt: 20n },
       () => 5n,
       (verbInt) => {
         seen = verbInt

From 41dc307bb9bb59a793ca8110ea00456fddfe2897 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 10 Jul 2026 18:10:52 -0700
Subject: [PATCH 04/82] chore(release): 8.2.1

---
 CHANGELOG.md      | 7 +++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 910067e5..fae40d30 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.1](https://github.com/soulcraftlabs/brainy/compare/v8.2.0...v8.2.1) (2026-07-10)
+
+- test: update graph-index operation constructors to the VerbEndpointInts signature (62a449d)
+- docs: RELEASES.md entry for 8.2.1 (transact forward-ref parity fix) (7089782)
+- fix: transact forward references resolve graph endpoint ints at execute time (a175406)
+
+
 ### [8.2.0](https://github.com/soulcraftlabs/brainy/compare/v8.1.0...v8.2.0) (2026-07-10)
 
 - docs: RELEASES.md entry for 8.2.0 (temporal VFS) (98ceadc)
diff --git a/package-lock.json b/package-lock.json
index 476a13e3..32d7c6bb 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.0",
+  "version": "8.2.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.0",
+      "version": "8.2.1",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 3492a639..5c0bda3f 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.0",
+  "version": "8.2.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 508a8e363ea8ed3f14978b3bb516d10df57ab87a Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sat, 11 Jul 2026 13:44:15 -0700
Subject: [PATCH 05/82] fix: transaction timeout rolls back applied operations
 (no torn state)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Transaction.execute() checked its time budget at the top of the operation
loop and threw TransactionTimeoutError from OUTSIDE the per-operation
try/catch, so a mid-flight timeout bypassed rollback entirely — only
per-operation failures rolled back. A bulk transact that crossed its 30s
budget mid-flight left the operations already applied to canonical storage
in place while the generation was never stamped: torn, generation-less
state. The generation-store commit path's abort cleanup explicitly assumes a
throw from execute() already restored the applied operations byte-identically
(it only discards the uncommitted staging directory), so the missing
rollback broke that invariant.

Give execute() a single rollback point: the operation loop is the sole
rollback-guarded region, and any error escaping it — an operation failure OR
a mid-flight timeout — rolls back every applied operation in reverse order,
then surfaces the original error (a rollback failure still supersedes it via
TransactionRollbackError). The per-exit-path rollback that let the timeout
throw slip past is gone; atomicity now holds by construction for every error
type. An aborted transaction leaves generation() unchanged and storage
byte-identical to its pre-transaction state.

Regression (tests/unit/transaction/timeout-rollback.test.ts): a mid-flight
timeout leaves an in-memory canonical store byte-identical with the tx in the
rolled_back terminal state; the operation-failure and rollback-failure paths
through the same single rollback point; and a clean transaction still commits.
---
 src/transaction/Transaction.ts                |  57 +++---
 .../unit/transaction/timeout-rollback.test.ts | 177 ++++++++++++++++++
 2 files changed, 211 insertions(+), 23 deletions(-)
 create mode 100644 tests/unit/transaction/timeout-rollback.test.ts

diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts
index ff19b571..16332109 100644
--- a/src/transaction/Transaction.ts
+++ b/src/transaction/Transaction.ts
@@ -98,49 +98,60 @@ export class Transaction implements TransactionContext {
     }
 
     try {
-      // Execute each operation in order
+      // Execute each operation in order. This loop is the sole rollback-guarded
+      // region: ANY error that escapes it — an operation failure OR a
+      // mid-flight timeout — is caught below and rolls back every operation
+      // applied so far, in reverse order. That single guarantee is the
+      // transaction's atomicity contract, and the generation-store commit path
+      // depends on it: its abort cleanup assumes a throw from here already
+      // restored the applied operations byte-identically (it only discards the
+      // uncommitted staging directory). A rollback per exit path — the previous
+      // design — let the timeout throw slip past rollback and strand the
+      // already-applied writes as torn, generation-less state in canonical
+      // storage.
       for (let i = 0; i < this.operations.length; i++) {
-        // Check timeout
+        // Budget check BEFORE starting the next operation. A trip here throws
+        // into the catch below and rolls back like any other failure — it must
+        // never bypass rollback.
         if (Date.now() - this.startTime > this.options.timeout) {
           throw new TransactionTimeoutError(this.options.timeout, i)
         }
 
         const operation = this.operations[i]
 
+        if (this.options.logging) {
+          prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`)
+        }
+
+        let rollbackAction: RollbackAction | undefined
         try {
-          if (this.options.logging) {
-            prodLog.info(`[Transaction] Executing operation ${i}: ${operation.name || 'unnamed'}`)
-          }
-
-          // Execute operation
-          const rollbackAction = await operation.execute()
-
-          // Record rollback action (if provided)
-          if (rollbackAction) {
-            this.rollbackActions.push(rollbackAction)
-          }
-
+          rollbackAction = await operation.execute()
         } catch (error) {
-          // Operation failed - rollback and re-throw
-          const executionError = new TransactionExecutionError(
+          // Normalize an operation failure to a TransactionExecutionError; the
+          // outer catch performs the (single) rollback and surfaces it.
+          throw new TransactionExecutionError(
             `Operation ${i} failed: ${(error as Error).message}`,
             i,
             operation.name,
             error as Error
           )
+        }
 
-          await this.rollback(executionError)
-          throw executionError
+        // Record rollback action (if the operation provided one)
+        if (rollbackAction) {
+          this.rollbackActions.push(rollbackAction)
         }
       }
-
-      // All operations succeeded - commit
-      this.commit()
-
     } catch (error) {
-      // Error already handled in rollback
+      // Single rollback point: undo everything applied so far, in reverse
+      // order, then surface the original error. A rollback failure supersedes
+      // it (rollback() throws TransactionRollbackError wrapping this error).
+      await this.rollback(error as Error)
       throw error
     }
+
+    // All operations succeeded — commit.
+    this.commit()
   }
 
   /**
diff --git a/tests/unit/transaction/timeout-rollback.test.ts b/tests/unit/transaction/timeout-rollback.test.ts
new file mode 100644
index 00000000..c09ff098
--- /dev/null
+++ b/tests/unit/transaction/timeout-rollback.test.ts
@@ -0,0 +1,177 @@
+/**
+ * @module tests/unit/transaction/timeout-rollback
+ * @description Regression for a consumer-reported P0 data-integrity bug:
+ * `Transaction.execute()` checked its time budget at the TOP of the operation
+ * loop and threw `TransactionTimeoutError` from OUTSIDE the per-operation
+ * try/catch — so the timeout bypassed rollback entirely (only per-operation
+ * failures rolled back). On a bulk transact that tripped the 30s budget
+ * mid-flight, the operations already applied to canonical storage were left in
+ * place while the generation was never stamped: torn, generation-less state.
+ * The generation-store commit path's abort cleanup explicitly ASSUMES a throw
+ * from `execute()` already restored the applied operations byte-identically
+ * (it only discards the uncommitted staging directory), so the missing
+ * rollback broke that invariant.
+ *
+ * Fix: `execute()` now has a SINGLE rollback point — any error escaping the
+ * operation loop (operation failure OR mid-flight timeout) rolls back every
+ * applied operation in reverse order, then surfaces the original error.
+ *
+ * These tests model canonical storage as an in-memory map and give each
+ * operation a real rollback action that restores the map byte-identically —
+ * exercising the exact fixed code path and asserting the reported requirement:
+ * a mid-flight timeout leaves storage byte-identical to its pre-transaction
+ * state.
+ */
+import { describe, it, expect } from 'vitest'
+import { Transaction } from '../../../src/transaction/Transaction.js'
+import type { Operation, RollbackAction } from '../../../src/transaction/types.js'
+import {
+  TransactionTimeoutError,
+  TransactionExecutionError,
+  TransactionRollbackError
+} from '../../../src/transaction/errors.js'
+
+/** A tiny stand-in for canonical storage. */
+type Store = Map<string, string>
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms))
+
+/**
+ * An operation that writes `value` at `key` in `store` and returns a rollback
+ * action restoring the key's PRIOR state byte-identically (delete if it was
+ * absent, else restore the previous value) — exactly how the real graph/metadata
+ * operations undo themselves. `delayMs` lets an op consume wall-clock so a small
+ * transaction budget trips on the NEXT loop iteration; `fail` makes it throw
+ * after doing its write-and-rollback capture (to exercise the shared path via an
+ * operation failure rather than a timeout).
+ */
+function writeOp(
+  store: Store,
+  key: string,
+  value: string,
+  opts: { delayMs?: number; fail?: boolean; name?: string } = {}
+): Operation & { executed: boolean } {
+  const op = {
+    name: opts.name ?? `write:${key}`,
+    executed: false,
+    async execute(): Promise<RollbackAction | undefined> {
+      op.executed = true
+      if (opts.delayMs) await sleep(opts.delayMs)
+      // Operations are individually atomic: a failing op throws having left NO
+      // net change (it never gets far enough to register an undo), so only the
+      // PRIOR operations need rolling back.
+      if (opts.fail) throw new Error(`operation ${key} failed`)
+      const had = store.has(key)
+      const prev = store.get(key)
+      store.set(key, value)
+      return async () => {
+        if (had) store.set(key, prev as string)
+        else store.delete(key)
+      }
+    }
+  }
+  return op
+}
+
+describe('Transaction — mid-flight timeout rolls back (P0 data integrity)', () => {
+  it('a mid-flight timeout leaves storage BYTE-IDENTICAL to its pre-transaction state', async () => {
+    const store: Store = new Map([['seed', 'unchanged']])
+    const snapshot = new Map(store)
+
+    // Budget 1ms; op0 sleeps 60ms so the budget is blown when the loop checks
+    // it before op1 — a deterministic mid-flight timeout at operation index 1.
+    const op0 = writeOp(store, 'A', 'applied-then-rolled-back', { delayMs: 60 })
+    const op1 = writeOp(store, 'B', 'never-applied')
+
+    const tx = new Transaction({ timeout: 1 })
+    tx.addOperation(op0)
+    tx.addOperation(op1)
+
+    await expect(tx.execute()).rejects.toBeInstanceOf(TransactionTimeoutError)
+
+    // op0 ran (and was rolled back); op1 never started.
+    expect(op0.executed).toBe(true)
+    expect(op1.executed).toBe(false)
+    // The whole point: canonical storage is byte-identical — no torn state.
+    expect(store).toEqual(snapshot)
+    expect(store.has('A')).toBe(false)
+    expect(store.has('B')).toBe(false)
+    // And the transaction ended in the rolled-back terminal state (so the
+    // manager counts it correctly), not stranded in 'executing'.
+    expect(tx.getState()).toBe('rolled_back')
+  })
+
+  it('an operation failure mid-flight rolls back byte-identically (the shared single-rollback path)', async () => {
+    const store: Store = new Map([['seed', 'unchanged']])
+    const snapshot = new Map(store)
+
+    const op0 = writeOp(store, 'A', 'applied-then-rolled-back')
+    const op1 = writeOp(store, 'B', 'partially-applied', { fail: true })
+    const op2 = writeOp(store, 'C', 'never-applied')
+
+    const tx = new Transaction()
+    tx.addOperation(op0)
+    tx.addOperation(op1)
+    tx.addOperation(op2)
+
+    await expect(tx.execute()).rejects.toBeInstanceOf(TransactionExecutionError)
+
+    expect(op0.executed).toBe(true)
+    expect(op1.executed).toBe(true)
+    expect(op2.executed).toBe(false)
+    // op1's partial write is undone by its own rollback; op0's write is undone;
+    // op2 never wrote. Byte-identical.
+    expect(store).toEqual(snapshot)
+    expect(tx.getState()).toBe('rolled_back')
+  })
+
+  it('a rollback failure during a timeout surfaces TransactionRollbackError wrapping the timeout (loud, not silent)', async () => {
+    const store: Store = new Map()
+
+    // op0 applies, but its rollback throws every time (maxRollbackRetries
+    // exhausted) — the manager must surface a TransactionRollbackError whose
+    // originalError is the timeout, never swallow it.
+    const op0: Operation & { executed: boolean } = {
+      name: 'unrollbackable',
+      executed: false,
+      async execute() {
+        op0.executed = true
+        await sleep(60)
+        store.set('A', 'applied')
+        return async () => {
+          throw new Error('rollback is impossible for this op')
+        }
+      }
+    }
+    const op1 = writeOp(store, 'B', 'never-applied')
+
+    const tx = new Transaction({ timeout: 1, maxRollbackRetries: 1 })
+    tx.addOperation(op0)
+    tx.addOperation(op1)
+
+    let caught: unknown
+    await tx.execute().catch((e) => (caught = e))
+
+    expect(caught).toBeInstanceOf(TransactionRollbackError)
+    expect((caught as TransactionRollbackError).originalError).toBeInstanceOf(
+      TransactionTimeoutError
+    )
+    expect(op1.executed).toBe(false)
+  })
+
+  it('a clean (non-timeout) transaction still commits and applies all writes', async () => {
+    const store: Store = new Map()
+    const op0 = writeOp(store, 'A', 'a')
+    const op1 = writeOp(store, 'B', 'b')
+
+    const tx = new Transaction()
+    tx.addOperation(op0)
+    tx.addOperation(op1)
+
+    await tx.execute()
+
+    expect(tx.getState()).toBe('committed')
+    expect(store.get('A')).toBe('a')
+    expect(store.get('B')).toBe('b')
+  })
+})

From ed97006eb93639105e8960b1e1422b26e93fe379 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sat, 11 Jul 2026 13:44:15 -0700
Subject: [PATCH 06/82] docs: RELEASES.md entry for 8.2.2 (transaction timeout
 rollback)

---
 RELEASES.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index 8a2d52ac..78b4cf22 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,32 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.2 — 2026-07-11 (P0: a timed-out transaction now rolls back — no torn state)
+
+Data-integrity fix. A transaction that exceeded its time budget **mid-flight** (e.g. a bulk
+`transact()` on slower hardware crossing the 30s ceiling) threw a timeout error WITHOUT rolling
+back the operations it had already applied. The budget check sat outside the per-operation
+rollback path, so only per-operation *failures* rolled back — a timeout stranded the partial
+writes in canonical storage while the generation was never stamped, leaving torn, generation-less
+state. This was caught by a downstream migration that crossed the ceiling on a large batch.
+
+`Transaction.execute()` now has a **single rollback point**: any error that escapes the operation
+loop — an operation failure OR a mid-flight timeout — rolls back every applied operation in
+reverse order, then surfaces the original error (a rollback failure supersedes it, loudly, as
+before). This restores the invariant the generation-store commit path already depended on: a throw
+from execute means the applied operations were undone byte-identically, and the aborted
+transaction leaves `generation()` unchanged and storage byte-identical to its pre-transaction
+state.
+
+No API change. Regression pins the reported requirement — after a mid-flight timeout,
+storage is byte-identical and the transaction ends in the `rolled_back` terminal state
+(`tests/unit/transaction/timeout-rollback.test.ts`), plus the operation-failure and
+rollback-failure paths through the same single rollback point.
+
+**Note on the 30s ceiling itself** (configurable/scaled timeout, batched embedding precompute,
+timeout telemetry) — that ergonomics work is tracked separately; this release fixes only the
+correctness bug (a timeout must never leave partial state), independent of where the ceiling sits.
+
 ## v8.2.1 — 2026-07-11 (transact forward references work on the native accelerator)
 
 Parity fix. `transact()` has always promised atomic forward references — `add` an entity and

From a9fc1f3f9bb91f900dbe3a68ab810d9be3f12c57 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sat, 11 Jul 2026 13:48:47 -0700
Subject: [PATCH 07/82] chore(release): 8.2.2

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fae40d30..7d2cef22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.2](https://github.com/soulcraftlabs/brainy/compare/v8.2.1...v8.2.2) (2026-07-11)
+
+- docs: RELEASES.md entry for 8.2.2 (transaction timeout rollback) (ed97006)
+- fix: transaction timeout rolls back applied operations (no torn state) (508a8e3)
+
+
 ### [8.2.1](https://github.com/soulcraftlabs/brainy/compare/v8.2.0...v8.2.1) (2026-07-10)
 
 - test: update graph-index operation constructors to the VerbEndpointInts signature (62a449d)
diff --git a/package-lock.json b/package-lock.json
index 32d7c6bb..73e4135b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.1",
+  "version": "8.2.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.1",
+      "version": "8.2.2",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 5c0bda3f..e1553285 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.1",
+  "version": "8.2.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 3b8fa5116139c87149fe290ca004e89a690c28a6 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 08:54:14 -0700
Subject: [PATCH 08/82] =?UTF-8?q?fix:=20transact=20durability=20barrier=20?=
 =?UTF-8?q?=E2=80=94=20committed=20transactions=20are=20durable=20on=20ret?=
 =?UTF-8?q?urn?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A committed transact() reported success while its canonical entity writes were
still only in the OS page cache (tmp+rename, not fsync'd), even though the
generation counter and manifest were fsync'd. A hard kill in that window could
leave the durable counter ahead of the persisted entity bytes — phantom
progress for any generation-based consumer resuming from the counter. Reported
from a downstream migration's crash-lifecycle forensics.

Add an optional transaction durability barrier to GenerationStorage
(beginWriteBarrier / flushWriteBarrier). FileSystemStorage implements it:
writeObjectToPath records each successful canonical write and
deleteObjectFromPath records each delete's parent dir, between begin and flush;
flush fsyncs every recorded write (contents + rename dir entries, via
syncRawObjects) and the parent dir of every delete. commitTransaction opens the
barrier before running the planned operations and flushes it after, BEFORE
persisting the counter and manifest — so the batch's entire canonical footprint
is durable before the generation stamp advances. The barrier is optional: the
generation store calls it through optional chaining, so in-memory and
durable-per-call (cloud object-PUT) adapters treat it as a no-op.

The single-op group-commit path is unchanged (deferred durability is the
Model-B design that avoids a 3-5x per-write fsync regression), but its
durability contract is now documented explicitly on commitSingleOp: transact =
durable on return; single-op = durable at the next flush()/close(), with the
counter buffered alongside the data so a crash loses both together (never a
torn counter-ahead-of-state store).

Regression (tests/integration/transact-durability-barrier.test.ts): the entity
writes fsync in an earlier syncRawObjects batch than the manifest for single-op
and multi-op (add+relate) transactions; a precommit-rejected batch opens no
barrier and advances nothing; MemoryStorage exposes no barrier (optional-chain
no-op).
---
 src/db/generationStore.ts                     |  23 +++
 src/db/types.ts                               |  18 +++
 src/storage/adapters/fileSystemStorage.ts     |  78 +++++++++
 .../transact-durability-barrier.test.ts       | 151 ++++++++++++++++++
 4 files changed, 270 insertions(+)
 create mode 100644 tests/integration/transact-durability-barrier.test.ts

diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts
index e440f230..28b6276c 100644
--- a/src/db/generationStore.ts
+++ b/src/db/generationStore.ts
@@ -686,12 +686,21 @@ export class GenerationStore {
         faultPoint('after-staging')
 
         // -- 4. Execute the planned batch -------------------------------------
+        // Durability barrier: record every canonical write/delete the operations
+        // make, then fsync them BELOW (before the counter advances). Without
+        // this, a hard kill after commitTransaction returns could leave the
+        // fsync'd generation counter ahead of still-page-cached entity bytes —
+        // phantom progress for any generation-based consumer.
+        this.storage.beginWriteBarrier?.()
         this.inTransact = true
         try {
           await args.execute()
         } finally {
           this.inTransact = false
         }
+        // The transaction's entire canonical footprint is now durable, so the
+        // counter/manifest advance below can never outrun the entity bytes.
+        await this.storage.flushWriteBarrier?.()
         faultPoint('after-execute')
 
         // -- 5. Counter + manifest rename (COMMIT POINT) ----------------------
@@ -787,6 +796,20 @@ export class GenerationStore {
    *   mid-flush is recovered by drop-without-restore
    *   (see {@link GenerationDelta.groupCommit}).
    *
+   * **Durability contract (single-op vs transact).** A single-op write's live
+   * canonical bytes are written via tmp+rename but NOT individually fsync'd —
+   * they are durable at the next {@link flushPendingSingleOps} or `close()`, not
+   * the instant the write resolves. On a hard kill before that flush, a
+   * single-op write can be lost even though the call returned; the group-commit
+   * counter is buffered alongside, so counter and data are lost together (no
+   * counter-ahead-of-state torn store). {@link commitTransaction} is the
+   * stronger contract: it runs a write barrier that fsyncs the whole batch's
+   * canonical footprint BEFORE advancing the generation counter, so a committed
+   * transact is durable on return. Callers that need per-write durability should
+   * use `transact()` (or `flush()` after a single-op); Model-B group-commit
+   * deliberately trades single-op fsync latency (a 3-5x write regression) for
+   * throughput.
+   *
    * The per-write generation is read by graph-index ops through the same
    * `generation()` watermark `transact()` uses — because this method, like
    * {@link commitTransaction}, holds the mutex across the counter bump AND
diff --git a/src/db/types.ts b/src/db/types.ts
index 4030533a..4f932e76 100644
--- a/src/db/types.ts
+++ b/src/db/types.ts
@@ -391,6 +391,24 @@ export interface GenerationStorage {
   /** Durability barrier: fsync the given object paths (no-op in memory). */
   syncRawObjects(paths: string[]): Promise<void>
 
+  /**
+   * OPTIONAL transaction durability barrier. `commitTransaction` calls
+   * {@link beginWriteBarrier} immediately before running the planned operations
+   * and {@link flushWriteBarrier} immediately after — BEFORE the generation
+   * counter and manifest are advanced. An adapter whose canonical writes are
+   * not synchronously durable (the filesystem adapter's tmp+rename lands in the
+   * page cache) MUST implement these so a transaction reported "committed" is
+   * durable before its generation stamp: otherwise a hard kill can leave the
+   * fsync'd counter ahead of the still-buffered entity bytes (phantom progress
+   * for any generation-based consumer). Adapters whose writes are already
+   * durable per-call (cloud object PUT) may leave these undefined — the store
+   * treats them as no-ops. `beginWriteBarrier` also resets any tracking left by
+   * an aborted prior transaction.
+   */
+  beginWriteBarrier?(): void
+  /** @see beginWriteBarrier — fsync every canonical write since begin. */
+  flushWriteBarrier?(): Promise<void>
+
   /** Read an entity's raw stored metadata+vector objects. */
   readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }>
   /** Restore an entity's raw stored objects (`null` part ⇒ delete that file). */
diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index 87e73e4a..b0bc4ab7 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -119,6 +119,18 @@ export class FileSystemStorage extends BaseStorage {
   private compressionEnabled: boolean = true  // Enable gzip compression by default for 60-80% disk savings
   private compressionLevel: number = 6  // zlib compression level (1-9, default: 6 = balanced)
 
+  // Transaction durability barrier (see GenerationStorage.beginWriteBarrier).
+  // Non-null ONLY between beginWriteBarrier() and flushWriteBarrier(). Because
+  // canonical writes are tmp+rename (durable only in the page cache until an
+  // fsync), the generation store fsyncs everything a transaction wrote before
+  // it advances the generation counter. `writeBarrierPaths` collects the
+  // root-relative object paths written; `writeBarrierDeleteDirs` the parent
+  // dirs of deleted objects (an unlink is durable only once its directory is
+  // fsync'd). Both are null outside a transaction, so the tracking `add`s below
+  // are free on the single-op and non-transactional write paths.
+  private writeBarrierPaths: Set<string> | null = null
+  private writeBarrierDeleteDirs: Set<string> | null = null
+
   /**
    * Initialize the storage adapter
    * @param rootDirectory The root directory for storage
@@ -373,6 +385,12 @@ export class FileSystemStorage extends BaseStorage {
         throw error
       }
     }
+
+    // Transaction durability barrier: record the write so the generation store
+    // can fsync it before advancing the counter. Reached only on a successful
+    // rename; the logical (non-`.gz`) path is stored — flushWriteBarrier's
+    // syncRawObjects resolves the compressed variant.
+    this.writeBarrierPaths?.add(pathStr)
   }
 
   /**
@@ -467,6 +485,11 @@ export class FileSystemStorage extends BaseStorage {
     if (deletedCount === 0) {
       // File doesn't exist - this is fine
     }
+
+    // Transaction durability barrier: an unlink is durable only once its parent
+    // directory is fsync'd. Record the dir (root-relative; '.' for a top-level
+    // object) so flushWriteBarrier can sync it before the counter advances.
+    this.writeBarrierDeleteDirs?.add(path.dirname(pathStr))
   }
 
   /**
@@ -630,6 +653,61 @@ export class FileSystemStorage extends BaseStorage {
     }
   }
 
+  /**
+   * Begin a transaction durability barrier: start recording every canonical
+   * object write and delete so {@link flushWriteBarrier} can fsync them before
+   * the generation counter advances. Resets unconditionally, discarding any
+   * tracking left by a transaction that aborted without flushing.
+   *
+   * @see GenerationStorage.beginWriteBarrier
+   */
+  public beginWriteBarrier(): void {
+    this.writeBarrierPaths = new Set()
+    this.writeBarrierDeleteDirs = new Set()
+  }
+
+  /**
+   * Flush the transaction durability barrier: fsync every canonical write since
+   * {@link beginWriteBarrier} (file contents AND the rename directory entries,
+   * via {@link syncRawObjects}), then fsync the parent directory of every
+   * canonical delete so the unlinks are durable too. Clears the tracking. After
+   * this resolves, the transaction's entire canonical footprint is on disk, so
+   * the generation counter/manifest can be advanced without risking a
+   * counter-ahead-of-state torn store on a hard kill.
+   *
+   * @see GenerationStorage.flushWriteBarrier
+   */
+  public async flushWriteBarrier(): Promise<void> {
+    const paths = this.writeBarrierPaths
+    const deleteDirs = this.writeBarrierDeleteDirs
+    this.writeBarrierPaths = null
+    this.writeBarrierDeleteDirs = null
+
+    if (paths && paths.size > 0) {
+      // syncRawObjects fsyncs each file and its parent directory.
+      await this.syncRawObjects([...paths])
+    }
+
+    if (deleteDirs && deleteDirs.size > 0) {
+      for (const relDir of deleteDirs) {
+        const dirPath = path.join(this.rootDir, relDir)
+        let handle: any
+        try {
+          handle = await fs.promises.open(dirPath, 'r')
+        } catch {
+          continue // directory vanished or platform disallows opening dirs
+        }
+        try {
+          await handle.sync()
+        } catch {
+          // Some platforms/filesystems reject directory fsync — best effort.
+        } finally {
+          await handle.close()
+        }
+      }
+    }
+  }
+
   /**
    * Append one line to `_system/tx-log.jsonl`. Plain `appendFile` — the
    * tx-log is the one append-in-place file in the store (and is byte-copied,
diff --git a/tests/integration/transact-durability-barrier.test.ts b/tests/integration/transact-durability-barrier.test.ts
new file mode 100644
index 00000000..9311ce67
--- /dev/null
+++ b/tests/integration/transact-durability-barrier.test.ts
@@ -0,0 +1,151 @@
+/**
+ * @module tests/integration/transact-durability-barrier
+ * @description Regression for a consumer-reported durability gap: a committed
+ * `transact()` reported success while its canonical entity writes were still
+ * only in the page cache (tmp+rename, not yet fsync'd), whereas the generation
+ * counter/manifest WERE fsync'd — so a hard kill could leave the counter ahead
+ * of the persisted entity bytes ("phantom progress" for any generation-based
+ * consumer resuming from the counter).
+ *
+ * Fix: `commitTransaction` opens a write barrier before running the planned
+ * operations and flushes it (fsync of every canonical write + the parent dir of
+ * every canonical delete) BEFORE advancing the counter and writing the manifest.
+ * These tests assert the ordering property directly by recording the storage's
+ * `syncRawObjects` calls: the entity writes are fsync'd in an earlier call than
+ * the manifest, and a precommit-rejected batch opens no barrier and advances
+ * nothing.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType, VerbType } from '../../src/types/graphTypes.js'
+import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
+
+const MANIFEST_REL = '_system/manifest.json'
+
+describe('transact durability barrier — entity writes fsync before the counter advances', () => {
+  let dir: string
+  let brain: any
+  let syncCalls: string[][]
+  let beginCount: number
+  let flushCount: number
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-durability-'))
+    brain = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      dimensions: 384,
+      silent: true
+    })
+    await brain.init()
+
+    // Instrument the real filesystem storage: record every fsync batch in order,
+    // and count barrier open/flush, delegating to the originals.
+    syncCalls = []
+    beginCount = 0
+    flushCount = 0
+    const storage = brain['storage']
+    const origSync = storage.syncRawObjects.bind(storage)
+    storage.syncRawObjects = async (paths: string[]) => {
+      syncCalls.push([...paths])
+      return origSync(paths)
+    }
+    const origBegin = storage.beginWriteBarrier.bind(storage)
+    storage.beginWriteBarrier = () => {
+      beginCount++
+      return origBegin()
+    }
+    const origFlush = storage.flushWriteBarrier.bind(storage)
+    storage.flushWriteBarrier = async () => {
+      flushCount++
+      return origFlush()
+    }
+  })
+
+  afterEach(async () => {
+    await brain.close()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  /** Index of the first sync call whose paths satisfy `pred` (or -1). */
+  const findSync = (pred: (p: string) => boolean): number =>
+    syncCalls.findIndex((paths) => paths.some(pred))
+
+  it('a committed transact fsyncs the entity write BEFORE the manifest', async () => {
+    const id = '00000000-0000-4000-8000-00000000ab01'
+    await brain.transact([{ op: 'add', id, data: 'durable entity', type: NounType.Thing }])
+
+    // The barrier opened and flushed exactly once.
+    expect(beginCount).toBe(1)
+    expect(flushCount).toBe(1)
+
+    // The entity's canonical metadata write was fsync'd…
+    const entitySyncIdx = findSync((p) => p.includes(`/${id}/`) && p.includes('entities/nouns'))
+    expect(entitySyncIdx).toBeGreaterThanOrEqual(0)
+    // …and the manifest (the commit point) was fsync'd in a LATER call.
+    const manifestSyncIdx = findSync((p) => p === MANIFEST_REL)
+    expect(manifestSyncIdx).toBeGreaterThanOrEqual(0)
+    expect(entitySyncIdx).toBeLessThan(manifestSyncIdx)
+  })
+
+  it('a multi-op transact (add + relate) fsyncs both endpoints and the edge before the manifest', async () => {
+    const a = '00000000-0000-4000-8000-00000000ab02'
+    const b = '00000000-0000-4000-8000-00000000ab03'
+    await brain.transact([
+      { op: 'add', id: a, data: 'A', type: NounType.Thing },
+      { op: 'add', id: b, data: 'B', type: NounType.Thing },
+      { op: 'relate', from: a, to: b, type: VerbType.Contains }
+    ])
+
+    expect(flushCount).toBe(1)
+    const aIdx = findSync((p) => p.includes(`/${a}/`))
+    const bIdx = findSync((p) => p.includes(`/${b}/`))
+    const manifestIdx = findSync((p) => p === MANIFEST_REL)
+    expect(aIdx).toBeGreaterThanOrEqual(0)
+    expect(bIdx).toBeGreaterThanOrEqual(0)
+    expect(manifestIdx).toBeGreaterThanOrEqual(0)
+    expect(Math.max(aIdx, bIdx)).toBeLessThan(manifestIdx)
+
+    // Sanity: the data is actually there and traverses.
+    expect((await brain.related(a)).map((r: any) => r.to)).toContain(b)
+  })
+
+  it('a precommit-rejected batch opens no barrier and advances nothing', async () => {
+    const anchor = await brain.add({ id: '00000000-0000-4000-8000-00000000ab04', data: 'anchor', type: NounType.Thing })
+    const genBefore = brain.generationStore.generation()
+    syncCalls.length = 0
+    beginCount = 0
+    flushCount = 0
+
+    const never = '00000000-0000-4000-8000-00000000ab05'
+    await expect(
+      brain.transact([
+        { op: 'add', id: never, data: 'never', type: NounType.Thing },
+        { op: 'update', id: anchor, metadata: { poke: 1 }, ifRev: 999 }
+      ])
+    ).rejects.toMatchObject({ name: 'RevisionConflictError' })
+
+    // Precommit throws before the barrier opens (it lives just before execute):
+    // the barrier never opens, nothing is flushed, and the generation counter is
+    // unchanged. (A manifest sync CAN appear here from transact() flushing the
+    // anchor's previously-buffered single-op — that is the anchor's deferred
+    // durability, not this rejected batch committing; beginCount is the clean
+    // signal that this batch's commit path never reached execute.)
+    expect(beginCount).toBe(0)
+    expect(flushCount).toBe(0)
+    expect(brain.generationStore.generation()).toBe(genBefore)
+    expect(await brain.get(never)).toBeNull()
+  })
+
+  it('MemoryStorage does not implement the barrier (optional-chaining no-op)', () => {
+    const mem = new MemoryStorage() as any
+    // In-memory has no durability seam; the generation store treats the absent
+    // barrier methods as no-ops via optional chaining.
+    expect(mem.beginWriteBarrier).toBeUndefined()
+    expect(mem.flushWriteBarrier).toBeUndefined()
+  })
+})

From be5ce0bcc3eaa6881f3d2cc670560b5403d95188 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 08:54:14 -0700
Subject: [PATCH 09/82] docs: RELEASES.md entry for 8.2.3 (transact durability
 barrier)

---
 RELEASES.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index 78b4cf22..b1f95f49 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,34 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.3 — 2026-07-12 (a committed transaction is durable on return)
+
+Durability fix. A `transact()` reported "committed" while its canonical entity writes were still
+only in the OS page cache (written via tmp+rename, not yet `fsync`'d), even though the generation
+counter and manifest WERE fsync'd. A hard kill (power loss, SIGKILL) in that window could leave the
+durable generation counter **ahead of** the persisted entity bytes — so a consumer resuming from the
+counter would see "phantom progress": a generation that claims writes the disk never kept. Reported
+from a downstream migration's crash-lifecycle forensics.
+
+`commitTransaction` now runs a **durability barrier**: it records every canonical write and delete
+the batch's operations make, then `fsync`s that entire footprint (file contents, the rename
+directory entries, and the parent directories of any deletes) **before** advancing the generation
+counter and manifest. A committed transaction is therefore durable the moment `transact()` returns —
+the counter can never outrun the entity bytes.
+
+**Durability contract, now explicit.** `transact()` is durable-on-return (above). A **single-op**
+write (`add`/`update`/`remove`/`relate`/…) is Model-B group-commit: its live bytes are written but
+become durable at the next `flush()` or `close()`, not the instant the call resolves — the counter
+is buffered alongside the data, so a crash loses both together (never a torn counter-ahead-of-state
+store). Need per-write durability? Use `transact()` (even for one op), or `flush()` after the write.
+This deliberately trades single-op fsync latency (a 3-5x write regression) for throughput.
+
+No API change; no accelerator involvement (the barrier is in the filesystem storage adapter). In-memory
+and durable-per-call (cloud object-PUT) adapters treat the barrier as a no-op. Regression:
+`tests/integration/transact-durability-barrier.test.ts` proves entity writes fsync in an earlier
+batch than the manifest, for single-op and multi-op (add+relate) transactions, and that a
+precommit-rejected batch opens no barrier and advances nothing.
+
 ## v8.2.2 — 2026-07-11 (P0: a timed-out transaction now rolls back — no torn state)
 
 Data-integrity fix. A transaction that exceeded its time budget **mid-flight** (e.g. a bulk

From b3e8d47d46fe96ee118f407f1a92dd0bc6397028 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 08:59:28 -0700
Subject: [PATCH 10/82] chore(release): 8.2.3

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7d2cef22..75c50a90 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.3](https://github.com/soulcraftlabs/brainy/compare/v8.2.2...v8.2.3) (2026-07-12)
+
+- docs: RELEASES.md entry for 8.2.3 (transact durability barrier) (be5ce0b)
+- fix: transact durability barrier — committed transactions are durable on return (3b8fa51)
+
+
 ### [8.2.2](https://github.com/soulcraftlabs/brainy/compare/v8.2.1...v8.2.2) (2026-07-11)
 
 - docs: RELEASES.md entry for 8.2.2 (transaction timeout rollback) (ed97006)
diff --git a/package-lock.json b/package-lock.json
index 73e4135b..82f4a631 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.2",
+  "version": "8.2.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.2",
+      "version": "8.2.3",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index e1553285..4e7a6615 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.2",
+  "version": "8.2.3",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From a2f4f6a55039a5c4366f40bb2ab1228d49101cb1 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 09:10:35 -0700
Subject: [PATCH 11/82] fix: non-destructive, crash-resumable restore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

restore() removed the entire live brain directory and then fs.cp'd the
snapshot in, so any copy failure left the store destroyed with only a partial
copy. The sharpest edge: fs.cp materialized the holes of sparse mmap blob
files, so a snapshot that fits on disk could balloon and ENOSPC mid-copy — the
recovery tool destroying the brain it was asked to recover.

Rewrite restoreFromDirectory as stage → verify → atomic swap:
- Copy the snapshot into a _restore_staging area BEFORE touching live data,
  sparse-aware (copyTreeSparse/copyFileSparse skip all-zero 4 MiB chunks, so a
  mostly-hole store restores at its true allocated size, not its apparent size).
- On any copy failure (ENOSPC included) remove only the half-written staging
  area and throw — the live store is left exactly as it was.
- Only after the copy succeeds, fsync a completion marker naming the staged
  entries, then swapStagedRestoreIn(): an idempotent per-entry
  rm-old + rename-staged-in (same-filesystem renames that cannot ENOSPC),
  removing stale live entries the snapshot lacks.
- completeInterruptedRestore(), wired into init() right after the root dir is
  ensured, finishes a crash mid-swap FORWARD (resume a committed swap) or
  discards an uncommitted staging area (live still authoritative).

_restore_staging is excluded from snapshots. persist() (hard-link snapshot)
was already safe and is unchanged; no public API change.

Regression (tests/integration/restore-nondestructive.test.ts): a forced copy
failure leaves live data fully intact and cleans staging; a normal restore
round-trips to the snapshot; an interrupted-but-committed restore completes on
reopen; an uncommitted staging area is discarded on open; the sparse copy is
byte-identical with allocated blocks far below apparent size.
---
 src/storage/adapters/fileSystemStorage.ts     | 233 +++++++++++++++++-
 .../restore-nondestructive.test.ts            | 168 +++++++++++++
 2 files changed, 388 insertions(+), 13 deletions(-)
 create mode 100644 tests/integration/restore-nondestructive.test.ts

diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index b0bc4ab7..e728cc41 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -223,6 +223,11 @@ export class FileSystemStorage extends BaseStorage {
       // Create the root directory if it doesn't exist
       await this.ensureDirectoryExists(this.rootDir)
 
+      // Finish any restore interrupted by a crash (resume the staged swap, or
+      // discard an uncommitted staging area) BEFORE counts/derived state load,
+      // so the rest of startup sees the completed store.
+      await this.completeInterruptedRestore()
+
       // Create the nouns directory if it doesn't exist
       await this.ensureDirectoryExists(this.nounsDir)
 
@@ -587,9 +592,30 @@ export class FileSystemStorage extends BaseStorage {
 
   /**
    * Top-level directories excluded from snapshots: process-local lock state
-   * (writer lock, flush-request RPC files) must never travel with the data.
+   * (writer lock, flush-request RPC files) must never travel with the data, and
+   * the restore staging area ({@link RESTORE_STAGING_DIR}) is transient scratch
+   * that must never be captured or restored.
    */
-  private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set<string>(['locks'])
+  private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set<string>(['locks', '_restore_staging'])
+
+  /**
+   * Transient top-level directory holding a restore-in-progress: the snapshot is
+   * fully copied here (sparse-aware) BEFORE any live data is touched, then an
+   * atomic per-entry swap moves it into place. Its presence + the completion
+   * marker let {@link completeInterruptedRestore} resume a crashed restore.
+   */
+  private static readonly RESTORE_STAGING_DIR = '_restore_staging'
+  /**
+   * Written+fsync'd inside the staging dir ONLY after the whole snapshot has
+   * copied successfully. Its presence authorizes the swap (and its resume): a
+   * staging dir WITHOUT this marker is an interrupted copy — discardable debris,
+   * live data still authoritative.
+   */
+  private static readonly RESTORE_MARKER = '.restore-manifest.json'
+  /** Chunk size for sparse-aware copying (holes are preserved at this grain). */
+  private static readonly SPARSE_CHUNK_BYTES = 4 * 1024 * 1024
+  /** A zero buffer the size of one sparse chunk, for all-zero (hole) detection. */
+  private static readonly SPARSE_ZERO_CHUNK = Buffer.alloc(4 * 1024 * 1024)
 
   /**
    * Remove every object under a storage-root-relative prefix — one recursive
@@ -899,6 +925,16 @@ export class FileSystemStorage extends BaseStorage {
    *
    * @param sourcePath - Absolute path of a directory produced by
    *   {@link FileSystemStorage.snapshotToDirectory}.
+   *
+   * Non-destructive: the snapshot is copied into a staging area (sparse-aware,
+   * so a store of mostly-hole mmap blobs cannot balloon and ENOSPC) BEFORE any
+   * live data is touched. Only once the full copy has succeeded and a completion
+   * marker is fsync'd does an atomic per-entry swap move it into place. A copy
+   * failure (including ENOSPC) leaves the live store exactly as it was; a crash
+   * mid-swap is resumed forward on the next {@link init} by
+   * {@link completeInterruptedRestore}. The previous implementation removed the
+   * live store first and then `fs.cp`'d — a copy failure destroyed the brain it
+   * was meant to recover.
    */
   public async restoreFromDirectory(sourcePath: string): Promise<void> {
     await this.ensureInitialized()
@@ -908,23 +944,194 @@ export class FileSystemStorage extends BaseStorage {
       throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`)
     }
 
-    // Clear current contents, preserving live lock state.
-    const currentEntries = await fs.promises.readdir(this.rootDir)
-    for (const entry of currentEntries) {
+    const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
+
+    // Discard any staging left by a prior aborted restore, then stage fresh.
+    await fs.promises.rm(staging, { recursive: true, force: true })
+    await fs.promises.mkdir(staging, { recursive: true })
+
+    // Copy the snapshot into staging (sparse-aware). ANY failure here — most
+    // importantly ENOSPC — leaves the live store untouched: we remove only the
+    // half-written staging area and re-throw.
+    const staged: string[] = []
+    try {
+      const sourceEntries = await fs.promises.readdir(sourcePath)
+      for (const entry of sourceEntries) {
+        if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
+        await this.copyTreeSparse(
+          path.join(sourcePath, entry),
+          path.join(staging, entry)
+        )
+        staged.push(entry)
+      }
+      // Commit point: fsync a marker naming the fully-staged entries. Only after
+      // this does the swap (and its resume) become authorized.
+      const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
+      await fs.promises.writeFile(markerPath, JSON.stringify({ entries: staged }))
+      const mfh = await fs.promises.open(markerPath, 'r')
+      try {
+        await mfh.sync()
+      } finally {
+        await mfh.close()
+      }
+      const dfh = await fs.promises.open(staging, 'r').catch(() => null)
+      if (dfh) {
+        try {
+          await dfh.sync()
+        } catch {
+          // platform may reject directory fsync — best effort
+        } finally {
+          await dfh.close()
+        }
+      }
+    } catch (error: any) {
+      await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
+      throw new Error(
+        `restoreFromDirectory: staging copy failed, live store left untouched: ${error.message}`
+      )
+    }
+
+    // Atomic per-entry swap (metadata-only renames within rootDir — cannot ENOSPC).
+    await this.swapStagedRestoreIn()
+    await this.reloadDerivedState()
+  }
+
+  /**
+   * Move a fully-staged, marker-committed restore into place, then clear the
+   * staging area. Idempotent and resumable: driven by the marker's entry list
+   * and by which staged entries remain, so a crash at any point is completed by
+   * simply calling it again (from {@link completeInterruptedRestore} on the next
+   * open). Every step is a same-filesystem rename or a remove — no operation can
+   * fail for disk space, so once the marker exists the store is guaranteed to
+   * reach the restored state.
+   */
+  private async swapStagedRestoreIn(): Promise<void> {
+    const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
+    const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
+
+    let staged: string[]
+    try {
+      const marker = JSON.parse(await fs.promises.readFile(markerPath, 'utf-8'))
+      staged = Array.isArray(marker?.entries) ? marker.entries : []
+    } catch {
+      return // no committed marker — nothing to swap
+    }
+    const stagedSet = new Set(staged)
+
+    // 1. Remove stale live entries the snapshot does not contain (excluding
+    //    process-local dirs and the staging area itself). An already-placed
+    //    staged entry is in stagedSet, so it is kept.
+    for (const entry of await fs.promises.readdir(this.rootDir)) {
       if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
+      if (entry === FileSystemStorage.RESTORE_STAGING_DIR) continue
+      if (stagedSet.has(entry)) continue
       await fs.promises.rm(path.join(this.rootDir, entry), { recursive: true, force: true })
     }
 
-    // Byte-copy the snapshot in (defensively skipping lock dirs in old snapshots).
-    const sourceEntries = await fs.promises.readdir(sourcePath)
-    for (const entry of sourceEntries) {
-      if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
-      await fs.promises.cp(path.join(sourcePath, entry), path.join(this.rootDir, entry), {
-        recursive: true
-      })
+    // 2. Place each staged entry (idempotent: a prior attempt that already moved
+    //    it leaves staging/entry absent, so we skip). `rm` the old first — a
+    //    rename onto an existing non-empty directory is not allowed; the staged
+    //    copy is the durable source until it is placed, so a crash between the
+    //    rm and the rename is recovered forward on the next call.
+    for (const entry of staged) {
+      const from = path.join(staging, entry)
+      const to = path.join(this.rootDir, entry)
+      const exists = await fs.promises.lstat(from).then(() => true, () => false)
+      if (!exists) continue
+      await fs.promises.rm(to, { recursive: true, force: true })
+      await fs.promises.rename(from, to)
     }
 
-    await this.reloadDerivedState()
+    // 3. Clear the staging area (marker last-standing entry).
+    await fs.promises.rm(staging, { recursive: true, force: true })
+  }
+
+  /**
+   * On open, finish any restore interrupted by a crash. A staging dir WITH the
+   * completion marker means the copy had succeeded — resume the swap forward
+   * (loudly). A staging dir WITHOUT the marker is an interrupted copy — pure
+   * debris; the live store is authoritative, so discard it. Called from
+   * {@link init} before counts/derived state load, so recovery is invisible to
+   * the rest of startup. Returns `true` if a swap was resumed.
+   */
+  private async completeInterruptedRestore(): Promise<boolean> {
+    const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
+    const stagingStat = await fs.promises.stat(staging).catch(() => null)
+    if (!stagingStat || !stagingStat.isDirectory()) return false
+
+    const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
+    const hasMarker = await fs.promises
+      .stat(markerPath)
+      .then(() => true, () => false)
+
+    if (!hasMarker) {
+      // Interrupted before the copy committed — live data untouched, discard.
+      await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
+      return false
+    }
+
+    console.log('♻️  Resuming an interrupted restore (completing the staged swap)')
+    await this.swapStagedRestoreIn()
+    return true
+  }
+
+  /**
+   * Recursively copy `src` to `dest`, preserving holes (sparse regions). Files
+   * are copied chunk-by-chunk skipping all-zero chunks, so a store of
+   * mostly-hole mmap blobs restores at its true allocated size instead of
+   * materializing every hole (the failure that made `fs.cp` ENOSPC a restore
+   * that would otherwise fit). Directories recurse; symlinks are recreated.
+   */
+  private async copyTreeSparse(src: string, dest: string): Promise<void> {
+    const stat = await fs.promises.lstat(src)
+    if (stat.isDirectory()) {
+      await fs.promises.mkdir(dest, { recursive: true })
+      for (const child of await fs.promises.readdir(src)) {
+        await this.copyTreeSparse(path.join(src, child), path.join(dest, child))
+      }
+    } else if (stat.isSymbolicLink()) {
+      await fs.promises.symlink(await fs.promises.readlink(src), dest)
+    } else if (stat.isFile()) {
+      await this.copyFileSparse(src, dest, stat.size, stat.mode)
+    }
+    // Other node types (sockets, devices) do not occur in a brain store.
+  }
+
+  /** Sparse-aware single-file copy — see {@link copyTreeSparse}. */
+  private async copyFileSparse(
+    src: string,
+    dest: string,
+    size: number,
+    mode: number
+  ): Promise<void> {
+    const CHUNK = FileSystemStorage.SPARSE_CHUNK_BYTES
+    const srcFh = await fs.promises.open(src, 'r')
+    try {
+      const destFh = await fs.promises.open(dest, 'w', mode)
+      try {
+        // Pre-size the destination so unwritten regions are holes.
+        await destFh.truncate(size)
+        const buf = Buffer.allocUnsafe(CHUNK)
+        let pos = 0
+        while (pos < size) {
+          const { bytesRead } = await srcFh.read(buf, 0, CHUNK, pos)
+          if (bytesRead === 0) break
+          const chunk = buf.subarray(0, bytesRead)
+          // Skip all-zero chunks: leaving them unwritten preserves the hole.
+          const isHole = chunk.equals(
+            FileSystemStorage.SPARSE_ZERO_CHUNK.subarray(0, bytesRead)
+          )
+          if (!isHole) {
+            await destFh.write(chunk, 0, bytesRead, pos)
+          }
+          pos += bytesRead
+        }
+      } finally {
+        await destFh.close()
+      }
+    } finally {
+      await srcFh.close()
+    }
   }
 
   // ===========================================================================
diff --git a/tests/integration/restore-nondestructive.test.ts b/tests/integration/restore-nondestructive.test.ts
new file mode 100644
index 00000000..b36d2620
--- /dev/null
+++ b/tests/integration/restore-nondestructive.test.ts
@@ -0,0 +1,168 @@
+/**
+ * @module tests/integration/restore-nondestructive
+ * @description Regression for a consumer-reported recovery hazard: `restore()`
+ * removed the entire live brain directory and THEN `fs.cp`'d the snapshot in, so
+ * a copy failure (most dangerously ENOSPC — `fs.cp` also materializes the holes
+ * of sparse mmap blobs, ballooning a snapshot that would otherwise fit) left the
+ * store destroyed with only a partial copy: the recovery tool could destroy the
+ * brain it was meant to recover.
+ *
+ * Fix: the snapshot is copied into a staging area (sparse-aware) BEFORE any live
+ * data is touched; only after the copy succeeds and a marker is fsync'd does an
+ * atomic per-entry swap move it into place. A copy failure leaves the live store
+ * exactly as it was; a crash mid-swap is resumed forward on the next open.
+ *
+ * These tests exercise the property directly: a forced copy failure leaves live
+ * data intact, a normal restore round-trips correctly, an interrupted-but-
+ * committed staging area is completed on reopen, and the sparse copy preserves
+ * holes byte-for-byte.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType } from '../../src/types/graphTypes.js'
+
+const STAGING = '_restore_staging'
+const MARKER = '.restore-manifest.json'
+
+describe('non-destructive restore (BRAINY-RESTORE-DESTRUCTIVE)', () => {
+  let root: string
+  let snap: string
+  let brain: any
+
+  const openBrain = async (dir: string) => {
+    const b = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      dimensions: 384,
+      silent: true
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    const base = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-restore-'))
+    root = path.join(base, 'store')
+    snap = path.join(base, 'snapshot')
+    brain = await openBrain(root)
+  })
+
+  afterEach(async () => {
+    try { await brain.close() } catch { /* already closed in a test */ }
+    fs.rmSync(path.dirname(root), { recursive: true, force: true })
+  })
+
+  it('a copy failure leaves the LIVE store completely intact (the core property)', async () => {
+    const keep = await brain.add({ id: '00000000-0000-4000-8000-0000000000c1', data: 'in snapshot', type: NounType.Thing })
+    await brain.flush()
+    await brain['storage'].snapshotToDirectory(snap)
+    // Mutate live AFTER the snapshot — this state must survive a failed restore.
+    const extra = await brain.add({ id: '00000000-0000-4000-8000-0000000000c2', data: 'live only', type: NounType.Thing })
+
+    // Force the staging copy to fail on its second entry (partial staging).
+    const storage = brain['storage']
+    const realCopy = storage.copyTreeSparse.bind(storage)
+    let calls = 0
+    storage.copyTreeSparse = async (src: string, dest: string) => {
+      if (++calls === 2) throw new Error('simulated ENOSPC')
+      return realCopy(src, dest)
+    }
+
+    await expect(brain.restore(snap, { confirm: true })).rejects.toThrow(/untouched/)
+
+    // Live data — BOTH the snapshot entity and the live-only mutation — survives.
+    expect(await brain.get(keep)).not.toBeNull()
+    expect(await brain.get(extra)).not.toBeNull()
+    // The half-written staging area was cleaned up.
+    expect(fs.existsSync(path.join(root, STAGING))).toBe(false)
+  })
+
+  it('a normal restore round-trips to the snapshot state', async () => {
+    const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c3', data: 'A', type: NounType.Thing })
+    await brain.flush()
+    await brain['storage'].snapshotToDirectory(snap)
+    // Diverge from the snapshot: add B, remove A.
+    const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c4', data: 'B', type: NounType.Thing })
+    await brain.remove(a)
+    expect(await brain.get(a)).toBeNull()
+    expect(await brain.get(b)).not.toBeNull()
+
+    await brain.restore(snap, { confirm: true })
+
+    // Back to exactly the snapshot: A present, B gone.
+    expect(await brain.get(a)).not.toBeNull()
+    expect(await brain.get(b)).toBeNull()
+    expect(fs.existsSync(path.join(root, STAGING))).toBe(false)
+  })
+
+  it('an interrupted-but-committed restore is completed on the next open (resume forward)', async () => {
+    const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c5', data: 'A', type: NounType.Thing })
+    await brain.flush()
+    await brain['storage'].snapshotToDirectory(snap)
+    const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c6', data: 'B', type: NounType.Thing })
+    await brain.flush()
+    await brain.close()
+
+    // Simulate the state right after the staging copy committed but before the
+    // swap ran: a _restore_staging holding the snapshot's entries + the marker.
+    const staging = path.join(root, STAGING)
+    fs.mkdirSync(staging, { recursive: true })
+    const entries: string[] = []
+    for (const entry of fs.readdirSync(snap)) {
+      if (entry === 'locks' || entry === STAGING) continue
+      fs.cpSync(path.join(snap, entry), path.join(staging, entry), { recursive: true })
+      entries.push(entry)
+    }
+    fs.writeFileSync(path.join(staging, MARKER), JSON.stringify({ entries }))
+
+    // Reopen → init resumes the swap → the store is the snapshot (A, no B).
+    brain = await openBrain(root)
+    expect(await brain.get(a)).not.toBeNull()
+    expect(await brain.get(b)).toBeNull()
+    expect(fs.existsSync(staging)).toBe(false)
+  })
+
+  it('an uncommitted staging area (no marker) is discarded on open, live untouched', async () => {
+    const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c7', data: 'A live', type: NounType.Thing })
+    await brain.flush()
+    await brain.close()
+
+    // A staging dir with NO marker = a copy that never committed → debris.
+    const staging = path.join(root, STAGING)
+    fs.mkdirSync(staging, { recursive: true })
+    fs.writeFileSync(path.join(staging, 'entities'), 'garbage partial copy')
+
+    brain = await openBrain(root)
+    expect(await brain.get(a)).not.toBeNull() // live authoritative
+    expect(fs.existsSync(staging)).toBe(false) // debris removed
+  })
+
+  it('the sparse copy preserves holes: content byte-identical, allocation far below apparent size', async () => {
+    const base = path.dirname(root)
+    const src = path.join(base, 'sparse-src.bin')
+    const dest = path.join(base, 'sparse-dest.bin')
+    const SIZE = 32 * 1024 * 1024 // 32 MiB apparent
+    const DATA = Buffer.from('hello sparse world')
+
+    // Create a sparse source: 32 MiB of holes with a little real data at the start.
+    const fh = fs.openSync(src, 'w')
+    fs.ftruncateSync(fh, SIZE)
+    fs.writeSync(fh, DATA, 0, DATA.length, 0)
+    fs.closeSync(fh)
+
+    const storage = brain['storage']
+    const stat = fs.statSync(src)
+    await storage.copyFileSparse(src, dest, stat.size, stat.mode)
+
+    // Same apparent size and byte-identical content…
+    const destStat = fs.statSync(dest)
+    expect(destStat.size).toBe(SIZE)
+    expect(fs.readFileSync(dest).equals(fs.readFileSync(src))).toBe(true)
+    // …but the destination is actually sparse (allocated bytes far below size).
+    expect(destStat.blocks * 512).toBeLessThan(SIZE / 2)
+  })
+})

From 457469593a5770cdd27c40a52c203b822da1b969 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 09:10:35 -0700
Subject: [PATCH 12/82] docs: RELEASES.md entry for 8.2.4 (non-destructive
 restore)

---
 RELEASES.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index b1f95f49..a77ae27b 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,32 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.4 — 2026-07-12 (restore can no longer destroy the store it's recovering)
+
+Recovery-safety fix. `restore()` removed the entire live brain directory and THEN copied the
+snapshot in — so any copy failure left the store destroyed with only a partial copy. The sharpest
+edge: the copy (`fs.cp`) materialized the holes of sparse mmap blob files, so a snapshot that fits
+on disk could balloon and `ENOSPC` mid-copy, and the recovery tool would have just destroyed the
+brain it was asked to recover. Reported from a downstream incident's recovery forensics.
+
+Restore is now **non-destructive and crash-resumable**:
+
+- The snapshot is copied into a staging area (`_restore_staging/`) **before any live data is
+  touched**. The copy is **sparse-aware** — all-zero regions are left as holes, so a store of
+  mostly-hole blobs restores at its true allocated size instead of its apparent size.
+- A copy failure (including `ENOSPC`) removes only the half-written staging area and throws; the
+  live store is left **exactly as it was**.
+- Only after the copy succeeds and a completion marker is `fsync`'d does an **atomic per-entry
+  swap** move the staged data into place — same-filesystem renames that cannot fail for disk space.
+- A crash mid-swap is finished **forward** on the next open: startup resumes a committed-but-
+  incomplete swap, or discards an uncommitted staging area (live data still authoritative).
+
+No API change — `restore(path, { confirm: true })` is unchanged. `persist()` was already safe
+(hard-link snapshot). Regression (`tests/integration/restore-nondestructive.test.ts`): a forced
+copy failure leaves live data fully intact, a normal restore round-trips, an interrupted-but-
+committed restore completes on reopen, an uncommitted staging area is discarded, and the sparse
+copy is byte-identical with allocation far below apparent size.
+
 ## v8.2.3 — 2026-07-12 (a committed transaction is durable on return)
 
 Durability fix. A `transact()` reported "committed" while its canonical entity writes were still

From 036e56c9f9f18af5ba052e259267b8d0850fd72f Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 09:21:02 -0700
Subject: [PATCH 13/82] chore(release): 8.2.4

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 75c50a90..0a7b221c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.4](https://github.com/soulcraftlabs/brainy/compare/v8.2.3...v8.2.4) (2026-07-12)
+
+- docs: RELEASES.md entry for 8.2.4 (non-destructive restore) (4574695)
+- fix: non-destructive, crash-resumable restore (a2f4f6a)
+
+
 ### [8.2.3](https://github.com/soulcraftlabs/brainy/compare/v8.2.2...v8.2.3) (2026-07-12)
 
 - docs: RELEASES.md entry for 8.2.3 (transact durability barrier) (be5ce0b)
diff --git a/package-lock.json b/package-lock.json
index 82f4a631..5865dcd0 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.3",
+  "version": "8.2.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.3",
+      "version": "8.2.4",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 4e7a6615..0496e7b4 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.3",
+  "version": "8.2.4",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 711d2f046a28e717ba44a8d22fa3344539d67a74 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 12:19:09 -0700
Subject: [PATCH 14/82] fix: honest response when a transaction rollback cannot
 complete
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When a transaction failed and its rollback ALSO failed to undo a canonical
write (retries exhausted), the old path logged, continued, threw
TransactionRollbackError, and set state='rolled_back' — while the record it
could not undo stayed durable on disk. The caller got an error implying the
write was undone; a read-back showed the record. A failed rollback had no
truthful response (the post-commit response lie).

Give a failed rollback a two-branch honest contract, decided by observation:
both commit paths already hold byte-identical before-images, so after a failed
rollback the store compares current canonical state to them and classifies each
touched record as reconciled, an additive orphan (present when it should be
gone), or a restorative loss (gone/wrong when it should have been restored).

- Adopt-forward: a single-op write whose only damage is a durably-present
  orphan — the record the caller asked for — is committed forward (its
  generation buffered) and returns success with a loud warning that the derived
  index may be incomplete for that id until the next rebuild/repairIndex(). No
  error, no double-write; the record is durable and get-able immediately.
- Fail loud + quarantine: a multi-op batch, or ANY restorative loss, throws the
  new StoreInconsistentError naming every unreconciled record and its
  disposition, and puts the brain into write-quarantine (reads keep working,
  writes refused via assertWritable) until repairIndex() reconciles the derived
  indexes against canonical and lifts it. The counter is not advanced, and
  Transaction.rollback now reports state 'inconsistent' instead of the
  'rolled_back' lie when any undo failed.

New: StoreInconsistentError + UnreconciledRecord exported from the root;
repairIndex() forces a rebuild and clears the quarantine. Regression
tests/integration/rollback-trapdoor.test.ts injects index-add-throws +
canonical-delete-undo-fails and pins adopt-forward (durable, get-able, not
quarantined), fail-loud (StoreInconsistentError + quarantine + reads work +
repairIndex lifts it), and the error's record naming.
---
 src/brainy.ts                               | 105 ++++++++++---
 src/db/errors.ts                            |  71 +++++++++
 src/db/generationStore.ts                   | 119 ++++++++++++++-
 src/index.ts                                |   4 +-
 src/transaction/Transaction.ts              |  13 +-
 src/transaction/types.ts                    |   4 +
 tests/integration/rollback-trapdoor.test.ts | 156 ++++++++++++++++++++
 7 files changed, 437 insertions(+), 35 deletions(-)
 create mode 100644 tests/integration/rollback-trapdoor.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index 422cb2e6..da44c252 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -174,7 +174,7 @@ import {
   type PendingChangeEvent
 } from './events/changeFeed.js'
 import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js'
-import { GenerationConflictError } from './db/errors.js'
+import { GenerationConflictError, StoreInconsistentError } from './db/errors.js'
 import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError } from './errors/brainyError.js'
 import { MemoryStorage } from './storage/adapters/memoryStorage.js'
 import type {
@@ -593,6 +593,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   // set this to ReaderMode so historical instances are protected the same way.
   private operationalMode: BaseOperationalMode
 
+  // Write-quarantine after a failed transaction rollback left the store
+  // inconsistent (see StoreInconsistentError). Set at the moment the failure is
+  // surfaced; every subsequent mutation is refused via assertWritable until
+  // repairIndex() reconciles canonical vs derived state and clears it. Reads are
+  // never affected. null = healthy.
+  private storeInconsistency: StoreInconsistentError | null = null
+
   // Ready Promise state (Unified readiness API)
   // Allows consumers to await brain.ready for initialization completion
   private _readyPromise: Promise<void> | null = null
@@ -758,6 +765,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         `Open in writer mode to modify data.`
       )
     }
+    // Write-quarantine: a prior transaction's rollback failed and left the store
+    // inconsistent. Refuse further writes (which would compound the damage) until
+    // repairIndex() reconciles and lifts the quarantine. Reads still work.
+    if (this.storeInconsistency) {
+      throw new Error(
+        `Cannot call ${method}() — the store is WRITE-QUARANTINED after a failed ` +
+        `transaction rollback left it inconsistent. Reads still work; run repairIndex() ` +
+        `to reconcile the derived indexes against canonical storage and lift the ` +
+        `quarantine. Original inconsistency: ${this.storeInconsistency.message}`
+      )
+    }
   }
 
   /**
@@ -1631,14 +1649,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       this.emitCommitted(pendingEvents, capturedBefore, undefined, timestamp)
       return { timestamp }
     }
-    const receipt = await this.generationStore.commitSingleOp({
-      touched,
-      precommit: captureAndCheck,
-      execute: () => this.transactionManager.executeTransaction(run)
-    })
+    let receipt
+    try {
+      receipt = await this.generationStore.commitSingleOp({
+        touched,
+        precommit: captureAndCheck,
+        execute: () => this.transactionManager.executeTransaction(run)
+      })
+    } catch (err) {
+      // A failed rollback that left the store inconsistent (a remove/update
+      // whose restore-undo failed) quarantines writes until repairIndex().
+      if (err instanceof StoreInconsistentError) this.storeInconsistency = err
+      throw err
+    }
     // POST-COMMIT ONLY: an aborted commit (CAS conflict, failed apply) throws
     // above and never reaches this line — the feed cannot announce a write
-    // that did not become durable.
+    // that did not become durable. (A degraded adopt-forward write DID commit —
+    // it carries a generation and emits normally.)
     this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp)
     return receipt
   }
@@ -7407,19 +7434,28 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       }
     }
 
-    const { generation, timestamp } = await this.generationStore.commitTransaction({
-      touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs },
-      meta: options?.meta,
-      ifAtGeneration: options?.ifAtGeneration,
-      precommit: casPrecommit,
-      execute: async () => {
-        await this.transactionManager.executeTransaction(async (tx) => {
-          for (const operation of plan.operations) {
-            tx.addOperation(operation)
-          }
-        })
-      }
-    })
+    let generation: number
+    let timestamp: number
+    try {
+      ;({ generation, timestamp } = await this.generationStore.commitTransaction({
+        touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs },
+        meta: options?.meta,
+        ifAtGeneration: options?.ifAtGeneration,
+        precommit: casPrecommit,
+        execute: async () => {
+          await this.transactionManager.executeTransaction(async (tx) => {
+            for (const operation of plan.operations) {
+              tx.addOperation(operation)
+            }
+          })
+        }
+      }))
+    } catch (err) {
+      // A batch whose rollback failed and left canonical records unreconciled
+      // quarantines writes until repairIndex() reconciles and lifts it.
+      if (err instanceof StoreInconsistentError) this.storeInconsistency = err
+      throw err
+    }
 
     // Aggregation-index maintenance: derived data, applied after the commit
     // point — exactly where the single-operation methods apply it.
@@ -14750,15 +14786,36 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   }
 
   /**
-   * Detect and repair corrupted metadata indexes
+   * Detect and repair corrupted metadata indexes.
    *
-   * Runs corruption detection and auto-rebuilds if corruption is found.
-   * This is the equivalent of the old init()-time corruption check,
-   * now available as an explicit operation.
+   * Runs corruption detection and auto-rebuilds if corruption is found. This is
+   * the equivalent of the old init()-time corruption check, now available as an
+   * explicit operation.
+   *
+   * It is ALSO the recovery path for a write-quarantine: when a transaction's
+   * rollback fails and leaves the store inconsistent ({@link StoreInconsistentError}),
+   * writes are refused until this method reconciles the derived indexes against
+   * canonical storage (a forced rebuild — orphaned/lost records reflected
+   * consistently) and lifts the quarantine. Canonical is the source of truth: a
+   * genuinely lost record cannot be resurrected here (restore from a snapshot for
+   * that), but the store is made internally consistent and writes re-enabled.
    */
   async repairIndex(): Promise<void> {
     await this.ensureInitialized()
     await this.metadataIndex.detectAndRepairCorruption()
+    // Lift a failed-rollback write-quarantine: force a full rebuild so the
+    // derived indexes are provably reconciled with canonical, then clear the
+    // flag so writes resume.
+    if (this.storeInconsistency) {
+      await this.rebuildIndexesIfNeeded(true)
+      const cleared = this.storeInconsistency
+      this.storeInconsistency = null
+      prodLog.warn(
+        `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` +
+        `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` +
+        `Writes are re-enabled.`
+      )
+    }
   }
 
   /**
diff --git a/src/db/errors.ts b/src/db/errors.ts
index e5f70add..7cf9c3d0 100644
--- a/src/db/errors.ts
+++ b/src/db/errors.ts
@@ -159,3 +159,74 @@ export class GenerationCompactedError extends Error {
     this.horizon = horizon
   }
 }
+
+/** One entity/relationship left in an unreconciled state by a failed rollback. */
+export interface UnreconciledRecord {
+  /** The entity or relationship id. */
+  id: string
+  /** `'noun'` (entity) or `'verb'` (relationship). */
+  kind: 'noun' | 'verb'
+  /**
+   * `'orphan'` — the record is durably PRESENT but the transaction aborted
+   * (an add whose delete-undo failed); `'loss'` — the record is durably GONE
+   * or wrong when it should have been restored (a remove/update whose
+   * restore-undo failed — actual data loss).
+   */
+  disposition: 'orphan' | 'loss'
+}
+
+/**
+ * @description Thrown when a transaction's rollback could not be fully applied
+ * and the resulting inconsistency cannot be safely adopted forward — i.e. a
+ * multi-operation batch, or ANY case where a record was lost (a remove/update
+ * whose restore-undo failed). The store is left in a known-inconsistent state:
+ * the {@link records} name every entity/relationship whose canonical state no
+ * longer matches what the aborted transaction should have produced.
+ *
+ * On throw, the brain enters **write-quarantine** — reads continue to work, but
+ * further writes are refused until {@link } `repairIndex()` reconciles the
+ * derived indexes against canonical storage and lifts the quarantine. This is
+ * the honest, loud failure: a visible inconsistency the operator must repair,
+ * never a silent partial write. The generation counter is NOT advanced.
+ *
+ * (A SINGLE-op add whose only damage is a durably-present orphan is NOT this
+ * error: it is adopted forward as a committed generation and returned as a
+ * degraded-but-successful write — the record the caller asked for exists.)
+ *
+ * @example
+ * try {
+ *   await brain.transact(ops)
+ * } catch (err) {
+ *   if (err instanceof StoreInconsistentError) {
+ *     console.error('store inconsistent:', err.records) // ids + dispositions
+ *     await brain.repairIndex() // reconcile + lift the write-quarantine
+ *   }
+ * }
+ */
+export class StoreInconsistentError extends Error {
+  /** Every record left in an unreconciled state by the failed rollback. */
+  public readonly records: UnreconciledRecord[]
+  /** The original error that triggered the (then-failed) rollback. */
+  public override readonly cause: Error
+
+  /**
+   * @param records - The unreconciled entities/relationships (ids + disposition).
+   * @param cause - The error that triggered the rollback.
+   */
+  constructor(records: UnreconciledRecord[], cause: Error) {
+    const orphans = records.filter((r) => r.disposition === 'orphan').length
+    const losses = records.filter((r) => r.disposition === 'loss').length
+    super(
+      `Store left inconsistent by a failed transaction rollback: ` +
+        `${records.length} record(s) could not be reconciled ` +
+        `(${orphans} orphaned, ${losses} lost). ` +
+        `The brain is now WRITE-QUARANTINED (reads still work) — run repairIndex() ` +
+        `to reconcile the derived indexes against canonical storage and lift the ` +
+        `quarantine. Ids: ${records.map((r) => `${r.id}:${r.disposition}`).join(', ')}. ` +
+        `Cause: ${cause.message}`
+    )
+    this.name = 'StoreInconsistentError'
+    this.records = records
+    this.cause = cause
+  }
+}
diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts
index 28b6276c..5492ed70 100644
--- a/src/db/generationStore.ts
+++ b/src/db/generationStore.ts
@@ -32,7 +32,9 @@
  */
 
 import { prodLog } from '../utils/logger.js'
-import { GenerationCompactedError, GenerationConflictError } from './errors.js'
+import { GenerationCompactedError, GenerationConflictError, StoreInconsistentError } from './errors.js'
+import type { UnreconciledRecord } from './errors.js'
+import { TransactionRollbackError } from '../transaction/errors.js'
 import type {
   ChangedIds,
   CompactHistoryOptions,
@@ -618,18 +620,21 @@ export class GenerationStore {
       // before staging; scoped outside the try so an abort can compensate.
       let txBlobHashes: string[] = []
 
+      // Hoisted so the catch can reconcile canonical state against them after a
+      // failed rollback (the trapdoor). Empty until populated below.
+      const nounBefore = new Map<string, GenerationRecord>()
+      const verbBefore = new Map<string, GenerationRecord>()
+
       try {
         // -- 3. Before-images + delta (the durable undo log) ------------------
         // Read every before-image FIRST, then run the caller's CAS
         // precondition against them, and only then stage to disk — so a
         // conflicting batch aborts with zero staging I/O. The maps hold the
         // byte-identical records the staged files are written from.
-        const nounBefore = new Map<string, GenerationRecord>()
         for (const id of nouns) {
           const prev = await this.storage.readNounRaw(id)
           nounBefore.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector })
         }
-        const verbBefore = new Map<string, GenerationRecord>()
         for (const id of verbs) {
           const prev = await this.storage.readVerbRaw(id)
           verbBefore.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector })
@@ -737,6 +742,16 @@ export class GenerationStore {
         if (crashSimulated) {
           throw err
         }
+        // The trapdoor for a batch: if rollback FAILED to fully apply, canonical
+        // storage may be inconsistent. A batch is never adopted forward (its
+        // other ops were rolled back — partial commit would break atomicity), so
+        // any unreconciled record is a fail-loud StoreInconsistentError. Compute
+        // it here (before the staging cleanup, which is always safe to run) and
+        // throw it in place of the raw error at the end.
+        let inconsistent: UnreconciledRecord[] = []
+        if (err instanceof TransactionRollbackError) {
+          inconsistent = await this.reconcileFailedRollback(nounBefore, verbBefore)
+        }
         // Failed before the manifest rename: nothing is committed. Remove the
         // staging directory; the TransactionManager already restored any
         // applied operation byte-identically.
@@ -763,11 +778,68 @@ export class GenerationStore {
         // Return the reservation when no concurrent bump consumed a later
         // number, so a failed transaction leaves generation() unchanged.
         if (this.counter === gen) this.counter = gen - 1
+        // A failed rollback that left canonical records unreconciled surfaces as
+        // a loud StoreInconsistentError (brainy quarantines writes until repair);
+        // otherwise the raw error (clean abort, or a derived-only undo failure
+        // the egress guard + rebuild handle).
+        if (inconsistent.length > 0) {
+          throw new StoreInconsistentError(
+            inconsistent,
+            (err as TransactionRollbackError).originalError
+          )
+        }
         throw err
       }
     })
   }
 
+  /**
+   * @description After a transaction's rollback FAILED to fully apply
+   * (`TransactionRollbackError`), determine which touched records are now
+   * unreconciled by comparing current canonical state to the before-images the
+   * commit captured. This observes reality rather than trusting the opaque undo
+   * closures — the authoritative signal for adopt-forward vs fail-loud.
+   *
+   * @param nounBefore - Byte-identical entity before-images captured pre-execute.
+   * @param verbBefore - Byte-identical relationship before-images.
+   * @returns Every record whose canonical state no longer matches its
+   *   before-image, tagged `'orphan'` (durably present when it should be gone —
+   *   an add whose delete-undo failed) or `'loss'` (durably gone/wrong when it
+   *   should have been restored — a remove/update whose restore-undo failed). An
+   *   empty array means canonical storage is cleanly rolled back (only a derived
+   *   index undo failed — the egress guard + a rebuild handle that).
+   */
+  private async reconcileFailedRollback(
+    nounBefore: Map<string, GenerationRecord>,
+    verbBefore: Map<string, GenerationRecord>
+  ): Promise<UnreconciledRecord[]> {
+    const out: UnreconciledRecord[] = []
+    const classify = (
+      before: GenerationRecord,
+      current: { metadata: unknown | null; vector: unknown | null }
+    ): 'orphan' | 'loss' | null => {
+      const beforeEmpty = before.metadata == null && before.vector == null
+      const currentEmpty = current.metadata == null && current.vector == null
+      if (beforeEmpty && currentEmpty) return null // add cleanly undone
+      if (beforeEmpty) return 'orphan' // add's delete-undo failed → still present
+      if (currentEmpty) return 'loss' // remove's restore-undo failed → gone
+      // Both present: same value = reconciled; different = restore left a wrong value.
+      const same =
+        JSON.stringify({ m: before.metadata, v: before.vector }) ===
+        JSON.stringify({ m: current.metadata, v: current.vector })
+      return same ? null : 'loss'
+    }
+    for (const [id, before] of nounBefore) {
+      const d = classify(before, await this.storage.readNounRaw(id))
+      if (d) out.push({ id, kind: 'noun', disposition: d })
+    }
+    for (const [id, before] of verbBefore) {
+      const d = classify(before, await this.storage.readVerbRaw(id))
+      if (d) out.push({ id, kind: 'verb', disposition: d })
+    }
+    return out
+  }
+
   // ==========================================================================
   // Single-operation commit (Model-B per-write group-commit)
   // ==========================================================================
@@ -830,7 +902,7 @@ export class GenerationStore {
     touched: { nouns?: string[]; verbs?: string[] }
     execute: () => Promise<void>
     precommit?: (before: CommitBeforeImages) => void
-  }): Promise<{ generation: number; timestamp: number }> {
+  }): Promise<{ generation: number; timestamp: number; degraded?: string[] }> {
     return this.withMutex(async () => {
       const nouns = args.touched.nouns ? [...new Set(args.touched.nouns)] : []
       const verbs = args.touched.verbs ? [...new Set(args.touched.verbs)] : []
@@ -873,9 +945,42 @@ export class GenerationStore {
         await args.execute()
       } catch (err) {
         this.inTransact = false
-        // Live write failed: nothing buffered, nothing on disk. Return the
-        // reservation (when no concurrent bump consumed a later number) so
-        // generation() is unchanged.
+        // A failed rollback (TransactionRollbackError) may have left canonical
+        // storage inconsistent — the trapdoor. Reconcile against the
+        // before-images to decide the honest response (David's ruling:
+        // adopt-forward when safe, else fail loud).
+        if (err instanceof TransactionRollbackError) {
+          const records = await this.reconcileFailedRollback(nounBefore, verbBefore)
+          if (records.length > 0 && records.every((r) => r.disposition === 'orphan')) {
+            // ADOPT FORWARD: the durably-present orphan(s) are exactly what this
+            // single-op add wrote. Keep + buffer the generation so the record is
+            // legitimately committed and readable; the derived index may be
+            // incomplete for these ids until the next rebuild/repairIndex (the
+            // egress guard prevents wrong results meanwhile). Loud, honest,
+            // no double-write.
+            this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp })
+            this.pendingGens.push(gen)
+            this.extendChains(gen, nouns, verbs)
+            prodLog.warn(
+              `[GenerationStore] Recovered a failed rollback FORWARD: single-op write ` +
+                `committed as generation ${gen} because its canonical undo could not be ` +
+                `applied (record(s) ${records.map((r) => r.id).join(', ')} are durable). ` +
+                `The derived index may be incomplete for these ids — run repairIndex() to heal.`
+            )
+            return { generation: gen, timestamp, degraded: records.map((r) => r.id) }
+          }
+          if (records.length > 0) {
+            // FAIL LOUD: a loss (or mixed) cannot be safely adopted. Return the
+            // reservation and throw — brainy quarantines writes until repair.
+            if (this.counter === gen) this.counter = gen - 1
+            throw new StoreInconsistentError(records, err.originalError)
+          }
+          // records.length === 0: canonical is cleanly rolled back (only a
+          // derived undo failed) — fall through to the clean-abort path below.
+        }
+        // Live write failed with canonical cleanly rolled back: nothing buffered,
+        // nothing durable. Return the reservation (when no concurrent bump
+        // consumed a later number) so generation() is unchanged.
         if (this.counter === gen) this.counter = gen - 1
         throw err
       }
diff --git a/src/index.ts b/src/index.ts
index a7698b39..3915025d 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -174,8 +174,10 @@ export type {
 export {
   GenerationConflictError,
   SpeculativeOverlayError,
-  GenerationCompactedError
+  GenerationCompactedError,
+  StoreInconsistentError
 } from './db/errors.js'
+export type { UnreconciledRecord } from './db/errors.js'
 export type {
   TxOperation,
   TxAddOperation,
diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts
index 16332109..75dccc05 100644
--- a/src/transaction/Transaction.ts
+++ b/src/transaction/Transaction.ts
@@ -219,15 +219,22 @@ export class Transaction implements TransactionContext {
       }
     }
 
-    this.state = 'rolled_back'
+    // Honest terminal state: only claim 'rolled_back' when EVERY undo applied.
+    // If any undo failed, the store is not cleanly rolled back — mark
+    // 'inconsistent' so the commit orchestration reconciles rather than trusts
+    // a false 'rolled_back'. (Fixes the state half of the post-commit response
+    // lie: the record whose undo failed is still durable.)
+    this.state = rollbackErrors.length > 0 ? 'inconsistent' : 'rolled_back'
     this.endTime = Date.now()
 
     if (this.options.logging) {
       const duration = this.endTime - (this.startTime || this.endTime)
-      prodLog.info(`[Transaction] Rolled back in ${duration}ms`)
+      prodLog.info(`[Transaction] ${this.state === 'inconsistent' ? 'Rollback INCOMPLETE' : 'Rolled back'} in ${duration}ms`)
     }
 
-    // If rollback encountered errors, wrap them with original error
+    // If rollback encountered errors, wrap them with original error. The
+    // orchestration keys on `instanceof TransactionRollbackError` to know the
+    // undo did not fully apply and reconciliation is required.
     if (rollbackErrors.length > 0) {
       throw new TransactionRollbackError(
         `Transaction rollback encountered ${rollbackErrors.length} errors during cleanup`,
diff --git a/src/transaction/types.ts b/src/transaction/types.ts
index 5917ea4c..9a3a2eaa 100644
--- a/src/transaction/types.ts
+++ b/src/transaction/types.ts
@@ -14,6 +14,10 @@ export type TransactionState =
   | 'committed'    // Successfully committed
   | 'rolling_back' // Rolling back due to failure
   | 'rolled_back'  // Successfully rolled back
+  | 'inconsistent' // Rollback ran but ≥1 undo could not be applied — the store
+                   // is NOT cleanly rolled back; the commit orchestration must
+                   // reconcile (adopt-forward or fail-loud). Never claim
+                   // 'rolled_back' when an undo failed.
 
 /**
  * Rollback action - undoes an operation
diff --git a/tests/integration/rollback-trapdoor.test.ts b/tests/integration/rollback-trapdoor.test.ts
new file mode 100644
index 00000000..cf2245b8
--- /dev/null
+++ b/tests/integration/rollback-trapdoor.test.ts
@@ -0,0 +1,156 @@
+/**
+ * @module tests/integration/rollback-trapdoor
+ * @description Regression for a consumer-reported data-integrity bug: a failed
+ * CANONICAL rollback undo left the record durable while the request errored (a
+ * post-commit response lie). Reproduced by a failure-injection coherence harness:
+ * a late index op throws → rollback runs → an earlier op's canonical undo
+ * (storage.deleteNounMetadata) fails persistently → the record stays on disk
+ * while Transaction.rollback threw TransactionRollbackError AND lied with
+ * state='rolled_back'.
+ *
+ * Fix (David's ruling — adopt-forward when safe, else fail loud):
+ *  - SINGLE-op add whose only damage is a durably-present orphan → adopt it
+ *    forward: commit the generation (the record IS what add() wanted), return
+ *    success + a loud warn; the derived index heals on rebuild/repairIndex.
+ *  - MULTI-op batch, OR any restorative LOSS → StoreInconsistentError naming the
+ *    unreconciled ids; the brain enters write-quarantine (reads OK, writes
+ *    refused) until repairIndex() reconciles and lifts it. Transaction state is
+ *    'inconsistent', never the 'rolled_back' lie.
+ *
+ * The injection mirrors that harness: fault index.addToIndex (trigger rollback) and
+ * storage.deleteNounMetadata (fail the canonical undo).
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType } from '../../src/types/graphTypes.js'
+import { StoreInconsistentError } from '../../src/db/errors.js'
+
+let seq = 0
+const freshId = (): string =>
+  `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}`
+
+describe('rollback trapdoor — failed canonical undo (data integrity)', () => {
+  let dir: string
+  let brain: any
+  let restore: Array<() => void>
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-trapdoor-'))
+    brain = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      dimensions: 384,
+      silent: true
+    })
+    await brain.init()
+    restore = []
+  })
+
+  afterEach(async () => {
+    for (const r of restore) r()
+    try { await brain.close() } catch { /* ignore */ }
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  /** Fault the metadata-index add so a write's rollback is triggered. */
+  const faultIndexAdd = () => {
+    const idx = brain['metadataIndex']
+    const orig = idx.addToIndex.bind(idx)
+    idx.addToIndex = async () => { throw new Error('injected: index add failed') }
+    restore.push(() => { idx.addToIndex = orig })
+  }
+  /** Fault the canonical additive undo (delete of a just-created record). */
+  const faultCanonicalDelete = () => {
+    const storage = brain['storage']
+    const orig = storage.deleteNounMetadata.bind(storage)
+    storage.deleteNounMetadata = async () => { throw new Error('injected: canonical undo failed') }
+    restore.push(() => { storage.deleteNounMetadata = orig })
+  }
+
+  it('ADOPT-FORWARD: a single-op add whose canonical undo fails commits as a durable, get-able record (no throw, no quarantine)', async () => {
+    const id = freshId()
+    const genBefore = brain.generationStore.generation()
+
+    faultIndexAdd()        // triggers rollback
+    faultCanonicalDelete() // the trapdoor: the metadata record can't be un-written
+
+    // add() must NOT throw — the record it wanted is durable, so it is adopted.
+    const returned = await brain.add({ id, data: 'orphan adopted', type: NounType.Thing })
+    expect(returned).toBe(id)
+
+    // Un-fault so reads/subsequent writes use the real methods.
+    for (const r of restore) r()
+    restore = []
+
+    // The record is durable and get-able, and the generation advanced.
+    expect(await brain.get(id)).not.toBeNull()
+    expect(brain.generationStore.generation()).toBe(genBefore + 1)
+
+    // The store is NOT quarantined — a subsequent write succeeds.
+    const ok = await brain.add({ id: freshId(), data: 'still writable', type: NounType.Thing })
+    expect(await brain.get(ok)).not.toBeNull()
+  })
+
+  it('FAIL-LOUD: a multi-op transact whose canonical undo fails throws StoreInconsistentError and write-quarantines the brain', async () => {
+    // A pre-existing record to prove reads keep working under quarantine.
+    const anchor = await brain.add({ id: freshId(), data: 'anchor', type: NounType.Thing })
+    await brain.flush()
+
+    faultIndexAdd()
+    faultCanonicalDelete()
+
+    const a = freshId()
+    const b = freshId()
+    await expect(
+      brain.transact([
+        { op: 'add', id: a, data: 'batch A', type: NounType.Thing },
+        { op: 'add', id: b, data: 'batch B', type: NounType.Thing }
+      ])
+    ).rejects.toBeInstanceOf(StoreInconsistentError)
+
+    // Writes are refused (quarantine); reads still work.
+    await expect(
+      brain.add({ id: freshId(), data: 'blocked', type: NounType.Thing })
+    ).rejects.toThrow(/WRITE-QUARANTINED/)
+    expect(await brain.get(anchor)).not.toBeNull()
+
+    // repairIndex() reconciles and lifts the quarantine → writes resume.
+    for (const r of restore) r()
+    restore = []
+    await brain.repairIndex()
+
+    const ok = await brain.add({ id: freshId(), data: 'writable again', type: NounType.Thing })
+    expect(await brain.get(ok)).not.toBeNull()
+  })
+
+  it('the StoreInconsistentError names the unreconciled records with dispositions', async () => {
+    faultIndexAdd()
+    faultCanonicalDelete()
+
+    const a = freshId()
+    const b = freshId()
+    let caught: unknown
+    await brain
+      .transact([
+        { op: 'add', id: a, data: 'A', type: NounType.Thing },
+        { op: 'add', id: b, data: 'B', type: NounType.Thing }
+      ])
+      .catch((e: unknown) => (caught = e))
+
+    expect(caught).toBeInstanceOf(StoreInconsistentError)
+    const err = caught as StoreInconsistentError
+    expect(err.records.length).toBeGreaterThan(0)
+    // Every unreconciled record here is a durably-present orphan.
+    expect(err.records.every((r) => r.disposition === 'orphan')).toBe(true)
+    expect(err.message).toMatch(/WRITE-QUARANTINED/)
+
+    // Recover so afterEach can close cleanly.
+    for (const r of restore) r()
+    restore = []
+    await brain.repairIndex()
+  })
+})

From a7c7aa5102b96d1ae8db969b4e5761a17e828251 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 12:19:09 -0700
Subject: [PATCH 15/82] docs: RELEASES.md entry for 8.2.5 (honest
 rollback-failure response)

---
 RELEASES.md | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index a77ae27b..a6bc927e 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,40 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.5 — 2026-07-12 (honest response when a transaction rollback can't complete)
+
+Data-integrity fix. When a transaction failed and its rollback then *also* failed to undo a
+canonical write (retries exhausted), the old behavior logged, continued, and threw
+`TransactionRollbackError` — while the record it couldn't undo stayed durable on disk, and the
+transaction even reported its state as `'rolled_back'`. The caller got an error implying the write
+was undone; a read-back showed the record. A failed rollback had no truthful response.
+
+Rollback now tells the truth, with a two-branch contract:
+
+- **Adopt-forward (safe case).** A single-op write (`add`/`update`/…) whose only damage is a
+  durably-present record — the write the caller asked for — is **adopted**: its generation is
+  committed, the write returns success, and a loud warning records that the derived index may be
+  incomplete for that id until the next rebuild/`repairIndex()`. No error, no double-write; the
+  record is immediately durable and retrievable by `get()`.
+- **Fail loud + quarantine (unsafe case).** A multi-operation batch, or *any* case where a record
+  was lost (a remove/update whose restore-undo failed), throws the new **`StoreInconsistentError`**
+  naming every unreconciled record and its disposition (`orphan` vs `loss`), and puts the brain into
+  **write-quarantine**: reads keep working, but writes are refused until `repairIndex()` reconciles
+  the derived indexes against canonical storage and lifts the quarantine. The generation counter is
+  not advanced, and a transaction whose rollback failed is now `'inconsistent'`, never the
+  `'rolled_back'` lie.
+
+The decision is made by *observation*, not guesswork: both commit paths already hold byte-identical
+before-images, so after a failed rollback the store compares current canonical state to them to
+classify exactly which records are orphaned or lost.
+
+New export: `StoreInconsistentError` (with `.records` and `.cause`) and the `UnreconciledRecord` type.
+No other API change; `repairIndex()` gains the quarantine-lift behavior. Regression
+(`tests/integration/rollback-trapdoor.test.ts`) injects the exact failure (index add throws →
+canonical delete-undo fails) and pins adopt-forward (durable, get-able, not quarantined), fail-loud
+(`StoreInconsistentError` + quarantine + reads work + `repairIndex()` lifts it), and the error's
+record naming.
+
 ## v8.2.4 — 2026-07-12 (restore can no longer destroy the store it's recovering)
 
 Recovery-safety fix. `restore()` removed the entire live brain directory and THEN copied the

From ffd81ea2065251415d6656bace7a958a24b87956 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 12:27:01 -0700
Subject: [PATCH 16/82] chore(release): 8.2.5

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a7b221c..8602bf02 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.5](https://github.com/soulcraftlabs/brainy/compare/v8.2.4...v8.2.5) (2026-07-12)
+
+- docs: RELEASES.md entry for 8.2.5 (honest rollback-failure response) (a7c7aa5)
+- fix: honest response when a transaction rollback cannot complete (711d2f0)
+
+
 ### [8.2.4](https://github.com/soulcraftlabs/brainy/compare/v8.2.3...v8.2.4) (2026-07-12)
 
 - docs: RELEASES.md entry for 8.2.4 (non-destructive restore) (4574695)
diff --git a/package-lock.json b/package-lock.json
index 5865dcd0..7968654e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.4",
+  "version": "8.2.5",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.4",
+      "version": "8.2.5",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 0496e7b4..d5c4e740 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.4",
+  "version": "8.2.5",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From eb9c4eb96307e5732c6c430b586266ec26c88a3f Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 12 Jul 2026 18:24:31 -0700
Subject: [PATCH 17/82] test: pin the read-your-writes contract under the
 single writer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A returned write must be immediately readable by id, metadata filter, vector
search, and graph traversal — no await, delay, or retry between the write
returning and the read. This holds by construction because every projection
(canonical + HNSW + metadata index + graph index) commits inside the write's
transaction; the generation counter is the {seq} a caller can pin. The test
locks that guarantee so index maintenance can never quietly move off the commit
path (which would turn a returned write into a not-yet-queryable one).

First brainy chapter of the write/index-spine hardening program.
---
 .../read-your-writes-contract.test.ts         | 133 ++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 tests/integration/read-your-writes-contract.test.ts

diff --git a/tests/integration/read-your-writes-contract.test.ts b/tests/integration/read-your-writes-contract.test.ts
new file mode 100644
index 00000000..aa0c93e5
--- /dev/null
+++ b/tests/integration/read-your-writes-contract.test.ts
@@ -0,0 +1,133 @@
+/**
+ * @module tests/integration/read-your-writes-contract
+ * @description The read-your-writes contract — brainy's spine guarantee that a
+ * write which has RETURNED is immediately readable, under the single writer, by
+ * BOTH id and every index (metadata filter, vector search, graph traversal),
+ * with NO await, delay, or retry between the write returning and the read.
+ *
+ * Why it holds by construction on the canonical + JS-index path: every mutation
+ * runs its canonical writes AND all derived-index projections (HNSW, metadata
+ * index, graph index) as operations inside ONE transaction that commits before
+ * the method returns — there is no post-return "eventual" indexing window. get()
+ * additionally rides the storage write-through cache (read-after-write within
+ * the process). The generation counter is the {seq} a caller can pin: a write
+ * returns generation N and every subsequent live read observes ≥ N.
+ *
+ * This pins the guarantee so no future change can quietly move index maintenance
+ * off the commit path (which would turn a returned write into a not-yet-queryable
+ * one — the "a 200 is still not durability/queryability" failure the spine
+ * program exists to make impossible). The native accelerator must honor the same
+ * in-commit contract; a background consolidation/optimization must never gate
+ * queryability of an acknowledged write.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType, VerbType } from '../../src/types/graphTypes.js'
+
+let seq = 0
+const freshId = (): string =>
+  `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}`
+
+describe('read-your-writes contract (single-writer, no await between write and read)', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ryw-'))
+    brain = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      dimensions: 384,
+      silent: true
+    })
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('add(): the entity is immediately readable by id, metadata filter, AND vector search', async () => {
+    const id = freshId()
+    const marker = `ryw-${id}`
+    await brain.add({ id, data: 'read your writes probe alpha', type: NounType.Concept, metadata: { marker } })
+
+    // id read (write-through cache / canonical)
+    expect(await brain.get(id)).not.toBeNull()
+    // metadata-filter read (metadata index committed in-transaction)
+    const byMeta = await brain.find({ where: { marker }, limit: 10 })
+    expect(byMeta.map((r: any) => r.id)).toContain(id)
+    // vector read (HNSW committed in-transaction)
+    const byVector = await brain.find({ query: 'read your writes probe alpha', limit: 10 })
+    expect(byVector.map((r: any) => r.id)).toContain(id)
+    // type-filter read
+    const byType = await brain.find({ type: NounType.Concept, limit: 100 })
+    expect(byType.map((r: any) => r.id)).toContain(id)
+  })
+
+  it('update(): the new metadata is immediately queryable and the old value is immediately gone', async () => {
+    const id = freshId()
+    await brain.add({ id, data: 'updatable', type: NounType.Thing, metadata: { phase: 'before' } })
+    await brain.update({ id, metadata: { phase: 'after' } })
+
+    const after = await brain.find({ where: { phase: 'after' }, limit: 10 })
+    expect(after.map((r: any) => r.id)).toContain(id)
+    const before = await brain.find({ where: { phase: 'before' }, limit: 10 })
+    expect(before.map((r: any) => r.id)).not.toContain(id)
+  })
+
+  it('relate(): the edge is immediately traversable', async () => {
+    const a = await brain.add({ id: freshId(), data: 'A', type: NounType.Thing })
+    const b = await brain.add({ id: freshId(), data: 'B', type: NounType.Thing })
+    await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
+    expect((await brain.related(a)).map((r: any) => r.to)).toContain(b)
+  })
+
+  it('remove(): the entity is immediately gone from id, metadata, and vector reads', async () => {
+    const id = freshId()
+    const marker = `gone-${id}`
+    await brain.add({ id, data: 'to be removed promptly', type: NounType.Thing, metadata: { marker } })
+    expect(await brain.get(id)).not.toBeNull()
+
+    await brain.remove(id)
+    expect(await brain.get(id)).toBeNull()
+    expect((await brain.find({ where: { marker }, limit: 10 })).map((r: any) => r.id)).not.toContain(id)
+    expect((await brain.find({ query: 'to be removed promptly', limit: 10 })).map((r: any) => r.id)).not.toContain(id)
+  })
+
+  it('transact(): every item of a committed batch is immediately readable by id, index, and traversal', async () => {
+    const a = freshId()
+    const b = freshId()
+    const marker = `batch-${a}`
+    const db = await brain.transact([
+      { op: 'add', id: a, data: 'batch node A', type: NounType.Thing, metadata: { marker } },
+      { op: 'add', id: b, data: 'batch node B', type: NounType.Thing, metadata: { marker } },
+      { op: 'relate', from: a, to: b, type: VerbType.Contains }
+    ])
+    expect(db.generation).toBeGreaterThan(0)
+
+    expect(await brain.get(a)).not.toBeNull()
+    expect(await brain.get(b)).not.toBeNull()
+    const byMeta = (await brain.find({ where: { marker }, limit: 10 })).map((r: any) => r.id)
+    expect(byMeta).toContain(a)
+    expect(byMeta).toContain(b)
+    expect((await brain.related(a)).map((r: any) => r.to)).toContain(b)
+  })
+
+  it('the write returns a generation ({seq}) and a live read observes it — the awaitable-read primitive', async () => {
+    const id = freshId()
+    const before = brain.generationStore.generation()
+    await brain.add({ id, data: 'generation stamped', type: NounType.Thing })
+    const after = brain.generationStore.generation()
+    // The write advanced the generation…
+    expect(after).toBeGreaterThan(before)
+    // …and a read pinned to that generation observes the write (asOf ≥ N sees it).
+    const pinned = await brain.asOf(after)
+    expect(await pinned.get(id)).not.toBeNull()
+  })
+})

From 119087a75c13b682325d545d3fe652535f48c407 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 08:50:07 -0700
Subject: [PATCH 18/82] =?UTF-8?q?fix:=20spine=20hardening=20pass=201=20(pa?=
 =?UTF-8?q?rt)=20=E2=80=94=20count=20symmetry,=20honest=20partial-load,=20?=
 =?UTF-8?q?flush=20durability,=20read-fault=20propagation?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Write/index-spine hardening, first batch of Pass 1. Each fix restores an
invariant the surrounding code already intended; every one has a
fail-before/pass-after test.

- Pattern C, finding 5 (baseStorage): delete now decrements the user-facing
  scalar total symmetrically — deleteNounMetadata was decrementing only the
  per-type bucket, deleteVerbMetadata neither the bucket nor the scalar, so
  getNounCount()/getVerbCount() inflated permanently (the stale scalar wins
  pagination via Math.max and is persisted). Invariant now holds:
  scalar total === Σ per-type across add/update/delete and reopen.

- Pattern A, finding 3 (graph/lsm/LSMTree): a partial SSTable-load failure no
  longer publishes the manifest's full relationship count as healthy. Any
  per-SSTable load failure throws after the batch, which resets to honest-empty
  and lets the existing size()===0 self-heal rebuild run — size()/isHealthy()
  can no longer lie about a partial load.

- Pattern B, finding 6 (hnsw/hnswIndex): deferred flush() no longer clears
  dirty nodes whose connections failed to persist — failed nodes stay in the
  retry set, and flush() throws HnswFlushError instead of returning a lying
  node count. The immediate-mode first-noun saveHNSWSystem is un-swallowed, so
  addItem() rejects rather than returning an id for a rootless index.

- Pattern B, finding 11 (part — storage reads): new shared isAbsentError()
  helper (utils/errorClassification, ENOENT-only absence) applied to
  loadBinaryBlob and readObjectFromPath — a real IO fault (EIO/EACCES/EMFILE)
  now propagates loudly instead of masquerading as "absent", which had driven
  needless rebuilds / empty reads (loadBinaryBlob feeds the native provider).

Regression: 78 green across the 3 new suites + db-mvcc, generationStore,
temporal-vfs, rollback-trapdoor, restore-nondestructive. Full gate runs before
the Pass-1 release (David-gated). Remaining Pass 1: finding 11 getNoun/getVerb
legs, finding 4 (ColumnStore), finding 8 (pending-flush), finding 10 (degraded),
finding 7 (clear). Pattern A guards (1,2,9) as a follow-up release.
---
 src/graph/lsm/LSMTree.ts                      |  20 +++
 src/hnsw/hnswIndex.ts                         |  94 ++++++++---
 src/storage/adapters/fileSystemStorage.ts     |  19 ++-
 src/storage/baseStorage.ts                    |  36 +++-
 src/utils/errorClassification.ts              |  37 +++++
 tests/integration/count-invariant.test.ts     | 109 ++++++++++++
 .../graph/lsm-partial-load-failclosed.test.ts |  76 +++++++++
 tests/unit/hnsw/flush-failure.test.ts         | 157 ++++++++++++++++++
 8 files changed, 518 insertions(+), 30 deletions(-)
 create mode 100644 src/utils/errorClassification.ts
 create mode 100644 tests/integration/count-invariant.test.ts
 create mode 100644 tests/unit/graph/lsm-partial-load-failclosed.test.ts
 create mode 100644 tests/unit/hnsw/flush-failure.test.ts

diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts
index bfd48df4..e19ec145 100644
--- a/src/graph/lsm/LSMTree.ts
+++ b/src/graph/lsm/LSMTree.ts
@@ -574,6 +574,7 @@ export class LSMTree {
    * Load SSTables from storage based on manifest
    */
   private async loadSSTables(): Promise<void> {
+    const failures: string[] = []
     const loadPromises: Promise<void>[] = []
 
     this.manifest.sstables.forEach((level, sstableId) => {
@@ -598,7 +599,12 @@ export class LSMTree {
             }
           }
         } catch (error) {
+          // A per-SSTable load failure means the persisted adjacency is INCOMPLETE.
+          // Record it and fail the whole load closed (below): a partially-loaded
+          // tree that still publishes its full manifest count via size() would
+          // serve silent-empty traversals as truth (the cold-load swallow class).
           prodLog.warn(`LSMTree: Failed to load SSTable ${sstableId}`, error)
+          failures.push(sstableId)
         }
       })()
 
@@ -606,6 +612,20 @@ export class LSMTree {
     })
 
     await Promise.all(loadPromises)
+
+    if (failures.length > 0) {
+      // Fail closed. loadManifest()'s catch resets sstables/totalRelationships/
+      // sstablesByLevel to honest-empty, so size() reports 0 and the graph
+      // self-heal (_initializeGraphIndex size()===0 → rebuild) restores the index
+      // from the canonical records. Honest-partial is never published.
+      throw new Error(
+        `LSMTree(${this.config.storagePrefix}): ${failures.length} of ` +
+          `${this.manifest.sstables.size} SSTable(s) failed to load ` +
+          `(${failures.join(', ')}) — failing the load closed so size() reports 0 ` +
+          `and the graph self-heal rebuilds from canonical.`
+      )
+    }
+
     prodLog.info(`LSMTree: Loaded ${this.manifest.sstables.size} SSTables`)
   }
 
diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts
index b4948bf8..605d2a0b 100644
--- a/src/hnsw/hnswIndex.ts
+++ b/src/hnsw/hnswIndex.ts
@@ -25,6 +25,29 @@ const DEFAULT_CONFIG: HNSWConfig = {
   ml: 16 // Max level
 }
 
+/**
+ * @description Thrown by {@link JsHnswVectorIndex.flush} when one or more dirty
+ * nodes (or the system record) could not be persisted. The failed nodes remain
+ * in the dirty set for the next flush; this error tells the caller the flush did
+ * NOT achieve durability instead of a node-count that lies. Mandate: loud
+ * errors, never quiet losses.
+ */
+export class HnswFlushError extends Error {
+  constructor(
+    public readonly failedNodeCount: number,
+    public readonly systemFailed: boolean,
+    public override readonly cause?: Error
+  ) {
+    super(
+      `HNSW flush did not achieve durability: ${failedNodeCount} node(s) failed to ` +
+        `persist${systemFailed ? ' and the system record (entryPoint/maxLevel) failed' : ''}. ` +
+        `Failed nodes remain dirty for retry.` +
+        (cause ? ` First error: ${cause.message}` : '')
+    )
+    this.name = 'HnswFlushError'
+  }
+}
+
 /**
  * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls
  * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native
@@ -150,41 +173,69 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
     const startTime = Date.now()
     const nodeCount = this.dirtyNodes.size
 
-    // Batch persist all dirty nodes concurrently
+    // Batch persist all dirty nodes concurrently. A node whose connections FAIL
+    // to persist must stay dirty (retried on the next flush) — clearing it would
+    // silently drop the write forever. Track failures; only successfully-
+    // persisted (or deleted) nodes leave the dirty set, so nodes added to it
+    // during this flush are preserved.
+    const failedNodes = new Set<string>()
+    let firstError: Error | null = null
+
     if (this.dirtyNodes.size > 0) {
       const batchSize = 50 // Reasonable batch size for cloud storage
       const nodeIds = Array.from(this.dirtyNodes)
 
       for (let i = 0; i < nodeIds.length; i += batchSize) {
         const batch = nodeIds.slice(i, i + batchSize)
-        const promises = batch.map(nodeId => {
+        const promises = batch.map(async nodeId => {
           const noun = this.nouns.get(nodeId)
-          if (!noun) return Promise.resolve() // Node was deleted
-
-          return this.persistNodeConnections(nodeId, noun).catch(error => {
-            console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error)
-          })
+          if (!noun) return // Node was deleted — drop it from the dirty set.
+          try {
+            await this.persistNodeConnections(nodeId, noun)
+          } catch (error) {
+            failedNodes.add(nodeId)
+            if (firstError === null) firstError = error as Error
+            prodLog.error(`[HNSW flush] Failed to persist node ${nodeId}: ${(error as Error).message}`)
+          }
         })
 
-        await Promise.allSettled(promises)
+        await Promise.all(promises)
       }
 
-      this.dirtyNodes.clear()
+      // Remove only nodes that were persisted (or deleted mid-flush); keep the
+      // failed ones dirty for the next attempt.
+      for (const nodeId of nodeIds) {
+        if (!failedNodes.has(nodeId)) this.dirtyNodes.delete(nodeId)
+      }
     }
 
-    // Persist system data if dirty
+    // Persist system data if dirty — keep it dirty on failure so the next flush
+    // retries rather than losing the entry-point/maxLevel update.
+    let systemFailed = false
     if (this.dirtySystem) {
-      await this.storage.saveHNSWSystem({
-        entryPointId: this.entryPointId,
-        maxLevel: this.maxLevel
-      }).catch(error => {
-        console.error('[HNSW flush] Failed to persist system data:', error)
-      })
-
-      this.dirtySystem = false
+      try {
+        await this.storage.saveHNSWSystem({
+          entryPointId: this.entryPointId,
+          maxLevel: this.maxLevel
+        })
+        this.dirtySystem = false
+      } catch (error) {
+        systemFailed = true
+        if (firstError === null) firstError = error as Error
+        prodLog.error(`[HNSW flush] Failed to persist system data: ${(error as Error).message}`)
+      }
     }
 
     const duration = Date.now() - startTime
+
+    // Loud failure: if ANY node or the system record could not be persisted the
+    // flush did not achieve durability. Throw so callers (close(), explicit
+    // flush(), the flush-request watcher) see the failure instead of a success
+    // count that lies. The failed nodes/system stay dirty for retry.
+    if (failedNodes.size > 0 || systemFailed) {
+      throw new HnswFlushError(failedNodes.size, systemFailed, firstError ?? undefined)
+    }
+
     if (nodeCount > 0) {
       prodLog.info(`[HNSW] Flushed ${nodeCount} dirty nodes in ${duration}ms`)
     }
@@ -396,13 +447,14 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
       this.maxLevel = nounLevel
       this.nouns.set(id, noun)
 
-      // Persist system data for first noun (previously skipped)
+      // Persist system data for first noun (previously skipped). Surface a
+      // persist failure loudly — the entry point is the root of the whole index;
+      // silently dropping it while addItem() returns the id would strand every
+      // future search on a rootless index. Mandate: never a quiet loss.
       if (this.storage && this.persistMode === 'immediate') {
         await this.storage.saveHNSWSystem({
           entryPointId: this.entryPointId,
           maxLevel: this.maxLevel
-        }).catch(error => {
-          console.error('Failed to persist initial HNSW system data:', error)
         })
       } else if (this.persistMode === 'deferred') {
         this.dirtySystem = true
diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index e728cc41..27f1af89 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -17,6 +17,7 @@ import {
   WriterLockInfo
 } from '../baseStorage.js'
 import { getBrainyVersion } from '../../utils/index.js'
+import { isAbsentError } from '../../utils/errorClassification.js'
 
 // Node.js modules - dynamically imported to avoid issues in browser environments
 let fs: any
@@ -446,8 +447,12 @@ export class FileSystemStorage extends BaseStorage {
         return null
       }
 
-      console.error(`Error reading object from ${pathStr}:`, error)
-      return null
+      // A real storage fault (EIO/EACCES/EMFILE/…) is NOT "object absent". The
+      // ENOENT branch (above) already returns null, and the corrupted-JSON
+      // branch (above) is a deliberate concurrent-write tolerance; a genuine
+      // fault reaching here must propagate loudly rather than masquerade as a
+      // missing object — which would corrupt reads and drive needless rebuilds.
+      throw error
     }
   }
 
@@ -1207,8 +1212,14 @@ export class FileSystemStorage extends BaseStorage {
     await this.ensureInitialized()
     try {
       return await fs.promises.readFile(this.blobPath(key))
-    } catch {
-      return null
+    } catch (err) {
+      // Absent blob → null (the documented contract). A real fault
+      // (EIO/EACCES/EMFILE/…) must NOT be masked as "absent": doing so makes a
+      // present-but-unreadable blob look missing and drives a needless rebuild
+      // or an empty read (the native provider consumes this). Mandate: loud
+      // errors, never quiet losses.
+      if (isAbsentError(err)) return null
+      throw err
     }
   }
 
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index ed4755c7..dd1d3fee 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -3234,10 +3234,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     const priorCounted = isCountedVisibility(record?.visibility)
     if (priorType) {
       if (priorCounted) {
+        // Symmetric with the counted-add increment in saveNounMetadata_internal():
+        // decrement BOTH the user-facing scalar total (getNounCount / counts.json) AND
+        // the per-type bucket, then persist. The scalar decrement was previously
+        // omitted here, so deletes permanently inflated getNounCount() — the stale
+        // scalar wins pagination via Math.max(totalNounCount, collected.length) and is
+        // persisted by scheduleCountPersist(). With this, the invariant
+        // `totalNounCount === Σ nounCountsByType` holds across add / update-flip / delete.
+        this.decrementEntityCount(priorType)
         const idx = TypeUtils.getNounIndex(priorType)
         if (this.nounCountsByType[idx] > 0) {
           this.nounCountsByType[idx]--
         }
+        this.scheduleCountPersist().catch(() => {
+          // Ignore persist errors — the in-memory count is authoritative; a later
+          // operation retries the persist.
+        })
       }
 
       // Symmetric subtype decrement — same non-empty-string guard as the write path.
@@ -3392,16 +3404,30 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     await this.ensureInitialized()
 
     // Direct O(1) delete with ID-first path. Read the canonical record BEFORE
-    // removing it so the verb-subtype decrement is sourced from the edge's own
-    // metadata (`verb` type + `subtype`) rather than an id-keyed cache — symmetric
-    // with the increment in `saveVerbMetadata_internal()`. Verb deletes do not
-    // touch `verbCountsByType` in this path (matching prior behavior), so no
-    // visibility read is needed here.
+    // removing it so every decrement is sourced from the edge's own metadata
+    // (`verb` type, `subtype`, `visibility`) rather than an id-keyed cache —
+    // symmetric with the increments in `saveVerbMetadata_internal()`.
     const path = getVerbMetadataPath(id)
     const record = await this.readCanonicalObject(path)
     await this.deleteCanonicalObject(path)
 
     const priorVerb = record?.verb as VerbType | undefined
+    // Symmetric count decrement (previously OMITTED — verb deletes touched neither the
+    // scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
+    // edge bumped BOTH the scalar (incrementVerbCount in saveVerbMetadata_internal) and
+    // the per-type bucket (the unconditional bump in saveVerb_internal that the metadata
+    // path keeps for counted edges). Delete must undo both, gated on the SAME visibility
+    // as the add, so `totalVerbCount === Σ verbCountsByType` holds across delete.
+    const priorCounted = isCountedVisibility(record?.visibility)
+    if (priorVerb && priorCounted) {
+      this.decrementVerbCount(priorVerb)
+      const idx = TypeUtils.getVerbIndex(priorVerb)
+      if (this.verbCountsByType[idx] > 0) this.verbCountsByType[idx]--
+      this.scheduleCountPersist().catch(() => {
+        // Ignore persist errors — in-memory count is authoritative; a later op retries.
+      })
+    }
+
     const priorSubtype = typeof record?.subtype === 'string' && (record.subtype as string).length > 0
       ? (record.subtype as string)
       : undefined
diff --git a/src/utils/errorClassification.ts b/src/utils/errorClassification.ts
new file mode 100644
index 00000000..a3e7dad8
--- /dev/null
+++ b/src/utils/errorClassification.ts
@@ -0,0 +1,37 @@
+/**
+ * @module utils/errorClassification
+ * @description Shared classification of caught errors into "genuine absence" vs
+ * "real fault" — the antidote to blind `catch { return null }` handlers that
+ * cannot tell ENOENT (the object is legitimately not on disk) from EIO / EACCES
+ * / EMFILE / … (a transient or permission fault on data that IS on disk).
+ * Masking a fault as absence yields wrong results (a present record read as
+ * "not found") or a needless rebuild. Mandate: loud errors, never quiet losses.
+ */
+
+/**
+ * The error `code`s that denote GENUINE absence of a file/object. Only `ENOENT`
+ * ("no such file or directory") qualifies — on every platform Node maps a
+ * missing file/directory to ENOENT, and no other errno means "simply not
+ * there". Every other errno (EIO, EACCES, EPERM, EMFILE, ENFILE, EBUSY,
+ * ENOTDIR, EISDIR, ELOOP) is a real fault and must propagate, as must any error
+ * without an errno `code` (parse/decompress failures, generic Errors).
+ * `ENOTDIR`/`EISDIR` are deliberately faults: a path component of the wrong
+ * type is corruption, not benign absence. Named constant so a future
+ * genuine-absence code can be added in one reviewed place.
+ */
+const ABSENCE_CODES: ReadonlySet<string> = new Set(['ENOENT'])
+
+/**
+ * @description True IFF `e` represents genuine absence (an ENOENT-class errno),
+ * for which returning `null`/`[]`/`undefined` is the correct answer. Returns
+ * `false` for every real fault, so the canonical call site is:
+ * `catch (e) { if (isAbsentError(e)) return null; throw e }`.
+ *
+ * @param e - The caught value (typed `unknown`; non-objects are never absence).
+ * @returns Whether the error means "the thing is simply not there".
+ */
+export function isAbsentError(e: unknown): boolean {
+  if (e === null || typeof e !== 'object') return false
+  const code = (e as { code?: unknown }).code
+  return typeof code === 'string' && ABSENCE_CODES.has(code)
+}
diff --git a/tests/integration/count-invariant.test.ts b/tests/integration/count-invariant.test.ts
new file mode 100644
index 00000000..3896b625
--- /dev/null
+++ b/tests/integration/count-invariant.test.ts
@@ -0,0 +1,109 @@
+/**
+ * @module tests/integration/count-invariant
+ * @description Pattern-C acceptance: the user-facing scalar total and the
+ * per-type array are ONE consistent projection — `getNounCount() === Σ
+ * nounCountsByType` (and the verb mirror) after any interleaving of add /
+ * update-visibility-flip / delete, AND across a reopen.
+ *
+ * The bug (finding 5): delete decremented the per-type array but never the
+ * scalar for nouns, and neither for verbs — so `getNounCount()`/`getVerbCount()`
+ * inflated permanently (the stale scalar wins pagination via
+ * `Math.max(total, collected.length)` and is persisted). The fix restores the
+ * symmetric decrement the code always intended (its own comments say "symmetric
+ * with the increments"). This test would report an inflated count before the
+ * fix and the exact count after.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType, VerbType } from '../../src/types/graphTypes.js'
+
+const sum = (a: Uint32Array): number => a.reduce((s, c) => s + c, 0)
+
+describe('count invariant — scalar total === Σ per-type, across delete + reopen (finding 5)', () => {
+  let dir: string
+  let brain: any
+
+  const open = async (d: string) => {
+    const b = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: d },
+      dimensions: 384,
+      silent: true
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-count-inv-'))
+    brain = await open(dir)
+  })
+
+  afterEach(async () => {
+    try { await brain.close() } catch { /* already closed */ }
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('noun delete decrements the scalar total, not just the per-type bucket', async () => {
+    const things: string[] = []
+    for (let i = 0; i < 5; i++) things.push(await brain.add({ data: `thing ${i}`, type: NounType.Thing }))
+    for (let i = 0; i < 3; i++) await brain.add({ data: `concept ${i}`, type: NounType.Concept })
+
+    const storage = brain['storage']
+    expect(await storage.getNounCount()).toBe(8)
+    expect(sum(storage.getNounCountsByType())).toBe(8)
+
+    // Delete 2 Things — the scalar must drop with the bucket (pre-fix it stayed 8).
+    await brain.remove(things[0])
+    await brain.remove(things[1])
+
+    expect(await storage.getNounCount()).toBe(6)
+    expect(sum(storage.getNounCountsByType())).toBe(6)
+    // The invariant: scalar === Σ per-type.
+    expect(await storage.getNounCount()).toBe(sum(storage.getNounCountsByType()))
+  })
+
+  it('verb delete decrements BOTH the scalar total AND the per-type bucket', async () => {
+    const ids: string[] = []
+    for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `node ${i}`, type: NounType.Thing }))
+    const edges: string[] = []
+    edges.push(await brain.relate({ from: ids[0], to: ids[1], type: VerbType.RelatedTo }))
+    edges.push(await brain.relate({ from: ids[1], to: ids[2], type: VerbType.RelatedTo }))
+    edges.push(await brain.relate({ from: ids[2], to: ids[3], type: VerbType.Contains }))
+    edges.push(await brain.relate({ from: ids[0], to: ids[3], type: VerbType.Contains }))
+
+    const storage = brain['storage']
+    expect(await storage.getVerbCount()).toBe(4)
+    expect(sum(storage.getVerbCountsByType())).toBe(4)
+
+    // Unrelate one edge — pre-fix, verb delete touched NEITHER counter (both stayed 4).
+    await brain.unrelate(edges[0])
+
+    expect(await storage.getVerbCount()).toBe(3)
+    expect(sum(storage.getVerbCountsByType())).toBe(3)
+    expect(await storage.getVerbCount()).toBe(sum(storage.getVerbCountsByType()))
+  })
+
+  it('the corrected counts persist and survive a reopen', async () => {
+    const things: string[] = []
+    for (let i = 0; i < 6; i++) things.push(await brain.add({ data: `t${i}`, type: NounType.Thing }))
+    const a = await brain.relate({ from: things[0], to: things[1], type: VerbType.RelatedTo })
+    await brain.relate({ from: things[1], to: things[2], type: VerbType.RelatedTo })
+    await brain.remove(things[5])
+    await brain.unrelate(a)
+    await brain.flush()
+    await brain.close()
+
+    // Reopen from disk — counts.json must carry the corrected totals.
+    brain = await open(dir)
+    const storage = brain['storage']
+    expect(await storage.getNounCount()).toBe(5)
+    expect(await storage.getVerbCount()).toBe(1)
+    expect(await storage.getNounCount()).toBe(sum(storage.getNounCountsByType()))
+    expect(await storage.getVerbCount()).toBe(sum(storage.getVerbCountsByType()))
+  })
+})
diff --git a/tests/unit/graph/lsm-partial-load-failclosed.test.ts b/tests/unit/graph/lsm-partial-load-failclosed.test.ts
new file mode 100644
index 00000000..7082c0c6
--- /dev/null
+++ b/tests/unit/graph/lsm-partial-load-failclosed.test.ts
@@ -0,0 +1,76 @@
+/**
+ * @module tests/unit/graph/lsm-partial-load-failclosed
+ * @description LSMTree partial-load fail-closed guard (Finding 3 of the honest
+ * index-readiness spine plan). `loadSSTables()` used to swallow a per-SSTable
+ * load failure with only a `prodLog.warn`, then `loadManifest()` still published
+ * the FULL persisted `totalRelationships` count — so on a partial cold load,
+ * `size()`/`isHealthy()` reported the full count and "healthy" while a subset of
+ * the tree's edges were silently unqueryable. Fixed by failing the whole load
+ * closed on ANY per-SSTable failure: `loadManifest()`'s existing catch then
+ * resets `sstables`/`totalRelationships`/`sstablesByLevel` to honest-empty, so
+ * `size()` reports 0 and the graph layer's existing `size()===0` self-heal
+ * rebuilds from canonical records — the dishonest partial state can no longer be
+ * observed.
+ */
+import { describe, it, expect } from 'vitest'
+import { LSMTree } from '../../../src/graph/lsm/LSMTree.js'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+
+describe('LSMTree partial SSTable load fails closed (honest size())', () => {
+  it('one SSTable fails to load → size() reports 0, not the manifest count', async () => {
+    const storage = new MemoryStorage()
+    await storage.init()
+    const prefix = 'test-lsm-partial'
+
+    const t1 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false })
+    await t1.init()
+    await t1.add('a', 'b'); await t1.flush()   // SSTable #1
+    await t1.add('c', 'd'); await t1.flush()   // SSTable #2
+    expect(t1.size()).toBeGreaterThanOrEqual(2)
+    await t1.close()
+
+    // Find an SSTable id from the persisted manifest.
+    const manifest = await storage.getMetadata(`${prefix}-manifest`)
+    const sstableIds = Object.keys((manifest!.data as any).sstables)
+    expect(sstableIds.length).toBeGreaterThanOrEqual(2)
+
+    // Reopen with ONE SSTable load forced to fail (deterministic).
+    const t2 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false })
+    const realGet = storage.getMetadata.bind(storage)
+    ;(storage as any).getMetadata = async (key: string) => {
+      if (key === `${prefix}-${sstableIds[1]}`) throw new Error('simulated SSTable load failure')
+      return realGet(key)
+    }
+    await t2.init()
+
+    // BEFORE THE FIX: t2.size() === (full manifest count)  ← LIE (partial load, full count)
+    // AFTER THE FIX:  t2.size() === 0                       ← fail-closed → self-heal rebuilds
+    expect(t2.size()).toBe(0)
+    expect(t2.isHealthy()).toBe(true) // honestly-empty-but-initialized, not "healthy with a lie"
+
+    await t2.close()
+  })
+
+  it('a clean load with no failures still reports the full honest count', async () => {
+    const storage = new MemoryStorage()
+    await storage.init()
+    const prefix = 'test-lsm-clean'
+
+    const t1 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false })
+    await t1.init()
+    await t1.add('a', 'b'); await t1.flush()
+    await t1.add('c', 'd'); await t1.flush()
+    const expectedSize = t1.size()
+    expect(expectedSize).toBeGreaterThanOrEqual(2)
+    await t1.close()
+
+    // Reopen with no injected failures — a normal, fully-successful cold load.
+    const t2 = new LSMTree(storage, { storagePrefix: prefix, enableCompaction: false })
+    await t2.init()
+
+    expect(t2.size()).toBe(expectedSize)
+    expect(t2.isHealthy()).toBe(true)
+
+    await t2.close()
+  })
+})
diff --git a/tests/unit/hnsw/flush-failure.test.ts b/tests/unit/hnsw/flush-failure.test.ts
new file mode 100644
index 00000000..14bdff6e
--- /dev/null
+++ b/tests/unit/hnsw/flush-failure.test.ts
@@ -0,0 +1,157 @@
+/**
+ * HNSW deferred-flush durability tests (Finding 6 — spine-plan Part B,
+ * "blind catch" audit).
+ *
+ * `JsHnswVectorIndex.flush()` used to swallow a per-node
+ * `persistNodeConnections` failure (and a system-record `saveHNSWSystem`
+ * failure) behind `console.error`, then unconditionally clear the dirty set
+ * and report the pre-flush node count as "success". A transient write fault
+ * therefore silently dropped that node's connections from durable storage
+ * forever, with `flush()` having lied about it. These tests pin the fix:
+ * a node (or the system record) that fails to persist stays in the
+ * dirty/retry set, and `flush()` throws {@link HnswFlushError} instead of
+ * returning a count. The immediate-mode first-noun `saveHNSWSystem` swallow
+ * (while `addItem()` still returned the id) is covered too.
+ */
+
+import { describe, it, expect, vi } from 'vitest'
+import { v4 as uuidv4 } from 'uuid'
+import { JsHnswVectorIndex, HnswFlushError } from '../../../src/hnsw/hnswIndex.js'
+import { euclideanDistance } from '../../../src/utils/index.js'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+
+// Helper: generate a random vector of given dimension (mirrors lazy-vectors.test.ts).
+function randomVector(dim: number): number[] {
+  return Array.from({ length: dim }, () => Math.random() * 2 - 1)
+}
+
+describe('HNSW deferred-flush durability (Finding 6)', () => {
+  const dim = 8
+
+  it('a clean flush persists every dirty node and clears the dirty set', async () => {
+    const storage = new MemoryStorage()
+    const index = new JsHnswVectorIndex(
+      { M: 4, efConstruction: 50, efSearch: 20 },
+      euclideanDistance,
+      { useParallelization: false, storage, persistMode: 'deferred' }
+    )
+
+    for (let i = 0; i < 5; i++) {
+      await index.addItem({ id: uuidv4(), vector: randomVector(dim) })
+    }
+
+    // Sanity: inserting several items in deferred mode does dirty something.
+    expect((index as any).dirtyNodes.size).toBeGreaterThan(0)
+
+    const flushed = await index.flush()
+    expect(typeof flushed).toBe('number')
+    expect((index as any).dirtyNodes.size).toBe(0)
+    expect((index as any).dirtySystem).toBe(false)
+  })
+
+  it('retains a node whose connections failed to persist and surfaces HnswFlushError instead of reporting success', async () => {
+    const storage = new MemoryStorage()
+    const index = new JsHnswVectorIndex(
+      { M: 4, efConstruction: 50, efSearch: 20 },
+      euclideanDistance,
+      { useParallelization: false, storage, persistMode: 'deferred' }
+    )
+
+    const ids: string[] = []
+    for (let i = 0; i < 5; i++) {
+      const id = uuidv4()
+      ids.push(id)
+      await index.addItem({ id, vector: randomVector(dim) })
+    }
+
+    // The very first node is guaranteed to become a neighbor of the second
+    // insert (it is the only existing node in the graph at that point), so it
+    // is deterministically present in the dirty set before any fault.
+    const failId = ids[0]
+    const dirtyBefore = (index as any).dirtyNodes as Set<string>
+    expect(dirtyBefore.has(failId)).toBe(true)
+
+    const originalSave = storage.saveVectorIndexData.bind(storage)
+    const spy = vi
+      .spyOn(storage, 'saveVectorIndexData')
+      .mockImplementation(async (nounId, hnswData) => {
+        if (nounId === failId) {
+          throw Object.assign(new Error('simulated write fault'), { code: 'EIO' })
+        }
+        return originalSave(nounId, hnswData)
+      })
+
+    await expect(index.flush()).rejects.toBeInstanceOf(HnswFlushError)
+
+    const dirtyAfter = (index as any).dirtyNodes as Set<string>
+    // The failed node stays dirty for the next retry ...
+    expect(dirtyAfter.has(failId)).toBe(true)
+    // ... and every node that DID persist successfully leaves the dirty set —
+    // the failure of one node must not re-dirty (or fail to clear) the rest.
+    expect(dirtyAfter.size).toBe(1)
+
+    // Recovery: once the fault clears, the retained node persists and the
+    // flush reports success again (the intended retry path).
+    spy.mockRestore()
+    const flushed = await index.flush()
+    expect(typeof flushed).toBe('number')
+    expect((index as any).dirtyNodes.size).toBe(0)
+  })
+
+  it('surfaces a system-record persist failure as HnswFlushError and keeps dirtySystem set for retry', async () => {
+    const storage = new MemoryStorage()
+    const index = new JsHnswVectorIndex(
+      { M: 4, efConstruction: 50, efSearch: 20 },
+      euclideanDistance,
+      { useParallelization: false, storage, persistMode: 'deferred' }
+    )
+
+    // First noun in deferred mode marks dirtySystem (entryPoint/maxLevel).
+    await index.addItem({ id: uuidv4(), vector: randomVector(dim) })
+    expect((index as any).dirtySystem).toBe(true)
+
+    const spy = vi
+      .spyOn(storage, 'saveHNSWSystem')
+      .mockRejectedValue(
+        Object.assign(new Error('simulated system write fault'), { code: 'EIO' })
+      )
+
+    let caught: unknown
+    try {
+      await index.flush()
+    } catch (error) {
+      caught = error
+    }
+
+    expect(caught).toBeInstanceOf(HnswFlushError)
+    expect((caught as HnswFlushError).systemFailed).toBe(true)
+    // The system record must stay dirty — a lost entry point/maxLevel update
+    // must be retried, not silently dropped.
+    expect((index as any).dirtySystem).toBe(true)
+
+    spy.mockRestore()
+    const flushed = await index.flush()
+    expect(typeof flushed).toBe('number')
+    expect((index as any).dirtySystem).toBe(false)
+  })
+
+  it('surfaces the immediate-mode first-noun system persist failure via a rejecting addItem()', async () => {
+    const storage = new MemoryStorage()
+    const index = new JsHnswVectorIndex(
+      { M: 4, efConstruction: 50, efSearch: 20 },
+      euclideanDistance,
+      { useParallelization: false, storage } // default persistMode: 'immediate'
+    )
+
+    vi.spyOn(storage, 'saveHNSWSystem').mockRejectedValue(
+      Object.assign(new Error('simulated system write fault'), { code: 'EIO' })
+    )
+
+    // Previously this swallowed the error (console.error) and addItem()
+    // still resolved with the id — stranding a brand-new index whose root
+    // (entry point) was never actually persisted. Now it must reject.
+    await expect(
+      index.addItem({ id: uuidv4(), vector: randomVector(dim) })
+    ).rejects.toThrow('simulated system write fault')
+  })
+})

From af5d2f389b0a5ff8fb0a4954481e0d46f18f5cb7 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 09:14:52 -0700
Subject: [PATCH 19/82] fix: surface segment/entity read faults loudly instead
 of masking as absent

ColumnStore silently skipped a manifest-listed segment it could not load: a
corrupt or missing segment dropped every one of its entities out of
filter/rangeQuery/sortTopK with no error, so an inconsistent index read as a
merely short result. It now throws a typed ColumnSegmentLoadError when a listed
segment yields undecodable or no bytes, and lets a genuine storage IO fault
propagate verbatim. Only a field with no manifest at all stays benign (nothing
was ever written for it).

baseStorage.getNoun_internal / getVerb_internal likewise caught every error and
returned null, reporting a present-but-unreadable entity as "not found". They
now return null only for genuine ENOENT-class absence (via isAbsentError) and
rethrow real faults and deserialize errors.

Pattern-B (blind catch) hardening: absence -> null, fault -> loud.
---
 src/indexes/columnStore/ColumnStore.ts        | 104 ++++++++++----
 src/storage/baseStorage.ts                    |  17 ++-
 .../columnStore/segment-load-fault.test.ts    | 128 ++++++++++++++++++
 3 files changed, 216 insertions(+), 33 deletions(-)
 create mode 100644 tests/unit/indexes/columnStore/segment-load-fault.test.ts

diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts
index 2c07d433..d33c05c4 100644
--- a/src/indexes/columnStore/ColumnStore.ts
+++ b/src/indexes/columnStore/ColumnStore.ts
@@ -69,6 +69,34 @@ interface HeapEntry {
  * await store.flush()
  * const newest = await store.sortTopK('createdAt', 'desc', 10) // bigint[]
  */
+/**
+ * @description Thrown when a segment the manifest lists cannot be loaded —
+ * either it yields no bytes (missing/empty on disk) or its bytes are
+ * undecodable (corruption). Previously such a segment was silently skipped,
+ * dropping ALL of its entities from every `filter`/`rangeQuery`/`sortTopK` with
+ * no error — a manifest↔segments divergence that looked like a short result.
+ * Surfacing it loudly makes the divergence visible and repairable. (A genuine
+ * storage IO fault is a different class — it propagates as the underlying error,
+ * not wrapped in this.)
+ */
+export class ColumnSegmentLoadError extends Error {
+  /** The indexed field whose segment failed to load. */
+  public readonly field: string
+  /** The manifest-listed segment id that could not be loaded. */
+  public readonly segmentId: number | string
+  constructor(field: string, segmentId: number | string, reason: string) {
+    super(
+      `ColumnStore segment ${field}:${segmentId} is listed in the manifest but ` +
+        `could not be loaded (${reason}). The index is inconsistent with its ` +
+        `manifest — rebuild/repair the metadata index rather than trusting a ` +
+        `short query result.`
+    )
+    this.name = 'ColumnSegmentLoadError'
+    this.field = field
+    this.segmentId = segmentId
+  }
+}
+
 export class ColumnStore implements ColumnStoreProvider {
   private storage!: StorageAdapter
   private idMapper!: EntityIdMapper
@@ -594,14 +622,15 @@ export class ColumnStore implements ColumnStoreProvider {
       let cursor = this.segmentCache.get(cacheKey)
 
       if (!cursor) {
-        const loaded = await this.loadSegmentCursor(field, seg)
-        if (loaded) {
-          cursor = loaded
-          this.segmentCache.set(cacheKey, cursor)
-        }
+        // loadSegmentCursor either returns a cursor or THROWS — a corrupt /
+        // missing manifest-listed segment raises ColumnSegmentLoadError and a
+        // real storage fault propagates, so a listed segment is never silently
+        // dropped from the result set.
+        cursor = await this.loadSegmentCursor(field, seg)
+        this.segmentCache.set(cacheKey, cursor)
       }
 
-      if (cursor) cursors.push(cursor)
+      cursors.push(cursor)
     }
 
     return cursors
@@ -615,39 +644,51 @@ export class ColumnStore implements ColumnStoreProvider {
    * `.cidx` object-path so indexes written before the format unification keep
    * loading correctly. Mirror of cortex's 2.3.1 read-side fallback.
    */
-  private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise<ColumnSegmentCursor | null> {
+  private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise<ColumnSegmentCursor> {
     const manifest = this.manifests.get(field)
-    if (!manifest) return null
+    if (!manifest) {
+      // Defensive: getSegmentCursors guards on the manifest before calling.
+      throw new ColumnSegmentLoadError(field, seg.id, 'no manifest for field')
+    }
 
-    try {
-      let buf: Buffer | null = null
+    let buf: Buffer | null = null
 
-      const storage = this.storage as unknown as {
-        loadBinaryBlob?: (key: string) => Promise<Buffer | null>
-        readObjectFromPath: (path: string) => Promise<any>
-      }
+    const storage = this.storage as unknown as {
+      loadBinaryBlob?: (key: string) => Promise<Buffer | null>
+      readObjectFromPath: (path: string) => Promise<any>
+    }
 
-      // Preferred: raw blob at the cor-shared key.
-      if (typeof storage.loadBinaryBlob === 'function') {
-        const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}`
-        const blob = await storage.loadBinaryBlob(key)
-        if (blob && blob.length > 0) buf = blob
-      }
+    // Preferred: raw blob at the cor-shared key. A real IO fault PROPAGATES —
+    // loadBinaryBlob throws on a fault and returns null only for genuine absence
+    // (a present-but-unreadable segment must not read as "missing").
+    if (typeof storage.loadBinaryBlob === 'function') {
+      const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}`
+      const blob = await storage.loadBinaryBlob(key)
+      if (blob && blob.length > 0) buf = blob
+    }
 
-      // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path.
-      if (!buf) {
-        const segPath = manifest.segmentPath(seg.level, seg.id)
-        const stored = await storage.readObjectFromPath(segPath)
-        if (!stored) return null
+    // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path.
+    if (!buf) {
+      const segPath = manifest.segmentPath(seg.level, seg.id)
+      const stored = await storage.readObjectFromPath(segPath)
+      if (stored) {
         if (stored._binary && stored.data) {
           buf = Buffer.from(stored.data, 'base64')
         } else if (Buffer.isBuffer(stored)) {
           buf = stored
-        } else {
-          return null
         }
       }
+    }
 
+    // A manifest-listed segment that yields NO loadable bytes is corruption, not
+    // benign absence: swallowing it silently dropped all of the segment's
+    // entities from every filter/rangeQuery/sortTopK with no error. Surface it
+    // loudly so the manifest↔segments divergence is visible (and repairable).
+    if (!buf) {
+      throw new ColumnSegmentLoadError(field, seg.id, 'manifest-listed segment has no loadable bytes')
+    }
+
+    try {
       const parsed = readSegmentFromBuffer(buf)
       return new ColumnSegmentCursor(
         parsed.header,
@@ -655,8 +696,13 @@ export class ColumnStore implements ColumnStoreProvider {
         parsed.entityIds,
         parsed.tombstones
       )
-    } catch {
-      return null
+    } catch (err) {
+      // Undecodable bytes for a listed segment — corruption, not a short result.
+      throw new ColumnSegmentLoadError(
+        field,
+        seg.id,
+        `segment decode failed: ${(err as Error).message}`
+      )
     }
   }
 
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index dd1d3fee..41ebeb45 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -29,6 +29,7 @@ import { getShardId } from './sharding.js'
 import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js'
 import { unwrapBinaryData } from './binaryDataCodec.js'
 import { prodLog } from '../utils/logger.js'
+import { isAbsentError } from '../utils/errorClassification.js'
 import { BrainyError } from '../errors/brainyError.js'
 import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js'
 import {
@@ -4064,8 +4065,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
         return this.deserializeNoun(noun)
       }
     } catch (error) {
-      // Entity not found
-      return null
+      // A real storage/deserialize fault is NOT "entity not found":
+      // readCanonicalObject returns null for genuine absence and only throws on
+      // a real fault, so masking it here reports a present-but-unreadable entity
+      // as missing. Absence → null, fault → propagate loudly.
+      if (isAbsentError(error)) return null
+      throw error
     }
 
     return null
@@ -4175,8 +4180,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
         return this.deserializeVerb(verb)
       }
     } catch (error) {
-      // Entity not found
-      return null
+      // A real storage/deserialize fault is NOT "relationship not found":
+      // readCanonicalObject returns null for genuine absence and only throws on
+      // a real fault, so masking it here reports a present-but-unreadable edge
+      // as missing. Absence → null, fault → propagate loudly.
+      if (isAbsentError(error)) return null
+      throw error
     }
 
     return null
diff --git a/tests/unit/indexes/columnStore/segment-load-fault.test.ts b/tests/unit/indexes/columnStore/segment-load-fault.test.ts
new file mode 100644
index 00000000..deb0868f
--- /dev/null
+++ b/tests/unit/indexes/columnStore/segment-load-fault.test.ts
@@ -0,0 +1,128 @@
+/**
+ * @module tests/unit/indexes/columnStore/segment-load-fault
+ * @description Pattern-B acceptance for the ColumnStore (finding 4): a segment
+ * the manifest LISTS but that cannot be loaded must never be silently skipped —
+ * doing so dropped every entity in that segment out of `filter`/`rangeQuery`/
+ * `sortTopK` with no error, so a corrupt index looked like a merely short result.
+ *
+ * The three failure classes and their required behaviour:
+ *  - a real storage IO fault (EIO) PROPAGATES verbatim — a present-but-unreadable
+ *    segment is not "absent", so it must not read as an empty result;
+ *  - a manifest-listed segment with undecodable bytes throws `ColumnSegmentLoadError`;
+ *  - a manifest-listed segment with NO bytes (gone on disk) throws `ColumnSegmentLoadError`.
+ * Only genuine absence stays benign: querying a field that has no manifest at all
+ * returns empty (nothing was ever written for it) — that is not a fault.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import {
+  ColumnStore,
+  ColumnSegmentLoadError
+} from '../../../../src/indexes/columnStore/ColumnStore.js'
+import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
+import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
+
+type FaultMode = 'none' | 'io' | 'corrupt' | 'missing'
+
+/**
+ * A MemoryStorage that can fault reads of persisted column SEGMENTS only
+ * (keys/paths containing the `L0-` segment marker). Manifest and DELETED-bitmap
+ * reads pass through untouched so a fresh store still initialises normally — the
+ * fault is isolated to the exact seam finding 4 hardened.
+ */
+class FaultInjectingStorage extends MemoryStorage {
+  public faultMode: FaultMode = 'none'
+
+  private eio(): Error {
+    const e = new Error('simulated disk read fault') as Error & { code: string }
+    e.code = 'EIO'
+    return e
+  }
+
+  public async loadBinaryBlob(key: string): Promise<Buffer | null> {
+    if (this.faultMode !== 'none' && key.includes('/L0-')) {
+      if (this.faultMode === 'io') throw this.eio()
+      // Too small to hold even a header → readSegmentFromBuffer throws → wrapped.
+      if (this.faultMode === 'corrupt') return Buffer.from([1, 2, 3, 4, 5])
+      if (this.faultMode === 'missing') return null
+    }
+    return super.loadBinaryBlob(key)
+  }
+}
+
+describe('ColumnStore segment-load faults surface loudly, absence stays benign (finding 4)', () => {
+  let storage: FaultInjectingStorage
+  let idMapper: EntityIdMapper
+
+  beforeEach(async () => {
+    storage = new FaultInjectingStorage()
+    await storage.init()
+    idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
+    await idMapper.init()
+
+    // Write one persisted L0 segment for `createdAt`, then close the writer.
+    const writer = new ColumnStore({ flushThreshold: 10 })
+    await writer.init(storage, idMapper)
+    for (let i = 0; i < 5; i++) {
+      writer.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), {
+        createdAt: (i + 1) * 100
+      })
+    }
+    await writer.flush()
+    await writer.close()
+    storage.faultMode = 'none'
+  })
+
+  // Fresh reader over the same storage: empty segment cache, so every query is
+  // forced to actually load the persisted segment (that is the seam under test).
+  const reopen = async (): Promise<ColumnStore> => {
+    const s = new ColumnStore({ flushThreshold: 10 })
+    await s.init(storage, idMapper)
+    return s
+  }
+
+  it('propagates a storage IO fault verbatim — not [] and not a ColumnSegmentLoadError', async () => {
+    storage.faultMode = 'io'
+    const store = await reopen()
+    await expect(store.filter('createdAt', 300)).rejects.toMatchObject({
+      code: 'EIO'
+    })
+    await store.close()
+  })
+
+  it('throws ColumnSegmentLoadError when a manifest-listed segment is undecodable', async () => {
+    storage.faultMode = 'corrupt'
+    const store = await reopen()
+    await expect(
+      store.sortTopK('createdAt', 'desc', 10)
+    ).rejects.toBeInstanceOf(ColumnSegmentLoadError)
+    await store.close()
+  })
+
+  it('throws ColumnSegmentLoadError when a manifest-listed segment has no loadable bytes', async () => {
+    storage.faultMode = 'missing'
+    const store = await reopen()
+    await expect(
+      store.rangeQuery('createdAt', 100, 500)
+    ).rejects.toBeInstanceOf(ColumnSegmentLoadError)
+    await store.close()
+  })
+
+  it('a field with no manifest is genuine absence — returns empty, never throws', async () => {
+    storage.faultMode = 'none'
+    const store = await reopen()
+    const bitmap = await store.filter('no_such_field', 'x')
+    expect(bitmap.size).toBe(0)
+    const sorted = await store.sortTopK('no_such_field', 'asc', 10)
+    expect(sorted).toEqual([])
+    await store.close()
+  })
+
+  it('with no fault, the persisted segment still loads and answers queries', async () => {
+    storage.faultMode = 'none'
+    const store = await reopen()
+    const sorted = await store.sortTopK('createdAt', 'desc', 10)
+    const uuids = sorted.map((id) => idMapper.getUuid(Number(id)))
+    expect(uuids).toEqual(['e4', 'e3', 'e2', 'e1', 'e0'])
+    await store.close()
+  })
+})

From d8301f8d08a5325e50b9a72c3c3abf10ef314812 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 09:18:36 -0700
Subject: [PATCH 20/82] fix: clear() wipes the full native/derived footprint,
 not a subset
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

clear() removed entities, indexes, system and _cas but left three top-level
trees on disk: _blobs (raw HNSW/LSM segment bytes and the native dkann index),
_id_mapper (the native shared mmap id-mapper), and _column_index (column-store
manifests). A cleared brain therefore re-read stale native state.

Worse, the column store splits its state across two of those trees — manifests
under _column_index/ and their segment bytes under _blobs/_column_index/ — so
removing one without the other stranded a manifest listing segments that no
longer exist, which the hardened segment-load path now (correctly) refuses with
ColumnSegmentLoadError. The three trees now fall together as a set, exactly as
_cas already does. locks/ is deliberately preserved: it is live coordination
state, not data.
---
 src/storage/adapters/fileSystemStorage.ts     | 18 +++++
 .../storage/clear-native-footprint.test.ts    | 74 +++++++++++++++++++
 2 files changed, 92 insertions(+)
 create mode 100644 tests/unit/storage/clear-native-footprint.test.ts

diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index 27f1af89..7e341ac5 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -1349,6 +1349,24 @@ export class FileSystemStorage extends BaseStorage {
       await fs.promises.rm(casDir, { recursive: true, force: true })
     }
 
+    // Remove the raw-blob + native-shared + column-index footprint. These
+    // top-level trees were NOT wiped before, so a cleared brain re-read stale
+    // native blobs (HNSW/LSM segments, the native dkann index), a stale native
+    // id-mapper, and — worst — orphaned column manifests. `_column_index` holds
+    // the column-store MANIFEST.json files while their segment bytes live under
+    // `_blobs/_column_index/...`; removing one without the other would strand a
+    // manifest listing segments that no longer exist, which the load path now
+    // (correctly) refuses with ColumnSegmentLoadError. They must fall together,
+    // as a set, exactly as `_cas` does — the complete derived footprint, not a
+    // subset. (`locks/` is deliberately left: it is live coordination state,
+    // not data.)
+    for (const nativeDir of ['_blobs', '_id_mapper', '_column_index']) {
+      const dir = path.join(this.rootDir, nativeDir)
+      if (await this.directoryExists(dir)) {
+        await fs.promises.rm(dir, { recursive: true, force: true })
+      }
+    }
+
     // Reset ALL shared derived state via the same path restore uses: the
     // write-through cache, id→type/subtype caches, per-type/subtype count
     // rollups, statistics cache, graph-index singleton, and the BlobStorage
diff --git a/tests/unit/storage/clear-native-footprint.test.ts b/tests/unit/storage/clear-native-footprint.test.ts
new file mode 100644
index 00000000..313ed9b8
--- /dev/null
+++ b/tests/unit/storage/clear-native-footprint.test.ts
@@ -0,0 +1,74 @@
+/**
+ * @module tests/unit/storage/clear-native-footprint
+ * @description clear() must remove the COMPLETE derived/native on-disk footprint
+ * (finding 7). Before the fix it wiped only entities/indexes/system/_cas and left
+ * three top-level trees behind:
+ *   - `_blobs`        raw binary blobs (HNSW/LSM segments, the native dkann index,
+ *                     and column-store segment bytes under `_blobs/_column_index/`)
+ *   - `_id_mapper`    the native shared mmap id-mapper
+ *   - `_column_index` the column-store MANIFEST.json files
+ * A cleared brain therefore re-read stale native state, and — worst — an orphaned
+ * column manifest whose segment bytes had been removed with `_blobs`, which the
+ * hardened load path now refuses with ColumnSegmentLoadError. The three must fall
+ * together as a set.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
+
+const exists = (p: string): boolean => fs.existsSync(p)
+
+describe('FileSystemStorage.clear() wipes the full native/derived footprint (finding 7)', () => {
+  let dir: string
+  let storage: any
+
+  beforeEach(async () => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-clear-footprint-'))
+    storage = new FileSystemStorage(dir)
+    await storage.init()
+  })
+
+  afterEach(() => {
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('removes _blobs, _id_mapper and _column_index (previously left stranded)', async () => {
+    // A raw blob (this is how HNSW/LSM/column segments persist) → creates _blobs.
+    await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([1, 2, 3, 4]))
+    // A column segment blob lives UNDER _blobs/_column_index/...
+    await storage.saveBinaryBlob('_column_index/createdAt/L0-000000', Buffer.from([5, 6, 7, 8]))
+
+    // Simulate the native id-mapper mmap dir and the column manifest object-tree.
+    const idMapperDir = path.join(dir, '_id_mapper')
+    fs.mkdirSync(idMapperDir, { recursive: true })
+    fs.writeFileSync(path.join(idMapperDir, 'main.slotmap'), Buffer.from([0, 1]))
+    const colDir = path.join(dir, '_column_index', 'createdAt')
+    fs.mkdirSync(colDir, { recursive: true })
+    fs.writeFileSync(
+      path.join(colDir, 'MANIFEST.json'),
+      JSON.stringify({ segments: [{ id: 0, level: 0, count: 1 }] })
+    )
+
+    // Sanity: everything is present before the clear.
+    expect(exists(path.join(dir, '_blobs'))).toBe(true)
+    expect(exists(idMapperDir)).toBe(true)
+    expect(exists(path.join(dir, '_column_index'))).toBe(true)
+
+    await storage.clear()
+
+    // The complete derived footprint is gone — no stale native state, and no
+    // orphaned column manifest pointing at removed segment bytes.
+    expect(exists(path.join(dir, '_blobs'))).toBe(false)
+    expect(exists(idMapperDir)).toBe(false)
+    expect(exists(path.join(dir, '_column_index'))).toBe(false)
+  })
+
+  it('clear() is a no-op-safe when the native dirs never existed', async () => {
+    // Fresh store, no blobs written — clear must not throw on absent dirs.
+    await expect(storage.clear()).resolves.toBeUndefined()
+    expect(exists(path.join(dir, '_blobs'))).toBe(false)
+    expect(exists(path.join(dir, '_column_index'))).toBe(false)
+  })
+})

From 54c183668cd672d06f1b11d3fd58d4682b114602 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 09:31:25 -0700
Subject: [PATCH 21/82] fix: refuse writes when single-op history cannot be
 made durable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The async group-commit flush that persists single-op generation history
swallowed persist failures as a bare warn: writes kept succeeding while their
before-images piled up in memory, never durable and unbounded, with no signal.

The generation store now accounts for every failed flush at one place
(flushPendingSingleOps), tolerates a transient blip (retry with capped
exponential backoff), and after PENDING_FLUSH_FAILURE_THRESHOLD consecutive
failures LATCHES a durability failure and refuses further single-op and transact
writes with a typed, exported PendingFlushDurabilityError — rather than promise a
durability it cannot deliver. Live canonical data is untouched; only the
immutable history is stuck. The latch self-heals: the moment a flush succeeds
(a retry, or an explicit flush()/close()) it lifts and writes resume.

Loud errors, never quiet losses.
---
 src/db/errors.ts                              |  59 ++++++++
 src/db/generationStore.ts                     | 143 +++++++++++++++++-
 src/index.ts                                  |   3 +-
 .../unit/db/pending-flush-durability.test.ts  | 106 +++++++++++++
 4 files changed, 302 insertions(+), 9 deletions(-)
 create mode 100644 tests/unit/db/pending-flush-durability.test.ts

diff --git a/src/db/errors.ts b/src/db/errors.ts
index 7cf9c3d0..22f405be 100644
--- a/src/db/errors.ts
+++ b/src/db/errors.ts
@@ -230,3 +230,62 @@ export class StoreInconsistentError extends Error {
     this.cause = cause
   }
 }
+
+/**
+ * @description Thrown by a write when the store cannot make single-op
+ * generation **history** durable: the asynchronous group-commit flush that
+ * persists buffered before-images to disk has failed repeatedly (a real
+ * storage fault — a full disk, an EIO, a permission loss — not a transient
+ * blip). Rather than keep accepting writes whose history silently piles up in
+ * memory and is never persisted — an invisible durability loss, and an
+ * unbounded leak — the store LATCHES this failure and refuses further writes
+ * until the pending tier drains.
+ *
+ * This is the loud, honest response to a broken history-durability path:
+ * - Live canonical data is unaffected (single-op live bytes are written before
+ *   the history flush; only the immutable before-image history is stuck).
+ * - The background flush keeps retrying with backoff; when the underlying fault
+ *   clears and a flush succeeds, the latch lifts and writes resume
+ *   automatically. An explicit `flush()` also clears it on success.
+ * - {@link cause} is the underlying storage error from the last failed flush.
+ *
+ * A caller seeing this should treat it exactly like a full disk: stop writing,
+ * resolve the storage fault, and retry. It is NOT a data-corruption error — no
+ * committed generation is lost — it is a refusal to *promise* durability the
+ * store currently cannot deliver.
+ *
+ * @example
+ * try {
+ *   await brain.add({ ... })
+ * } catch (err) {
+ *   if (err instanceof PendingFlushDurabilityError) {
+ *     // History can't be persisted right now (disk fault). Resolve storage,
+ *     // then retry — the store self-heals once a flush succeeds.
+ *     console.error('history durability stalled:', err.cause)
+ *   }
+ * }
+ */
+export class PendingFlushDurabilityError extends Error {
+  /** The underlying storage error from the most recent failed flush. */
+  public override readonly cause: Error
+  /** How many consecutive flush attempts had failed when the latch tripped. */
+  public readonly failedAttempts: number
+
+  /**
+   * @param cause - The storage error from the last failed pending-tier flush.
+   * @param failedAttempts - Consecutive failed flush attempts at latch time.
+   */
+  constructor(cause: Error, failedAttempts: number) {
+    super(
+      `Write refused: single-op generation history could not be made durable ` +
+        `after ${failedAttempts} consecutive flush attempts (${cause.message}). ` +
+        `Live data is intact, but buffered history is not yet persisted — the ` +
+        `store refuses further writes rather than silently accumulate ` +
+        `un-durable history. Resolve the underlying storage fault; the store ` +
+        `self-heals and resumes writes once a flush succeeds. Cause: ${cause.message}`
+    )
+    this.name = 'PendingFlushDurabilityError'
+    this.cause = cause
+    this.failedAttempts = failedAttempts
+  }
+}
diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts
index 5492ed70..d7a302c9 100644
--- a/src/db/generationStore.ts
+++ b/src/db/generationStore.ts
@@ -32,7 +32,7 @@
  */
 
 import { prodLog } from '../utils/logger.js'
-import { GenerationCompactedError, GenerationConflictError, StoreInconsistentError } from './errors.js'
+import { GenerationCompactedError, GenerationConflictError, PendingFlushDurabilityError, StoreInconsistentError } from './errors.js'
 import type { UnreconciledRecord } from './errors.js'
 import { TransactionRollbackError } from '../transaction/errors.js'
 import type {
@@ -284,6 +284,26 @@ export class GenerationStore {
   /** Coalescing window (ms) before an idle pending tier is flushed. */
   private static readonly PENDING_FLUSH_DELAY_MS = 50
 
+  /**
+   * Latched pending-flush durability failure. Set once background flushes have
+   * failed {@link PENDING_FLUSH_FAILURE_THRESHOLD} consecutive times (a real,
+   * persistent storage fault — not a blip); while set, writes are REFUSED with
+   * a {@link PendingFlushDurabilityError} rather than silently accumulate
+   * un-durable history in memory. Cleared the moment a flush finally succeeds
+   * (the store self-heals). `null` = history-durability path is healthy.
+   */
+  private pendingFlushError: Error | null = null
+  /** Consecutive failed background pending-flush attempts (resets on success). */
+  private pendingFlushFailures = 0
+  /**
+   * Consecutive failed flush attempts before the durability latch trips and
+   * writes start refusing. A small tolerance so a single transient fault does
+   * not trip the alarm, while a genuinely stuck disk stops the bleed fast.
+   */
+  private static readonly PENDING_FLUSH_FAILURE_THRESHOLD = 3
+  /** Upper bound (ms) on the exponential retry backoff between failed flushes. */
+  private static readonly PENDING_FLUSH_RETRY_CAP_MS = 30_000
+
   /** Live pin refcounts, keyed by pinned generation. */
   private readonly pins = new Map<number, number>()
 
@@ -592,6 +612,11 @@ export class GenerationStore {
     execute: () => Promise<void>
   }): Promise<{ generation: number; timestamp: number }> {
     return this.withMutex(async () => {
+      // A latched history-durability failure compromises the whole generation
+      // spine — refuse a transact too (advancing the manifest past stuck,
+      // un-durable single-op generations would be inconsistent). Same loud
+      // error; self-clears when the pending tier drains.
+      this.assertHistoryDurable()
       if (args.ifAtGeneration !== undefined && args.ifAtGeneration !== this.counter) {
         throw new GenerationConflictError(args.ifAtGeneration, this.counter)
       }
@@ -904,6 +929,11 @@ export class GenerationStore {
     precommit?: (before: CommitBeforeImages) => void
   }): Promise<{ generation: number; timestamp: number; degraded?: string[] }> {
     return this.withMutex(async () => {
+      // Refuse to accept a write whose history we cannot make durable: if the
+      // pending-tier flush has latched a persistent failure, accepting more
+      // writes would silently pile un-durable before-images into memory. Fail
+      // loud instead (the latch self-clears when a flush finally succeeds).
+      this.assertHistoryDurable()
       const nouns = args.touched.nouns ? [...new Set(args.touched.nouns)] : []
       const verbs = args.touched.verbs ? [...new Set(args.touched.verbs)] : []
       const gen = ++this.counter
@@ -1030,6 +1060,22 @@ export class GenerationStore {
    */
   async flushPendingSingleOps(): Promise<void> {
     if (this.pendingGens.length === 0) return
+    // Centralize durability accounting here so EVERY failure path — the
+    // background scheduler AND an explicit flush()/close() — latches
+    // consistently, and success (from any caller) clears the latch. The
+    // scheduler owns only the retry cadence.
+    try {
+      await this.flushPendingSingleOpsUnlocked()
+      this.onPendingFlushSuccess()
+    } catch (err) {
+      this.recordPendingFlushFailure(err as Error)
+      throw err
+    }
+  }
+
+  /** The actual pending-tier flush under the mutex (see {@link flushPendingSingleOps},
+   *  which wraps this with durability accounting). */
+  private async flushPendingSingleOpsUnlocked(): Promise<void> {
     return this.withMutex(async () => {
       if (this.pendingGens.length === 0) return
       this.clearPendingFlushTimer()
@@ -1140,24 +1186,105 @@ export class GenerationStore {
     })
   }
 
+  /**
+   * @description Reset the durability-failure state after a successful flush.
+   * If a failure was latched (writes were being refused), log the recovery so
+   * the transition out of the loud state is as visible as the transition in.
+   */
+  private onPendingFlushSuccess(): void {
+    if (this.pendingFlushError) {
+      prodLog.info(
+        `[GenerationStore] pending single-op history flush recovered after ` +
+          `${this.pendingFlushFailures} failed attempt(s); writes resume.`
+      )
+    }
+    this.pendingFlushFailures = 0
+    this.pendingFlushError = null
+  }
+
+  /**
+   * @description Account for a failed pending-tier flush (called from
+   * {@link flushPendingSingleOps}' catch, so it fires on EVERY failure path —
+   * background scheduler and explicit flush()/close() alike). Escalates a
+   * transient blip (a warn, keep retrying) into a latched durability failure
+   * once {@link PENDING_FLUSH_FAILURE_THRESHOLD} consecutive attempts fail —
+   * from that point writes are refused with a {@link PendingFlushDurabilityError}
+   * until a flush finally succeeds. Always ensures a backoff retry is scheduled
+   * so a recovering disk self-heals without operator action, regardless of who
+   * triggered the failing flush. The pending before-images are retained (the
+   * failed flush never cleared them), so no history is lost — only not-yet-durable.
+   */
+  private recordPendingFlushFailure(err: Error): void {
+    this.pendingFlushFailures++
+    const attempts = this.pendingFlushFailures
+    if (attempts >= GenerationStore.PENDING_FLUSH_FAILURE_THRESHOLD) {
+      // Latch: refuse further writes rather than pile un-durable history.
+      this.pendingFlushError = err
+      prodLog.error(
+        `[GenerationStore] pending single-op history flush has FAILED ${attempts} ` +
+          `consecutive times (${err.message}). Generation history is not durable; ` +
+          `writes are now REFUSED until it drains. Live canonical data is intact. ` +
+          `Retrying with backoff; the store resumes writes when a flush succeeds.`
+      )
+    } else {
+      prodLog.warn(
+        `[GenerationStore] pending single-op flush failed (attempt ${attempts}/` +
+          `${GenerationStore.PENDING_FLUSH_FAILURE_THRESHOLD}): ${err.message}; retrying with backoff.`
+      )
+    }
+    this.scheduleFlushRetry(attempts)
+  }
+
+  /**
+   * @description (Re)arm the pending-flush timer at a capped exponential
+   * backoff after a failure, so a persistent fault is retried at a decaying
+   * cadence instead of hot-looping. Shares the single {@link pendingFlushTimer}
+   * slot with {@link schedulePendingFlush} (only one flush is ever scheduled).
+   * The retry's own failure is re-accounted inside {@link flushPendingSingleOps}.
+   */
+  private scheduleFlushRetry(attempt: number): void {
+    if (this.pendingFlushTimer !== null) return
+    const backoff = Math.min(
+      GenerationStore.PENDING_FLUSH_DELAY_MS * 2 ** Math.min(attempt, 6),
+      GenerationStore.PENDING_FLUSH_RETRY_CAP_MS
+    )
+    this.pendingFlushTimer = setTimeout(() => {
+      this.pendingFlushTimer = null
+      // flushPendingSingleOps records the failure + reschedules internally.
+      void this.flushPendingSingleOps().catch(() => {})
+    }, backoff)
+    const t = this.pendingFlushTimer as unknown as { unref?: () => void }
+    if (t && typeof t.unref === 'function') t.unref()
+  }
+
+  /**
+   * @description Throw if the store has a latched history-durability failure.
+   * Called at the top of every write commit so a broken durability path fails
+   * loud and immediately instead of silently accumulating un-durable history.
+   */
+  private assertHistoryDurable(): void {
+    if (this.pendingFlushError) {
+      throw new PendingFlushDurabilityError(this.pendingFlushError, this.pendingFlushFailures)
+    }
+  }
+
   /** Schedule a coalesced pending-tier flush (size trigger fires immediately on
    *  the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both
-   *  defer outside the current mutex section so the flush can re-acquire it. */
+   *  defer outside the current mutex section so the flush can re-acquire it. A
+   *  failed flush is accounted + rescheduled inside {@link flushPendingSingleOps}
+   *  (latch-on-persistent-failure), never swallowed as a bare log line. */
   private schedulePendingFlush(): void {
     if (this.pendingGens.length >= this.pendingFlushThreshold) {
+      // Failure is recorded + retried inside flushPendingSingleOps.
       void Promise.resolve()
         .then(() => this.flushPendingSingleOps())
-        .catch((err) =>
-          prodLog.warn(`[GenerationStore] pending single-op flush failed: ${(err as Error).message}`)
-        )
+        .catch(() => {})
       return
     }
     if (this.pendingFlushTimer !== null) return
     this.pendingFlushTimer = setTimeout(() => {
       this.pendingFlushTimer = null
-      void this.flushPendingSingleOps().catch((err) =>
-        prodLog.warn(`[GenerationStore] pending single-op flush failed: ${(err as Error).message}`)
-      )
+      void this.flushPendingSingleOps().catch(() => {})
     }, GenerationStore.PENDING_FLUSH_DELAY_MS)
     // A background flush must not keep the process alive (Node only).
     const t = this.pendingFlushTimer as unknown as { unref?: () => void }
diff --git a/src/index.ts b/src/index.ts
index 3915025d..63a07690 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -175,7 +175,8 @@ export {
   GenerationConflictError,
   SpeculativeOverlayError,
   GenerationCompactedError,
-  StoreInconsistentError
+  StoreInconsistentError,
+  PendingFlushDurabilityError
 } from './db/errors.js'
 export type { UnreconciledRecord } from './db/errors.js'
 export type {
diff --git a/tests/unit/db/pending-flush-durability.test.ts b/tests/unit/db/pending-flush-durability.test.ts
new file mode 100644
index 00000000..f8501c46
--- /dev/null
+++ b/tests/unit/db/pending-flush-durability.test.ts
@@ -0,0 +1,106 @@
+/**
+ * @module tests/unit/db/pending-flush-durability
+ * @description Finding 8: the async group-commit flush that persists single-op
+ * generation HISTORY must not swallow persist failures. Before the fix a failed
+ * background flush was a bare `prodLog.warn` — writes kept succeeding while their
+ * before-images silently piled up in memory, never durable and unbounded.
+ *
+ * The contract now:
+ *  - a single transient flush failure does NOT trip the alarm (tolerance), and
+ *  - after PENDING_FLUSH_FAILURE_THRESHOLD consecutive failures the store LATCHES
+ *    and REFUSES further writes with a typed PendingFlushDurabilityError rather
+ *    than accumulate un-durable history, and
+ *  - it self-heals: once a flush finally succeeds the latch lifts and writes resume.
+ * Live canonical data is never touched — only the immutable history is stuck.
+ *
+ * Fake timers keep the background retry/coalesce timers from firing, so the exact
+ * number of flush attempts (and thus the latch point) is deterministic — the test
+ * drives every flush explicitly.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+import { GenerationStore } from '../../../src/db/generationStore.js'
+import { PendingFlushDurabilityError } from '../../../src/db/errors.js'
+import { NounType } from '../../../src/types/graphTypes.js'
+
+const ID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}`
+const meta = (v: number): Record<string, unknown> => ({
+  noun: NounType.Document,
+  subtype: 'note',
+  data: `payload-v${v}`,
+  version: v,
+  _rev: v
+})
+
+describe('db/GenerationStore pending-flush durability (finding 8)', () => {
+  let storage: MemoryStorage
+  let store: GenerationStore
+
+  beforeEach(async () => {
+    vi.useFakeTimers()
+    storage = new MemoryStorage()
+    await storage.init()
+    store = new GenerationStore(storage)
+    await store.open()
+    // High size threshold so single-ops never auto-flush — the test owns every flush.
+    ;(store as any).pendingFlushThreshold = 1000
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  /** One single-op write that buffers a pending generation. */
+  const singleOp = (id: string, v: number) =>
+    store.commitSingleOp({
+      touched: { nouns: [id], verbs: [] },
+      execute: async () => {
+        await storage.saveNounMetadata(id, meta(v))
+      }
+    })
+
+  it('latches after repeated flush failures, refuses writes, then self-heals', async () => {
+    // Buffer a pending generation while storage is healthy.
+    await singleOp(ID('aa'), 1)
+
+    // Fault the HISTORY flush (raw generation-record writes) — NOT the live
+    // canonical write, which uses a different path (saveNounMetadata).
+    const fault = Object.assign(new Error('EIO simulated'), { code: 'EIO' })
+    const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(fault)
+
+    // Each explicit flush fails and is accounted; the third trips the latch.
+    for (let i = 0; i < 3; i++) {
+      await expect(store.flushPendingSingleOps()).rejects.toThrow('EIO simulated')
+    }
+
+    // Writes are now refused LOUDLY rather than piling up un-durable history.
+    await expect(singleOp(ID('bb'), 1)).rejects.toBeInstanceOf(PendingFlushDurabilityError)
+    // The live counter did not advance for the refused write (no generation consumed).
+    expect(store.generation()).toBe(1)
+
+    // Storage recovers → an explicit flush drains the retained tier and lifts the latch.
+    spy.mockRestore()
+    await expect(store.flushPendingSingleOps()).resolves.toBeUndefined()
+
+    // Writes resume; the next single-op commits normally.
+    const receipt = await singleOp(ID('cc'), 1)
+    expect(receipt.generation).toBe(2)
+  })
+
+  it('a single transient flush failure does NOT latch — writes continue', async () => {
+    await singleOp(ID('aa'), 1)
+
+    const spy = vi.spyOn(storage as any, 'writeRawObject').mockRejectedValue(new Error('blip'))
+    await expect(store.flushPendingSingleOps()).rejects.toThrow('blip') // failure #1, below threshold
+    spy.mockRestore()
+
+    // Below the latch threshold → the store still accepts writes.
+    const receipt = await singleOp(ID('bb'), 1)
+    expect(receipt.generation).toBe(2)
+
+    // A subsequent healthy flush drains the tier and resets the failure counter.
+    await expect(store.flushPendingSingleOps()).resolves.toBeUndefined()
+    expect((store as any).pendingFlushFailures).toBe(0)
+    expect((store as any).pendingFlushError).toBeNull()
+  })
+})

From 7feba49d94f72eaae7e1c1dc4f0456017dbf473d Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 09:35:30 -0700
Subject: [PATCH 22/82] fix: saveBinaryBlob never acks a durable write that
 stored nothing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With the unique per-writer temp suffix, a rename ENOENT can no longer mean "a
concurrent idempotent writer already renamed it" — nobody else holds this
writer's temp. It means our just-written temp vanished before the rename, so the
bytes did NOT land. The old code returned success on that ENOENT, acknowledging
a write that persisted nothing; the native provider mmaps these blobs, so a
phantom-acked blob is a silent-loss.

saveBinaryBlob now retries once with a fresh temp (self-healing a transient
external-sweeper/crash-cleanup race), and if the temp vanishes again it throws
loud rather than acknowledging a store-nothing write. Non-ENOENT rename faults
propagate verbatim as before. The unique-temp suffix already eliminated the
concurrent same-key collision that originally motivated the ENOENT shortcut.
---
 src/storage/adapters/fileSystemStorage.ts     | 59 +++++++++-----
 .../unit/storage/blob-save-durability.test.ts | 76 +++++++++++++++++++
 2 files changed, 114 insertions(+), 21 deletions(-)
 create mode 100644 tests/unit/storage/blob-save-durability.test.ts

diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index 7e341ac5..95d3951d 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -1178,27 +1178,44 @@ export class FileSystemStorage extends BaseStorage {
     await this.ensureInitialized()
     const filePath = this.blobPath(key)
     await this.ensureDirectoryExists(path.dirname(filePath))
-    // Unique per-writer temp suffix — matches the pattern used at every other
-    // atomic-write site in this file (lines 336, 551, 744, 781, 1529, 2908).
-    // Without a unique suffix, two concurrent saveBinaryBlob() calls for the
-    // same key collide on `${filePath}.tmp`: both writeFile, the first rename
-    // succeeds, the second fires against a missing temp and throws ENOENT.
-    // Reproduced in production: column-store compaction running alongside an
-    // explicit flush() repeatedly raced on `_column_index/<field>/DELETED.bin`.
-    const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}`
-    await fs.promises.writeFile(tmpPath, data)
-    try {
-      await fs.promises.rename(tmpPath, filePath)
-    } catch (err) {
-      const code = (err as NodeJS.ErrnoException).code
-      // The writes are idempotent for a given key — every caller persists the
-      // same logical bytes for that key — so ENOENT on rename means the temp
-      // is already gone (rare with the unique suffix, but defensive against
-      // crash-resume cleanup paths and external file-system sweepers).
-      if (code === 'ENOENT') return
-      // Any other failure: clean up our own temp to avoid orphans before rethrow.
-      await fs.promises.unlink(tmpPath).catch(() => {})
-      throw err
+
+    // Atomic write via a UNIQUE per-writer temp suffix — matches the pattern
+    // used at every other atomic-write site in this file (lines 336, 551, 744,
+    // 781, 1529, 2908). The unique suffix (pid+time+random) means NO other
+    // writer ever touches this temp, so two concurrent saveBinaryBlob() calls
+    // for the same key can never collide on the temp path (the shared-`.tmp`
+    // race that once threw ENOENT — column-store compaction vs an explicit
+    // flush() on `_column_index/<field>/DELETED.bin` — is gone).
+    //
+    // Crucially, BECAUSE the temp is unique, a rename ENOENT can no longer mean
+    // "a concurrent idempotent writer already renamed it": nobody else has this
+    // temp. It means OUR just-written temp vanished before the rename, so the
+    // bytes did NOT land — returning success would acknowledge a write that
+    // stored nothing (the native provider mmaps these blobs; a phantom-acked
+    // blob is exactly the silent-loss class). The only way that happens with a
+    // unique temp is an external sweeper / crash-cleanup removing it mid-write,
+    // so retry ONCE with a fresh temp; if it vanishes again, FAIL LOUD.
+    const writeOnce = async (): Promise<'ok' | 'temp-vanished'> => {
+      const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}`
+      await fs.promises.writeFile(tmpPath, data)
+      try {
+        await fs.promises.rename(tmpPath, filePath)
+        return 'ok'
+      } catch (err) {
+        // Clean up our own temp (best-effort) so no failure path orphans it.
+        await fs.promises.unlink(tmpPath).catch(() => {})
+        if ((err as NodeJS.ErrnoException).code === 'ENOENT') return 'temp-vanished'
+        throw err
+      }
+    }
+
+    if ((await writeOnce()) === 'temp-vanished' && (await writeOnce()) === 'temp-vanished') {
+      throw new Error(
+        `saveBinaryBlob('${key}'): the temp file was removed before rename on two ` +
+          `successive attempts — the blob did NOT persist. Some external process is ` +
+          `deleting files under ${this.blobsDir} mid-write. Failing loud rather than ` +
+          `acknowledging a durable write that stored nothing.`
+      )
     }
   }
 
diff --git a/tests/unit/storage/blob-save-durability.test.ts b/tests/unit/storage/blob-save-durability.test.ts
new file mode 100644
index 00000000..3c4b0a1a
--- /dev/null
+++ b/tests/unit/storage/blob-save-durability.test.ts
@@ -0,0 +1,76 @@
+/**
+ * @module tests/unit/storage/blob-save-durability
+ * @description Finding 13 (cortex sweep): saveBinaryBlob must never acknowledge a
+ * durable write that stored nothing. With the UNIQUE per-writer temp suffix, a
+ * rename ENOENT can no longer mean "a concurrent idempotent writer already
+ * renamed it" — nobody else has this writer's temp — so it means our temp
+ * vanished before the rename and the bytes did NOT land. The old code returned
+ * success on that ENOENT; the native provider mmaps these blobs, so a
+ * phantom-acked blob is the exact silent-loss class the registered-blob work
+ * chases. The fix retries once with a fresh temp, then fails loud.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
+
+const enoent = (): Error => Object.assign(new Error('temp removed'), { code: 'ENOENT' })
+
+describe('FileSystemStorage.saveBinaryBlob durable-write honesty (finding 13)', () => {
+  let dir: string
+  let storage: any
+
+  beforeEach(async () => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-blob-save-'))
+    storage = new FileSystemStorage(dir)
+    await storage.init()
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('persists and reads back a blob under normal conditions', async () => {
+    await storage.saveBinaryBlob('graph-lsm/seg-1', Buffer.from([1, 2, 3, 4]))
+    expect(await storage.loadBinaryBlob('graph-lsm/seg-1')).toEqual(Buffer.from([1, 2, 3, 4]))
+  })
+
+  it('throws (never acks) when the temp vanishes on every rename attempt', async () => {
+    vi.spyOn(fs.promises, 'rename').mockRejectedValue(enoent())
+
+    await expect(
+      storage.saveBinaryBlob('k/silent-loss', Buffer.from([9]))
+    ).rejects.toThrow(/did NOT persist/)
+
+    // Prove the write truly did not land — and was not acknowledged. (loadBinaryBlob
+    // uses readFile, not rename, so it reflects the real on-disk state.)
+    expect(await storage.loadBinaryBlob('k/silent-loss')).toBeNull()
+  })
+
+  it('self-heals: a single transient temp-vanish retries and persists', async () => {
+    const real = fs.promises.rename.bind(fs.promises)
+    let calls = 0
+    vi.spyOn(fs.promises, 'rename').mockImplementation(async (from: any, to: any) => {
+      calls++
+      if (calls === 1) throw enoent()
+      return real(from, to)
+    })
+
+    await expect(
+      storage.saveBinaryBlob('k/heals', Buffer.from([7, 7]))
+    ).resolves.toBeUndefined()
+    expect(calls).toBe(2) // first attempt vanished, retry landed
+    expect(await storage.loadBinaryBlob('k/heals')).toEqual(Buffer.from([7, 7]))
+  })
+
+  it('a non-ENOENT rename fault propagates verbatim (not swallowed)', async () => {
+    vi.spyOn(fs.promises, 'rename').mockRejectedValue(
+      Object.assign(new Error('disk fault'), { code: 'EIO' })
+    )
+    await expect(
+      storage.saveBinaryBlob('k/eio', Buffer.from([1]))
+    ).rejects.toMatchObject({ code: 'EIO' })
+  })
+})

From ba958d97b5ca9c17c7a979a6ebc073cf752f67d8 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 09:49:38 -0700
Subject: [PATCH 23/82] fix: surface a degraded derived index on reads instead
 of serving it silently
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two known-degraded states were recorded but never consulted by the read paths, so
a partial result looked authoritative:

- commitSingleOp returns `degraded` ids on an adopt-forward failed-rollback
  recovery (the canonical record is durable but its derived-index entry may be
  incomplete). persistSingleOp dropped that list on the floor — no health flag,
  no read signal. It now records them in a queryable degraded set.
- A non-fatal index-rebuild failure at init (_indexRebuildFailed) was folded into
  checkHealth()/validateIndexConsistency() but no read consulted it.

find() and get() now emit ONE loud warning per degraded window (reads still
return — canonical is the source of truth — but the caller is told results may be
partial and to run repairIndex()). Both degraded sources fold into the two health
surfaces, and repairIndex() reconciles from canonical and clears them.
---
 src/brainy.ts                                 | 108 +++++++++++++++++-
 .../brainy/degraded-reads-surfaced.test.ts    |  79 +++++++++++++
 2 files changed, 184 insertions(+), 3 deletions(-)
 create mode 100644 tests/unit/brainy/degraded-reads-surfaced.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index da44c252..32b8757b 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -478,6 +478,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * during rebuild is NOT recorded here — it re-throws and aborts init() loudly.
    */
   private _indexRebuildFailed: Error | null = null
+  /**
+   * Ids of records that committed via an adopt-forward failed-rollback recovery
+   * ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is
+   * durable but its derived index entry may be incomplete until
+   * {@link repairIndex}. Non-empty = a queryable degraded state, folded into
+   * {@link checkHealth}/{@link validateIndexConsistency} and warned on the read
+   * paths. Cleared by {@link repairIndex}.
+   */
+  private _indexDegradedIds: Set<string> = new Set()
+  /** One-shot guard so the degraded-reads warning fires once per degraded window
+   *  (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */
+  private _degradedReadWarned = false
   /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */
   private _metadataConsistencyProbed = false
   /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */
@@ -1608,7 +1620,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     run: TransactionFunction<void>,
     precommit?: (before: CommitBeforeImages) => void,
     pendingEvents?: PendingChangeEvent[]
-  ): Promise<{ generation?: number; timestamp: number }> {
+  ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> {
     // Change-feed capture: when this write will emit, hold a reference to the
     // commit's before-images so `remove` events can carry the record's last
     // committed state (free — the commit reads them anyway for Model B).
@@ -1667,6 +1679,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // that did not become durable. (A degraded adopt-forward write DID commit —
     // it carries a generation and emits normally.)
     this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp)
+    // An adopt-forward failed-rollback recovery committed the record but may have
+    // left its derived index incomplete (generationStore already warned once).
+    // Record the ids so reads and checkHealth() surface the incompleteness rather
+    // than silently returning partial data; repairIndex() clears them.
+    if (receipt.degraded && receipt.degraded.length > 0) {
+      for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId)
+      this._degradedReadWarned = false // re-arm the read-path warning
+      if (!this.config.silent) {
+        prodLog.warn(
+          `[Brainy] A single-op write committed in a DEGRADED state — the derived ` +
+          `index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` +
+          `Reads may return partial results until repairIndex() reconciles them.`
+        )
+      }
+    }
     return receipt
   }
 
@@ -2157,6 +2184,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    */
   async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
     await this.ensureInitialized()
+    this.warnIfReadsDegraded('get')
 
     // Id normalization (8.0): a caller may read by their natural key — resolve
     // it to the same canonical UUID add() stored. A real UUID passes through.
@@ -5562,8 +5590,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider)
     // Note: init() is async but definitions can be registered synchronously.
     // State loading happens lazily on first query.
-    this._aggregationIndex.init().catch(() => {
-      // Non-fatal — aggregation state will be empty but definitions still work
+    this._aggregationIndex.init().catch((err) => {
+      // Non-fatal — definitions still work and state backfills on first query —
+      // but a failed state load means aggregates read empty/stale until then, so
+      // surface it loudly rather than swallow it.
+      if (!this.config.silent) {
+        prodLog.warn(
+          `[Brainy] Aggregation index state failed to load at init ` +
+          `(${(err as Error).message}). Aggregates may read empty until a ` +
+          `backfill-on-query repopulates them.`
+        )
+      }
     })
   }
 
@@ -5643,6 +5680,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // metadata counterpart of the graph cold-load guard. No-op for the JS index.
     await this.ensureMetadataConsistencyProbed()
 
+    // Loudly flag a degraded derived index (failed init rebuild, or an
+    // adopt-forward degraded commit) so a partial result is never mistaken for
+    // authoritative. The streaming search() generator delegates to find(), so it
+    // is covered here.
+    this.warnIfReadsDegraded('find')
+
     // Parse natural language queries
     let params: FindParams<T> =
       typeof query === 'string' ? await this.parseNaturalQuery(query) : query
@@ -12318,6 +12361,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           (result.recommendation ? ` Also: ${result.recommendation}` : '')
       }
     }
+    if (this._indexDegradedIds.size > 0) {
+      return {
+        ...result,
+        healthy: false,
+        recommendation:
+          `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
+          `failed-rollback recovery — the derived index may be incomplete for them. ` +
+          `Run repairIndex() to reconcile.` +
+          (result.recommendation ? ` Also: ${result.recommendation}` : '')
+      }
+    }
     return result
   }
 
@@ -14746,6 +14800,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           (result.recommendation ? ` Also: ${result.recommendation}` : '')
       }
     }
+    if (this._indexDegradedIds.size > 0) {
+      return {
+        ...result,
+        healthy: false,
+        recommendation:
+          `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
+          `failed-rollback recovery — the derived index may be incomplete for them. ` +
+          `Run repairIndex() to reconcile.` +
+          (result.recommendation ? ` Also: ${result.recommendation}` : '')
+      }
+    }
     return result
   }
 
@@ -14800,6 +14865,34 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * genuinely lost record cannot be resurrected here (restore from a snapshot for
    * that), but the store is made internally consistent and writes re-enabled.
    */
+  /**
+   * Emit ONE loud warning per degraded window when a read is served while the
+   * derived index is known-incomplete — either a non-fatal init rebuild failure
+   * ({@link _indexRebuildFailed}) or an adopt-forward degraded commit
+   * ({@link _indexDegradedIds}). Reads still return (canonical is the source of
+   * truth), but the caller is told the result may be partial and how to heal it.
+   * No-op when healthy or when the brain is configured `silent`.
+   * @param op - The read method name for the message (e.g. `'find'`, `'get'`).
+   */
+  private warnIfReadsDegraded(op: string): void {
+    const degraded =
+      this._indexRebuildFailed !== null || this._indexDegradedIds.size > 0
+    if (!degraded) {
+      this._degradedReadWarned = false
+      return
+    }
+    if (this._degradedReadWarned || this.config.silent) return
+    this._degradedReadWarned = true
+    const reason = this._indexRebuildFailed
+      ? `index rebuild failed at init() (${this._indexRebuildFailed.message})`
+      : `${this._indexDegradedIds.size} record(s) committed in a degraded state`
+    prodLog.warn(
+      `[Brainy] ${op}() is serving reads while the derived index is INCOMPLETE ` +
+      `(${reason}). Results may be missing entries — run repairIndex() to ` +
+      `reconcile the derived indexes against canonical storage.`
+    )
+  }
+
   async repairIndex(): Promise<void> {
     await this.ensureInitialized()
     await this.metadataIndex.detectAndRepairCorruption()
@@ -14816,6 +14909,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         `Writes are re-enabled.`
       )
     }
+    // detectAndRepairCorruption() above rebuilt the derived indexes from
+    // canonical, so any adopt-forward degraded ids and a non-fatal init
+    // rebuild failure are now reconciled — clear the queryable degraded state
+    // and re-arm the read-path warning.
+    if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) {
+      this._indexDegradedIds.clear()
+      this._indexRebuildFailed = null
+      this._degradedReadWarned = false
+    }
   }
 
   /**
diff --git a/tests/unit/brainy/degraded-reads-surfaced.test.ts b/tests/unit/brainy/degraded-reads-surfaced.test.ts
new file mode 100644
index 00000000..29a8a77c
--- /dev/null
+++ b/tests/unit/brainy/degraded-reads-surfaced.test.ts
@@ -0,0 +1,79 @@
+/**
+ * @module tests/unit/brainy/degraded-reads-surfaced
+ * @description Finding 10: a known-degraded derived index must be SURFACED, not
+ * silently served as authoritative. Two degraded sources:
+ *   (a) a non-fatal index rebuild failure at init() (`_indexRebuildFailed`), and
+ *   (b) an adopt-forward failed-rollback recovery (`commitSingleOp`'s `degraded`
+ *       ids, previously dropped on the floor by persistSingleOp).
+ * Both now fold into checkHealth()/validateIndexConsistency() and emit ONE loud
+ * read-path warning per degraded window; repairIndex() reconciles + clears them.
+ *
+ * The private fields are the observable contract of the fix, so the test drives
+ * them directly (a real failed rollback is exercised elsewhere).
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../../src/brainy.js'
+import { NounType } from '../../../src/types/graphTypes.js'
+import { prodLog } from '../../../src/utils/logger.js'
+
+const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}`
+
+describe('Finding 10 — degraded derived-index state is surfaced on reads', () => {
+  beforeEach(() => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+  })
+  afterEach(() => vi.restoreAllMocks())
+
+  it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => {
+    const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
+    await brain.init()
+    ;(brain as any)._indexDegradedIds.add(UUID('de'))
+
+    const health = await brain.checkHealth()
+    expect(health.healthy).toBe(false)
+    expect(health.recommendation).toMatch(/repairIndex\(\)/)
+  })
+
+  it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => {
+    const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
+    const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
+    await brain.init()
+    await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document })
+    ;(brain as any)._indexRebuildFailed = new Error('rebuild boom')
+
+    await brain.find({ type: NounType.Document })
+    await brain.get(UUID('a1')) // second read: must NOT double-warn
+    const degradedWarns = warn.mock.calls.filter((c) =>
+      String(c[0]).includes('derived index is INCOMPLETE')
+    )
+    expect(degradedWarns.length).toBe(1)
+
+    await brain.repairIndex()
+    expect((brain as any)._indexRebuildFailed).toBeNull()
+    expect((brain as any)._indexDegradedIds.size).toBe(0)
+
+    warn.mockClear()
+    await brain.find({ type: NounType.Document })
+    expect(warn.mock.calls.filter((c) => String(c[0]).includes('INCOMPLETE')).length).toBe(0)
+  })
+
+  it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => {
+    const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false })
+    await brain.init()
+    // Simulate a degraded receipt by wrapping the generation store's commitSingleOp.
+    const gs: any = (brain as any).generationStore
+    const realCommit = gs.commitSingleOp.bind(gs)
+    vi.spyOn(gs, 'commitSingleOp').mockImplementation(async (args: any) => {
+      const r = await realCommit(args)
+      return { ...r, degraded: [...(args.touched.nouns ?? [])] }
+    })
+
+    await brain.add({ id: UUID('b2'), data: 'y', type: NounType.Document })
+    expect((brain as any)._indexDegradedIds.size).toBeGreaterThan(0)
+    expect((brain as any)._indexDegradedIds.has(UUID('b2'))).toBe(true)
+
+    const health = await brain.checkHealth()
+    expect(health.healthy).toBe(false)
+    expect(health.recommendation).toMatch(/repairIndex\(\)/)
+  })
+})

From 02eff64b780e251331d85823e1d301b234b303f2 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 09:49:38 -0700
Subject: [PATCH 24/82] fix: aggregation surfaces materialize/state-load
 failures loudly

The debounced materialization caught its failure with an empty `catch (() => {})`,
silently leaving the materialized Measurement entity stale; the aggregation-index
init() state-load failure was swallowed the same way, leaving aggregates reading
empty with no signal. Both are non-fatal (values rebuild via backfill-on-query),
but a silent stale/empty read violates loud-errors-never-quiet-losses. Both now
emit a loud warning naming the affected group / cause.
---
 src/aggregation/materializer.ts               | 12 ++++-
 .../materialize-failure-loud.test.ts          | 51 +++++++++++++++++++
 2 files changed, 61 insertions(+), 2 deletions(-)
 create mode 100644 tests/unit/aggregation/materialize-failure-loud.test.ts

diff --git a/src/aggregation/materializer.ts b/src/aggregation/materializer.ts
index 3a8dcb3f..f752648a 100644
--- a/src/aggregation/materializer.ts
+++ b/src/aggregation/materializer.ts
@@ -14,6 +14,7 @@ import type {
   AggregateMetricDef
 } from '../types/brainy.types.js'
 import { serializeGroupKey } from './AggregationIndex.js'
+import { prodLog } from '../utils/logger.js'
 
 /**
  * Callback interface for the materializer to create/update Brainy entities.
@@ -86,8 +87,15 @@ export class AggregateMaterializer {
     if (existing) clearTimeout(existing)
 
     this.debounceTimers.set(key, setTimeout(() => {
-      this.materializeOne(key).catch(() => {
-        // Non-fatal — materialization is derived data
+      this.materializeOne(key).catch((err) => {
+        // Materialization is derived data (rebuildable via backfill-on-query), so
+        // a failure is non-fatal — but it leaves the materialized Measurement
+        // entity STALE. Surface it loudly rather than swallow it silently.
+        prodLog.warn(
+          `[Aggregation] Failed to materialize aggregate group '${key}': ` +
+          `${(err as Error).message}. The materialized value is stale until the ` +
+          `next successful materialization or a backfill-on-query.`
+        )
       })
     }, debounceMs))
   }
diff --git a/tests/unit/aggregation/materialize-failure-loud.test.ts b/tests/unit/aggregation/materialize-failure-loud.test.ts
new file mode 100644
index 00000000..9aa2d1cb
--- /dev/null
+++ b/tests/unit/aggregation/materialize-failure-loud.test.ts
@@ -0,0 +1,51 @@
+/**
+ * @module tests/unit/aggregation/materialize-failure-loud
+ * @description Aggregation (Pattern C): a failed materialization must not be
+ * swallowed. `scheduleMaterialize`'s debounced `materializeOne(key).catch(() => {})`
+ * silently discarded the error, leaving the materialized Measurement entity stale
+ * with no signal. It now emits a loud warning (the value is still rebuildable via
+ * backfill-on-query, so it stays non-fatal — but visible). Load-bearing assertion:
+ * prodLog.warn fires instead of an empty catch.
+ */
+import { describe, it, expect, vi, afterEach } from 'vitest'
+import { AggregateMaterializer } from '../../../src/aggregation/materializer.js'
+import { prodLog } from '../../../src/utils/logger.js'
+import { NounType } from '../../../src/types/graphTypes.js'
+
+describe('Aggregation — swallowed materialize failure is surfaced', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+    vi.useRealTimers()
+  })
+
+  it('logs loudly when materializeOne rejects', async () => {
+    vi.useFakeTimers()
+    const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {})
+
+    // Brain access whose add()/update() always reject → doMaterialize throws.
+    const brain = {
+      add: vi.fn().mockRejectedValue(new Error('storage down')),
+      update: vi.fn().mockRejectedValue(new Error('storage down'))
+    }
+    const m = new AggregateMaterializer(brain as any, 10)
+    const def: any = {
+      name: 'sales',
+      source: { type: NounType.Document },
+      groupBy: ['region'],
+      metrics: { total: { op: 'count' } },
+      materialize: { debounceMs: 10 }
+    }
+    m.scheduleMaterialize(
+      'sales',
+      def,
+      { region: 'us' },
+      { groupKey: { region: 'us' }, metrics: { total: { count: 1 } } } as any
+    )
+
+    await vi.advanceTimersByTimeAsync(20) // fire the debounce timer
+    await Promise.resolve()
+
+    expect(warn).toHaveBeenCalled()
+    expect(String(warn.mock.calls[0][0])).toMatch(/Failed to materialize aggregate group/)
+  })
+})

From 36c10c1e205955ac1a45ac2604368f85b5a5cfa6 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 10:44:11 -0700
Subject: [PATCH 25/82] chore: hold loadBinaryBlob fault-propagation for the
 cortex column-store lockstep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Pass-1 hardening made loadBinaryBlob distinguish genuine absence (ENOENT →
null) from a real fault (EIO/EACCES/… → throw), so a present-but-unreadable blob
stops masquerading as "absent". The native column-store still has two call sites
that rely on the old null-on-error contract; publishing the throw ahead of them
would break a consumer running new brainy + old cortex.

Per the cortex coordination (accept: cor first, then brainy), this temporarily
restores the shipped 8.2.5 swallow-on-fault behavior for loadBinaryBlob ONLY, so
the rest of Pass 1 — saveBinaryBlob write-honesty, clear() native wipe,
pending-flush durability, count symmetry, degraded surfacing, aggregation
loudness — can release now. The method carries an inline restore guide; the
throwing form goes back in and ships lockstep with the native hardening. No other
Pass-1 change depends on this behavior (ColumnStore's own fault test uses a
direct-throwing storage stub), so the split is behaviour-neutral for every
consumer versus 8.2.5.
---
 src/storage/adapters/fileSystemStorage.ts | 28 +++++++++++++++--------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index 95d3951d..f676d091 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -17,7 +17,6 @@ import {
   WriterLockInfo
 } from '../baseStorage.js'
 import { getBrainyVersion } from '../../utils/index.js'
-import { isAbsentError } from '../../utils/errorClassification.js'
 
 // Node.js modules - dynamically imported to avoid issues in browser environments
 let fs: any
@@ -1229,14 +1228,25 @@ export class FileSystemStorage extends BaseStorage {
     await this.ensureInitialized()
     try {
       return await fs.promises.readFile(this.blobPath(key))
-    } catch (err) {
-      // Absent blob → null (the documented contract). A real fault
-      // (EIO/EACCES/EMFILE/…) must NOT be masked as "absent": doing so makes a
-      // present-but-unreadable blob look missing and drives a needless rebuild
-      // or an empty read (the native provider consumes this). Mandate: loud
-      // errors, never quiet losses.
-      if (isAbsentError(err)) return null
-      throw err
+    } catch {
+      // COORDINATION HOLD — restore the fault-propagating form lockstep with
+      // cortex 3.0.13 (CORTEX-BILLION-SCALE-SPINE). The correct behavior
+      // distinguishes genuine absence from a real fault so a present-but-
+      // unreadable blob no longer masquerades as "absent" (which drove needless
+      // rebuilds / empty reads). Restore to:
+      //     } catch (err) {
+      //       // re-add: import { isAbsentError } from '../../utils/errorClassification.js'
+      //       if (isAbsentError(err)) return null
+      //       throw err
+      //     }
+      // The throw is HELD because the native column-store still has 2 call sites
+      // that rely on null-on-error; cortex hardens them in 3.0.13, then this
+      // reverts to the throwing form and ships lockstep. Until then this
+      // preserves the shipped 8.2.5 swallow-on-fault behavior, so a consumer on
+      // new-brainy + old-cortex can never break. All other Pass-1 hardening
+      // (saveBinaryBlob honesty, clear(), pending-flush durability, count
+      // symmetry, degraded surfacing, aggregation) ships now, independent of this.
+      return null
     }
   }
 

From a873852e613f477fa1510a2dd86fe1f5d41a841e Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 10:45:39 -0700
Subject: [PATCH 26/82] docs: RELEASES.md entry for 8.2.6 (write/index-spine
 hardening)

Consumer-facing summary of the Pass-1 durability + integrity release: durable
blob-write honesty, single-op history durability (PendingFlushDurabilityError),
degraded-index surfacing, full-footprint clear(), count symmetry, and loud
index-maintenance / aggregation failures. loadBinaryBlob fault-propagation is
held for the native lockstep and deliberately omitted here.
---
 RELEASES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index a6bc927e..f67cb452 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,53 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.6 — 2026-07-13 (write/index-spine hardening — loud errors, never quiet losses)
+
+Durability + integrity hardening across the write and index paths. Every fix converts a place that
+could *silently* lose data, serve a partial result, or acknowledge a write that did not land into a
+loud, observable failure. No breaking API changes; one new exported error.
+
+- **A durable blob write that stored nothing is no longer acknowledged.** The atomic blob write
+  (`saveBinaryBlob`, used for vector/graph index segments and the native index files) uses a unique
+  per-writer temp file, so a rename `ENOENT` can only mean *our* temp vanished before the rename —
+  the bytes never landed. It previously returned success on that path; it now retries once with a
+  fresh temp and, if the temp vanishes again, throws instead of acknowledging a write that persisted
+  nothing. A downstream deployment that mmaps these files no longer finds a "successfully written"
+  blob missing on the next open.
+
+- **Single-op history durability is enforced, not assumed.** The asynchronous group-commit flush
+  that persists single-op generation history previously swallowed a persist failure as a warning
+  while writes kept succeeding and their history piled up, undurable, in memory. It now tolerates a
+  transient blip (retry with capped backoff) and, after repeated failures, **refuses further writes**
+  with the new **`PendingFlushDurabilityError`** rather than promise a durability it cannot deliver.
+  Live canonical data is untouched; the latch self-heals the moment a flush succeeds. Callers needing
+  hard per-write durability should keep using `transact()` (or `flush()` after a single-op).
+
+- **A degraded derived index is surfaced on reads, not served silently.** When a non-fatal index
+  rebuild fails at open, or a write commits via adopt-forward recovery with an incomplete derived
+  index, `find()` and `get()` now emit one loud warning per degraded window (reads still return —
+  canonical is the source of truth), the state folds into `checkHealth()` /
+  `validateIndexConsistency()`, and `repairIndex()` reconciles and clears it.
+
+- **`clear()` removes the full derived footprint.** It previously left raw index blobs, the native
+  id-mapper, and column-index manifests on disk, so a cleared brain could re-read stale native state
+  on reopen. It now wipes them together as a set.
+
+- **Counts stay honest across deletes.** A delete decremented the per-type breakdown but not the
+  scalar total, so the total inflated permanently (and won pagination). Delete now decrements both;
+  the invariant `total === Σ per-type` holds across any interleaving of add / visibility-flip /
+  delete and across a reopen.
+
+- **Index-maintenance and aggregation failures are loud.** A partial LSM/segment load no longer
+  publishes the manifest's full count as if healthy; an HNSW flush that can't persist a node throws
+  (`HnswFlushError`) instead of returning a lying count and dropping the node; a corrupt or missing
+  manifest-listed column segment throws (`ColumnSegmentLoadError`) instead of silently dropping its
+  entities from every query; and aggregation materialization / state-load failures now warn instead
+  of vanishing into an empty catch.
+
+New export: **`PendingFlushDurabilityError`** (with `.cause` and `.failedAttempts`). No other public
+API change. Each fix ships with a dedicated regression test.
+
 ## v8.2.5 — 2026-07-12 (honest response when a transaction rollback can't complete)
 
 Data-integrity fix. When a transaction failed and its rollback then *also* failed to undo a

From 76843b782e8ea54aa0346b5a35f227a887be2e49 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 10:59:55 -0700
Subject: [PATCH 27/82] chore(release): 8.2.6

---
 CHANGELOG.md      | 14 ++++++++++++++
 package-lock.json |  4 ++--
 package.json      |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8602bf02..1b89e4df 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,20 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.6](https://github.com/soulcraftlabs/brainy/compare/v8.2.5...v8.2.6) (2026-07-13)
+
+- docs: RELEASES.md entry for 8.2.6 (write/index-spine hardening) (a873852)
+- chore: hold loadBinaryBlob fault-propagation for the cortex column-store lockstep (36c10c1)
+- fix: aggregation surfaces materialize/state-load failures loudly (02eff64)
+- fix: surface a degraded derived index on reads instead of serving it silently (ba958d9)
+- fix: saveBinaryBlob never acks a durable write that stored nothing (7feba49)
+- fix: refuse writes when single-op history cannot be made durable (54c1836)
+- fix: clear() wipes the full native/derived footprint, not a subset (d8301f8)
+- fix: surface segment/entity read faults loudly instead of masking as absent (af5d2f3)
+- fix: spine hardening pass 1 (part) — count symmetry, honest partial-load, flush durability, read-fault propagation (119087a)
+- test: pin the read-your-writes contract under the single writer (eb9c4eb)
+
+
 ### [8.2.5](https://github.com/soulcraftlabs/brainy/compare/v8.2.4...v8.2.5) (2026-07-12)
 
 - docs: RELEASES.md entry for 8.2.5 (honest rollback-failure response) (a7c7aa5)
diff --git a/package-lock.json b/package-lock.json
index 7968654e..84693b23 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.5",
+  "version": "8.2.6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.5",
+      "version": "8.2.6",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index d5c4e740..3206fc73 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.5",
+  "version": "8.2.6",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From b6c70397693a3b8101d87bbc273d6f6258d5c32e Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 12:09:55 -0700
Subject: [PATCH 28/82] fix: restore loadBinaryBlob fault-propagation (native
 column-store lockstep)

The Pass-1 hardening that makes loadBinaryBlob distinguish genuine absence
(ENOENT -> null) from a real fault (EIO/EACCES/... -> throw) was held out of
8.2.6 because the native accelerator's two column-store read sites still relied
on null-on-error. That accelerator release has now hardened those sites to
handle the throw (a faulted segment read marks the field unavailable and throws
a named error), so the fault-propagating form is restored and ships lockstep.

A present-but-unreadable index blob no longer masquerades as absent (which drove
needless rebuilds / empty reads). Adds the loadBinaryBlob leg to the blob
durability suite (absent -> null; present -> bytes; real fault -> throws).
---
 RELEASES.md                                   | 18 ++++++++++
 src/storage/adapters/fileSystemStorage.ts     | 31 +++++++---------
 .../unit/storage/blob-save-durability.test.ts | 35 +++++++++++++++++++
 3 files changed, 65 insertions(+), 19 deletions(-)

diff --git a/RELEASES.md b/RELEASES.md
index f67cb452..759fcfeb 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,24 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.7 — 2026-07-13 (loadBinaryBlob fault-propagation — the lockstep completion of 8.2.6)
+
+Completes the Pass-1 spine hardening. `loadBinaryBlob` (the raw-blob read a native accelerator mmaps
+for its index files) previously returned `null` on ANY read error, so a real IO fault
+(EIO/EACCES/EMFILE) on a present-but-unreadable index blob masqueraded as "the blob is absent" —
+driving a needless full rebuild or an empty read. It now distinguishes genuine absence (ENOENT →
+`null`, the documented contract) from a real fault (→ throw), so a transient disk fault surfaces
+loudly instead of silently degrading the index.
+
+This one change was deliberately held out of 8.2.6 and ships now, in lockstep with the native
+accelerator release that hardened its two column-store read sites to handle the throw (a faulted
+segment read marks the field unavailable and throws a named error, instead of relying on
+null-on-error). A consumer on new brainy + an older accelerator was never at risk: 8.2.6 kept the
+prior swallow-on-fault behavior for this method until the accelerator was ready.
+
+No API change. Regression: `tests/unit/storage/blob-save-durability.test.ts` gains the loadBinaryBlob
+leg (absent → null; present → bytes; real fault → throws).
+
 ## v8.2.6 — 2026-07-13 (write/index-spine hardening — loud errors, never quiet losses)
 
 Durability + integrity hardening across the write and index paths. Every fix converts a place that
diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index f676d091..9ff0539e 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -17,6 +17,7 @@ import {
   WriterLockInfo
 } from '../baseStorage.js'
 import { getBrainyVersion } from '../../utils/index.js'
+import { isAbsentError } from '../../utils/errorClassification.js'
 
 // Node.js modules - dynamically imported to avoid issues in browser environments
 let fs: any
@@ -1228,25 +1229,17 @@ export class FileSystemStorage extends BaseStorage {
     await this.ensureInitialized()
     try {
       return await fs.promises.readFile(this.blobPath(key))
-    } catch {
-      // COORDINATION HOLD — restore the fault-propagating form lockstep with
-      // cortex 3.0.13 (CORTEX-BILLION-SCALE-SPINE). The correct behavior
-      // distinguishes genuine absence from a real fault so a present-but-
-      // unreadable blob no longer masquerades as "absent" (which drove needless
-      // rebuilds / empty reads). Restore to:
-      //     } catch (err) {
-      //       // re-add: import { isAbsentError } from '../../utils/errorClassification.js'
-      //       if (isAbsentError(err)) return null
-      //       throw err
-      //     }
-      // The throw is HELD because the native column-store still has 2 call sites
-      // that rely on null-on-error; cortex hardens them in 3.0.13, then this
-      // reverts to the throwing form and ships lockstep. Until then this
-      // preserves the shipped 8.2.5 swallow-on-fault behavior, so a consumer on
-      // new-brainy + old-cortex can never break. All other Pass-1 hardening
-      // (saveBinaryBlob honesty, clear(), pending-flush durability, count
-      // symmetry, degraded surfacing, aggregation) ships now, independent of this.
-      return null
+    } catch (err) {
+      // Absent blob → null (the documented contract). A real fault
+      // (EIO/EACCES/EMFILE/…) must NOT be masked as "absent": doing so makes a
+      // present-but-unreadable blob look missing and drives a needless rebuild
+      // or an empty read (the native provider consumes this). Mandate: loud
+      // errors, never quiet losses. Fault-propagation restored lockstep with
+      // cortex 3.0.13, whose two column-store call sites now handle the throw
+      // (they mark the field unavailable + throw a named error) instead of
+      // relying on null-on-error (CORTEX-BILLION-SCALE-SPINE).
+      if (isAbsentError(err)) return null
+      throw err
     }
   }
 
diff --git a/tests/unit/storage/blob-save-durability.test.ts b/tests/unit/storage/blob-save-durability.test.ts
index 3c4b0a1a..f775ebdf 100644
--- a/tests/unit/storage/blob-save-durability.test.ts
+++ b/tests/unit/storage/blob-save-durability.test.ts
@@ -74,3 +74,38 @@ describe('FileSystemStorage.saveBinaryBlob durable-write honesty (finding 13)',
     ).rejects.toMatchObject({ code: 'EIO' })
   })
 })
+
+describe('FileSystemStorage.loadBinaryBlob fault-propagation (finding 11; cor 3.0.13 lockstep)', () => {
+  let dir: string
+  let storage: any
+
+  beforeEach(async () => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-blob-load-'))
+    storage = new FileSystemStorage(dir)
+    await storage.init()
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('returns null for a genuinely absent blob (ENOENT)', async () => {
+    expect(await storage.loadBinaryBlob('never/written')).toBeNull()
+  })
+
+  it('reads back a present blob', async () => {
+    await storage.saveBinaryBlob('present/blob', Buffer.from([1, 2, 3]))
+    expect(await storage.loadBinaryBlob('present/blob')).toEqual(Buffer.from([1, 2, 3]))
+  })
+
+  it('propagates a real IO fault instead of masking it as absent', async () => {
+    await storage.saveBinaryBlob('present/blob', Buffer.from([1, 2, 3]))
+    vi.spyOn(fs.promises, 'readFile').mockRejectedValue(
+      Object.assign(new Error('disk fault'), { code: 'EIO' })
+    )
+    // A present-but-unreadable blob must NOT read as null (that drove needless
+    // native rebuilds / empty reads); cor 3.0.13's column-store handles the throw.
+    await expect(storage.loadBinaryBlob('present/blob')).rejects.toMatchObject({ code: 'EIO' })
+  })
+})

From 36d4e80ba22b60f693a10a3d5fc947e1421ca409 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 12:18:26 -0700
Subject: [PATCH 29/82] chore(release): 8.2.7

---
 CHANGELOG.md      | 5 +++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1b89e4df..7d53ac05 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.7](https://github.com/soulcraftlabs/brainy/compare/v8.2.6...v8.2.7) (2026-07-13)
+
+- fix: restore loadBinaryBlob fault-propagation (native column-store lockstep) (b6c7039)
+
+
 ### [8.2.6](https://github.com/soulcraftlabs/brainy/compare/v8.2.5...v8.2.6) (2026-07-13)
 
 - docs: RELEASES.md entry for 8.2.6 (write/index-spine hardening) (a873852)
diff --git a/package-lock.json b/package-lock.json
index 84693b23..d81b67e9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.6",
+  "version": "8.2.7",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.6",
+      "version": "8.2.7",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 3206fc73..5d7ed63d 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.6",
+  "version": "8.2.7",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From d0f69c731f3fa87146c5284d81af9b2e18f5cfa3 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 13:08:52 -0700
Subject: [PATCH 30/82] =?UTF-8?q?fix:=20honest=20index=20readiness=20?=
 =?UTF-8?q?=E2=80=94=20no=20silently-empty=20queries=20on=20a=20cold=20ind?=
 =?UTF-8?q?ex?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes the "dishonest readiness proxy" anti-pattern (Pattern A): size()>0 /
isInitialized were treated as "this index serves queries", but a cold native
index can load its COUNT before its SERVING structure, so a query returned a
silent [] indistinguishable from "no such data". A shared assessIndexReadiness()
now reads only the provider's honest isReady() signal (never size()), applied at
every site:

- Vector: a one-shot verifyVectorLive() guard on the semantic/proximity search
  path (a pure semantic find({query}) has no filter, so nothing guarded it). It
  prefers isReady(), else a known-vector self-match probe; self-heals via rebuild
  or throws the new VectorIndexNotReadyError instead of a silent [].
- Graph: getVerbsBySource/ByTarget skip the fast path when the provider reports
  not-ready (falling to the canonical shard scan), plus a one-shot probe that
  self-heals a no-isReady provider whose adjacency did not cold-load.
- getIndexStatus(): folds in per-index honest `ready` (making `populated`
  honest) + rebuildFailed/rebuildError/degradedIds, so a readiness probe never
  200s a brain that is still warming up or degraded.

Unblocked by the native providers now reporting serving-truth (graph via
SSTable-residency readiness, vector via durableBaseLoadFailed). New export:
VectorIndexNotReadyError. getIndexStatus gains additive fields. No breaking API.
13 new tests; existing readiness guards green.
---
 RELEASES.md                                   |  33 +++
 src/brainy.ts                                 | 215 +++++++++++++++++-
 src/errors/brainyError.ts                     |  27 +++
 src/index.ts                                  |   2 +-
 src/storage/baseStorage.ts                    |  79 ++++++-
 src/utils/indexReadiness.ts                   |  38 ++++
 tests/unit/get-index-status-readiness.test.ts |  59 +++++
 .../graph-fastpath-honest-readiness.test.ts   |  95 ++++++++
 tests/unit/vector-cold-read-guard.test.ts     | 115 ++++++++++
 9 files changed, 654 insertions(+), 9 deletions(-)
 create mode 100644 src/utils/indexReadiness.ts
 create mode 100644 tests/unit/get-index-status-readiness.test.ts
 create mode 100644 tests/unit/graph/graph-fastpath-honest-readiness.test.ts
 create mode 100644 tests/unit/vector-cold-read-guard.test.ts

diff --git a/RELEASES.md b/RELEASES.md
index 759fcfeb..a90dcce4 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,39 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.2.8 — 2026-07-13 (honest index readiness — no more silently-empty queries on a cold index)
+
+Closes the last of the three spine anti-patterns: the "dishonest readiness proxy," where `size() > 0`
+was treated as "this index actually serves queries." On a cold open (fresh boot, restart, crash
+recovery) a native index can load its **count** before its **serving structure** — so for a brief
+window it has data but cannot answer, and a query returned a silent empty result indistinguishable
+from "no such data." Every fix here asks the index whether it can *actually serve*, and if not,
+self-heals or fails loudly instead of returning `[]`.
+
+- **Semantic search no longer returns a silent `[]` on a cold vector index.** A pure semantic
+  `find({ query })` has no filter, so nothing previously guarded the vector index. A new one-shot
+  guard verifies the vector index serves a known persisted vector on the first semantic/proximity
+  search — preferring the provider's honest `isReady()` signal, else a known-vector self-match probe.
+  It rebuilds from canonical records if the serving structure did not load, and throws the new
+  **`VectorIndexNotReadyError`** only if a rebuild still cannot serve — never a silent empty result.
+
+- **Relationship reads fall back to the canonical scan instead of an empty result on a cold graph
+  index.** `getVerbsBySource`/`getVerbsByTarget` (used by relationship queries and virtual-filesystem
+  traversal) skipped the fast path only on `isInitialized` — which reads true once the manifest loaded
+  even if the source→target adjacency did not. They now consult the honest readiness signal and, when
+  the adjacency is not serving, take the correct-but-slower canonical shard scan. A one-shot self-heal
+  probe covers providers that expose no readiness signal.
+
+- **`getIndexStatus()` tells the truth for readiness probes.** It reported `populated: size>0` only, so
+  a Kubernetes readiness check could route traffic to a brain still warming up. It now folds in the
+  honest per-index `ready` signal (making `populated` honest), plus the degraded states already
+  surfaced by `checkHealth()`/`validateIndexConsistency()` (`rebuildFailed`/`rebuildError` and a
+  `degradedIds` count) — so a probe never reports 200-ready over a known-degraded index.
+
+This completes the write/index-spine hardening end to end. New export: **`VectorIndexNotReadyError`**;
+`getIndexStatus()` gains additive fields (`rebuildFailed`, `rebuildError?`, `degradedIds`, per-index
+`ready?`). No breaking API change. Each fix ships with a dedicated regression test.
+
 ## v8.2.7 — 2026-07-13 (loadBinaryBlob fault-propagation — the lockstep completion of 8.2.6)
 
 Completes the Pass-1 spine hardening. `loadBinaryBlob` (the raw-blob read a native accelerator mmaps
diff --git a/src/brainy.ts b/src/brainy.ts
index 32b8757b..72cd6fa3 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -175,7 +175,8 @@ import {
 } from './events/changeFeed.js'
 import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js'
 import { GenerationConflictError, StoreInconsistentError } from './db/errors.js'
-import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError } from './errors/brainyError.js'
+import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js'
+import { assessIndexReadiness } from './utils/indexReadiness.js'
 import { MemoryStorage } from './storage/adapters/memoryStorage.js'
 import type {
   CompactHistoryOptions,
@@ -500,6 +501,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   private _metadataVerified = false
   /** Re-entrancy guard for {@link verifyMetadataLive}. */
   private _metadataVerifying = false
+  /** Vector-index cold-read guard: verified-serving this session (one-shot). */
+  private _vectorVerified = false
+  /** Re-entrancy guard for {@link verifyVectorLive}. */
+  private _vectorVerifying = false
   /**
    * Coordinated migration LOCK (#18): dedup guards so the "upgrading, blocking"
    * and "upgrade complete, resumed" lines each log once per migration window,
@@ -3523,6 +3528,142 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     return null
   }
 
+  /**
+   * @description The vector-index counterpart of {@link verifyGraphAdjacencyLive}
+   * / {@link verifyMetadataLive}. On a cold open a native vector provider can
+   * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded
+   * its serving structure (the mmap/DiskANN graph) — so a pure semantic
+   * `find({ query })` silently returns `[]`. A pure semantic query has
+   * `hasFilterCriteria === false`, so the metadata guard never fires; this guard
+   * closes that gap. Run one-shot on the first vector/proximity search:
+   *  - **Preferred (honest signal):** the provider exposes `isReady()`. `false`
+   *    → rebuild from storage, re-check; if still `false`, throw
+   *    {@link VectorIndexNotReadyError} rather than serving `[]`.
+   *  - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled +
+   *    hydrated) is searched against the index; if it does not self-match, the
+   *    serving structure did not load — rebuild + re-probe, else throw.
+   * Inconclusive cases (empty store, no probeable vector, `size()===0` — where
+   * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as
+   * live: never a false rebuild. A migrating provider is skipped (it owns its
+   * locked rebuild).
+   * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it.
+   */
+  private async verifyVectorLive(): Promise<'live' | 'rebuilt'> {
+    if (this._vectorVerified) return 'live'
+    // Migration LOCK (#18): a migrating provider owns its in-place rebuild.
+    if (this.providerIsMigrating(this.index)) return 'live'
+    // Re-entrancy: rebuild() can trigger reads that call back into this guard.
+    if (this._vectorVerifying) return 'live'
+    this._vectorVerifying = true
+    try {
+      // ── Strategy 1: honest isReady() signal (native provider) ──────────────
+      const readiness = assessIndexReadiness(this.index)
+      if (readiness !== 'unknown') {
+        if (readiness === 'ready') {
+          this._vectorVerified = true
+          return 'live'
+        }
+        // Not ready: the serving structure did not load on open. Rebuild.
+        if (!this.config.silent) {
+          console.warn(
+            `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` +
+              `vector index did not load on open. Rebuilding from storage…`
+          )
+        }
+        await this.index.rebuild()
+        if (assessIndexReadiness(this.index) === 'ready') {
+          this._vectorVerified = true
+          return 'rebuilt'
+        }
+        throw new VectorIndexNotReadyError(
+          `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` +
+            `proximity search cannot be served reliably for this brain (a silent empty result ` +
+            `would misrepresent existing data).`
+        )
+      }
+
+      // ── Strategy 2: known-vector probe (providers without isReady()) ───────
+      const claimed = this.index.size()
+      if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job
+
+      const probe = await this.pickVectorProbe()
+      if (!probe) {
+        // Empty store, or nothing with a probeable vector — inconclusive.
+        this._vectorVerified = true
+        return 'live'
+      }
+      const p = probe
+
+      const probeServes = async (): Promise<boolean> => {
+        // The failure mode we guard is the SILENT EMPTY result: a cold index that
+        // loaded its COUNT but not its serving structure returns `[]` for a
+        // known-present vector, while a warm index returns at least one hit. We
+        // check for a NON-EMPTY result, NOT an exact self-match — HNSW is
+        // approximate and `get()` may return a re-hydrated/normalized vector, so
+        // demanding the exact self as top-1 would false-positive on a perfectly
+        // healthy index (and wrongly rebuild → throw).
+        const hits = await this.index.search(p.vector, 1)
+        return hits.length > 0
+      }
+      void p.id // probe keyed on the vector; id retained for diagnostics only
+
+      if (await probeServes()) {
+        this._vectorVerified = true
+        return 'live' // serving structure is live — the common case
+      }
+
+      if (!this.config.silent) {
+        console.warn(
+          `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` +
+            `returns no results — the serving structure did not load on open. Rebuilding…`
+        )
+      }
+      await this.index.rebuild()
+
+      if (await probeServes()) {
+        this._vectorVerified = true
+        return 'rebuilt'
+      }
+      throw new VectorIndexNotReadyError(
+        `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` +
+          `results even after a rebuild — semantic find({ query }) cannot be served reliably ` +
+          `for this brain (a silent empty result would misrepresent existing data).`
+      )
+    } catch (err) {
+      if (err instanceof VectorIndexNotReadyError) throw err
+      // A transient probe/rebuild failure must not break the query NOR mask as
+      // "no data". Allow a re-check on the next vector read and fall through.
+      this._vectorVerified = false
+      if (!this.config.silent) {
+        console.warn(`[Brainy] Vector consistency check skipped (transient): ${err}`)
+      }
+      return 'live'
+    } finally {
+      this._vectorVerifying = false
+    }
+  }
+
+  /**
+   * @description Sample a KNOWN persisted noun and hydrate its vector, to probe
+   * the vector index with. `get()` omits vectors by default, so this passes
+   * `{ includeVectors: true }`. Samples a few (a system-only / vectorless entity
+   * must not make every open inconclusive). Returns `null` when nothing has a
+   * probeable vector.
+   */
+  private async pickVectorProbe(): Promise<{ id: string; vector: number[] } | null> {
+    const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } })
+    for (const noun of sample.items ?? []) {
+      const id = (noun as { id?: string }).id
+      if (!id) continue
+      const full = await this.get(id, { includeVectors: true })
+      const vector = (full as { vector?: number[] } | null)?.vector
+      if (Array.isArray(vector) && vector.length > 0) {
+        return { id, vector }
+      }
+    }
+    return null
+  }
+
   // -------------------------------------------------------------------------
 
   /**
@@ -10175,17 +10316,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     migrating: boolean
     /** Structured progress while {@link migrating}; absent otherwise. */
     migration?: MigrationProgress
+    /** `true` when a non-fatal index rebuild failed at init() (degraded — queries
+     *  may be incomplete). Folds in the same `_indexRebuildFailed` signal that
+     *  {@link validateIndexConsistency} / {@link checkHealth} already expose. */
+    rebuildFailed: boolean
+    /** The rebuild failure message when {@link rebuildFailed}; absent otherwise. */
+    rebuildError?: string
+    /** Count of records committed via adopt-forward failed-rollback recovery whose
+     *  derived index may be incomplete until repairIndex() (the 8.2.6 degraded
+     *  set). Non-zero = degraded — a readiness probe should not report 200-ready. */
+    degradedIds: number
     hnswIndex: {
       size: number
+      /** Honest "serving" — the provider's `isReady()` when exposed, else `size>0`. */
       populated: boolean
+      /** The provider's honest `isReady()` signal; absent when unexposed. */
+      ready?: boolean
     }
     metadataIndex: {
       entries: number
       populated: boolean
+      ready?: boolean
     }
     graphIndex: {
       relationships: number
       populated: boolean
+      ready?: boolean
     }
     storage: {
       totalEntities: number
@@ -10200,6 +10356,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         lazyRebuildCompleted: this.lazyRebuildCompleted,
         disableAutoRebuild: this.config.disableAutoRebuild || false,
         migrating: false,
+        rebuildFailed: this._indexRebuildFailed != null,
+        ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}),
+        degradedIds: this._indexDegradedIds.size,
         hnswIndex: { size: 0, populated: false },
         metadataIndex: { entries: 0, populated: false },
         graphIndex: { relationships: 0, populated: false },
@@ -10211,6 +10370,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     const hnswSize = this.index.size()
     const graphSize = await this.graphIndex.size()
 
+    // Honest readiness: when a provider exposes isReady(), it is the truth of
+    // whether the index actually SERVES (a native index can report a non-zero
+    // size/count yet not have loaded its serving structure). `populated` reflects
+    // that honest signal when present, falling back to size>0 only for providers
+    // that do not expose isReady(). Mirrors validateIndexConsistency/checkHealth,
+    // which already fold in the same signals.
+    const readyOf = (p: unknown): boolean | undefined => {
+      const r = assessIndexReadiness(p)
+      return r === 'unknown' ? undefined : r === 'ready'
+    }
+    const hnswReady = readyOf(this.index)
+    const metadataReady = readyOf(this.metadataIndex)
+    const graphReady = readyOf(this.graphIndex)
+
     // Check storage entity count
     let storageEntityCount = 0
     try {
@@ -10224,18 +10397,29 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       initialized: this.initialized,
       lazyRebuildCompleted: this.lazyRebuildCompleted,
       disableAutoRebuild: this.config.disableAutoRebuild || false,
+      // A non-fatal index-rebuild failure recorded at init(), or adopt-forward
+      // degraded ids, are degraded states (queries may be incomplete) — surface
+      // them here alongside the same signals validateIndexConsistency()/
+      // checkHealth() already expose, so a readiness probe never reports 200-ready
+      // over a known-degraded index.
+      rebuildFailed: this._indexRebuildFailed != null,
+      ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}),
+      degradedIds: this._indexDegradedIds.size,
       ...this.migrationSnapshot(),
       hnswIndex: {
         size: hnswSize,
-        populated: hnswSize > 0
+        populated: hnswReady ?? hnswSize > 0,
+        ...(hnswReady !== undefined ? { ready: hnswReady } : {})
       },
       metadataIndex: {
         entries: metadataStats.totalEntries,
-        populated: metadataStats.totalEntries > 0
+        populated: metadataReady ?? metadataStats.totalEntries > 0,
+        ...(metadataReady !== undefined ? { ready: metadataReady } : {})
       },
       graphIndex: {
         relationships: graphSize,
-        populated: graphSize > 0
+        populated: graphReady ?? graphSize > 0,
+        ...(graphReady !== undefined ? { ready: graphReady } : {})
       },
       storage: {
         totalEntities: storageEntityCount
@@ -12863,6 +13047,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     candidateIds?: string[],
     allowedIds?: OpaqueIdSet
   ): Promise<Result<T>[]> {
+    // Vector cold-read guard: before trusting a semantic/vector result, verify the
+    // vector index actually SERVES a known persisted vector (one-shot per brain).
+    // A pure semantic find({ query }) has no filter, so verifyMetadataLive never
+    // fires — a cold native index that loaded its COUNT but not its serving
+    // structure would return a silent []. This self-heals (rebuild) or throws
+    // VectorIndexNotReadyError instead.
+    await this.verifyVectorLive()
+
     const vector = params.vector || (await this.embed(params.query!))
     const limit = params.limit || 10
 
@@ -12903,6 +13095,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   private async executeProximitySearch(params: FindParams<T>): Promise<Result<T>[]> {
     if (!params.near) return []
 
+    // Vector cold-read guard (see executeVectorSearch): proximity search also
+    // hits this.index.search — verify it serves before trusting an empty result.
+    await this.verifyVectorLive()
+
     // Teaching error: without an anchor id the constraint is meaningless, and
     // letting it fall through produces an opaque storage-layer sharding error.
     if (!params.near.id) {
@@ -14157,8 +14353,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       return
     }
 
-    // If indexes already populated, mark as complete and skip
-    if (this.index.size() > 0) {
+    // If indexes already populated AND honestly serving, mark complete and skip.
+    // Honest gate: when the provider exposes isReady(), that REPLACES the size()>0
+    // proxy (a native index can report a non-zero size while its serving structure
+    // is not loaded — the silent-empty cold-load class). A not-ready provider falls
+    // through so the rebuild path can load it; verifyVectorLive() is the query-time
+    // backstop either way. Providers without isReady() keep the size() heuristic
+    // (the JS index's size()>0 genuinely means loaded).
+    const vectorReadiness = assessIndexReadiness(this.index)
+    if (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) {
       this.lazyRebuildCompleted = true
       return
     }
diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts
index f627ba00..c2667009 100644
--- a/src/errors/brainyError.ts
+++ b/src/errors/brainyError.ts
@@ -14,6 +14,7 @@ export type BrainyErrorType =
   | 'FIELD_NOT_INDEXED'
   | 'GRAPH_INDEX_NOT_READY'
   | 'METADATA_INDEX_NOT_READY'
+  | 'VECTOR_INDEX_NOT_READY'
   | 'MIGRATION_IN_PROGRESS'
 
 /**
@@ -280,6 +281,32 @@ export class MetadataIndexNotReadyError extends BrainyError {
     }
 }
 
+/**
+ * Thrown when the vector index reports vectors (`size() > 0` or, on a native
+ * provider, `isReady() === false`) but cannot return a KNOWN persisted vector
+ * even after a rebuild — i.e. the semantic serving structure did not load on a
+ * cold open and could not be restored. The vector-search counterpart of
+ * {@link GraphIndexNotReadyError} / {@link MetadataIndexNotReadyError}: it
+ * replaces the silent-empty failure mode (a cold `find({ query })` returning
+ * `[]` indistinguishable from "no similar data") with a loud, catchable error,
+ * so a consumer never renders "nothing found" over data that is simply
+ * not-yet-warm.
+ *
+ * Detected once per brain by a known-vector serving probe on the first
+ * semantic / proximity `find()`; brainy self-heals (rebuilds the index from the
+ * canonical records) first and only raises this if the rebuild still cannot
+ * serve the known vector.
+ */
+export class VectorIndexNotReadyError extends BrainyError {
+    constructor(message: string, originalError?: Error) {
+        super(message, 'VECTOR_INDEX_NOT_READY', false, originalError)
+        this.name = 'VectorIndexNotReadyError'
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, VectorIndexNotReadyError)
+        }
+    }
+}
+
 /**
  * Thrown when a data-plane read or write is issued against a brain that is
  * running its one-time, automatic 7.x → 8.0 on-disk upgrade — the coordinated
diff --git a/src/index.ts b/src/index.ts
index 63a07690..619a3470 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -150,7 +150,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js
 
 // Base error + typed migration-lock error — thrown by any data-plane call while a
 // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
-export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError } from './errors/brainyError.js'
+export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError } from './errors/brainyError.js'
 export type { BrainyErrorType } from './errors/brainyError.js'
 
 // ============= 8.0 Db API — generational MVCC =============
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 41ebeb45..159a0593 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -5,6 +5,7 @@
 
 import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js'
 import type { GraphEntityIdResolver } from '../graph/graphAdjacencyIndex.js'
+import { assessIndexReadiness } from '../utils/indexReadiness.js'
 
 import {
   GraphVerb,
@@ -248,6 +249,8 @@ function isCountedVisibility(visibility: unknown): boolean {
  */
 export abstract class BaseStorage extends BaseStorageAdapter {
   protected isInitialized = false
+  /** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
+  private _graphFastPathProbed = false
   protected graphIndex?: GraphAdjacencyIndex
   protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
   /**
@@ -2681,6 +2684,61 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
     return this.graphIndex
   }
+
+  /**
+   * @description One-shot cold-load self-heal for a graph provider that does NOT
+   * expose the honest `isReady()` signal. A native provider wired via
+   * {@link setGraphIndex} bypasses `_initializeGraphIndex`'s `size()===0`
+   * self-heal, so a cold-open that loaded the COUNT but not the source→target
+   * adjacency would let the fast path return a silent `[]`. This probe (run once,
+   * before the fast path) samples ONE known persisted edge: if the index claims
+   * relationships (`size() > 0`) yet that edge's source resolves to no verb ints,
+   * the adjacency did not load → rebuild once from storage. Providers that expose
+   * `isReady()` are covered by the honest gate in
+   * getVerbsBy{Source,Target}_internal and skip this probe; the JS index (which
+   * self-heals in `_initializeGraphIndex`) resolves its known edge and no-ops.
+   */
+  private async ensureGraphFastPathProbed(): Promise<void> {
+    if (this._graphFastPathProbed) return
+    const index = this.graphIndex
+    const resolver = this.graphEntityIdResolver
+    if (!index || !index.isInitialized || !resolver) return
+    // isReady()-capable providers report serving honestly — the gate handles them.
+    if (assessIndexReadiness(index) !== 'unknown') {
+      this._graphFastPathProbed = true
+      return
+    }
+    if (index.size() <= 0) {
+      this._graphFastPathProbed = true
+      return // no edges claimed — nothing to verify
+    }
+    try {
+      const sample = await this.getVerbs({ pagination: { limit: 1 } })
+      const verb = sample.items?.[0]
+      if (!verb || !verb.sourceId) {
+        this._graphFastPathProbed = true
+        return // no edges in storage — stale count, harmless
+      }
+      const sourceInt = resolver.getInt(verb.sourceId)
+      if (sourceInt === undefined) {
+        this._graphFastPathProbed = true
+        return // foreign / unmapped sample — inconclusive, never a false rebuild
+      }
+      const verbInts = await index.getVerbIdsBySource(BigInt(sourceInt))
+      if (verbInts.length === 0) {
+        prodLog.warn(
+          `[BaseStorage] Graph fast path: index reports ${index.size()} relationship(s) but a ` +
+            `known persisted edge resolves to none — the persisted adjacency did not load. ` +
+            `Rebuilding once from storage.`
+        )
+        await index.rebuild()
+      }
+      this._graphFastPathProbed = true
+    } catch (error) {
+      // Transient probe/rebuild failure must not break the read; re-arm for next call.
+      prodLog.debug(`[BaseStorage] Graph fast-path probe skipped (transient): ${error}`)
+    }
+  }
   /**
    * Clear all data from storage
    * This method should be implemented by each specific adapter
@@ -4199,13 +4257,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     sourceId: string
   ): Promise<HNSWVerbWithMetadata[]> {
     await this.ensureInitialized()
+    await this.ensureGraphFastPathProbed()
 
     prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`)
 
     // Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
     // 8.0 BigInt boundary: convert the UUID to an entity int up front and
     // resolve returned verb ints back to verb-id strings.
-    if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
+    // Honest gate: a provider that exposes isReady() and reports not-ready
+    // (count/manifest loaded but source→target edges NOT) is SKIPPED so we fall
+    // to the correct-but-slower canonical shard scan below instead of a silent [].
+    if (
+      this.graphIndex &&
+      this.graphIndex.isInitialized &&
+      this.graphEntityIdResolver &&
+      assessIndexReadiness(this.graphIndex) !== 'not-ready'
+    ) {
       try {
         const sourceInt = this.graphEntityIdResolver.getInt(sourceId)
         if (sourceInt === undefined) {
@@ -4424,11 +4491,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     targetId: string
   ): Promise<HNSWVerbWithMetadata[]> {
     await this.ensureInitialized()
+    await this.ensureGraphFastPathProbed()
 
     // Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
     // 8.0 BigInt boundary: convert the UUID to an entity int up front and
     // resolve returned verb ints back to verb-id strings.
-    if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
+    // Honest gate: a not-ready provider (count loaded but edges NOT) is SKIPPED so
+    // we fall to the correct-but-slower canonical shard scan instead of a silent [].
+    if (
+      this.graphIndex &&
+      this.graphIndex.isInitialized &&
+      this.graphEntityIdResolver &&
+      assessIndexReadiness(this.graphIndex) !== 'not-ready'
+    ) {
       try {
         const targetInt = this.graphEntityIdResolver.getInt(targetId)
         if (targetInt === undefined) {
diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts
new file mode 100644
index 00000000..16266bec
--- /dev/null
+++ b/src/utils/indexReadiness.ts
@@ -0,0 +1,38 @@
+/**
+ * @module indexReadiness
+ * @description The single honest-readiness classifier shared by the vector,
+ * graph and metadata index sites. It exists to kill "Pattern A" — the dishonest
+ * readiness proxy where `size() > 0` / `isInitialized` is treated as "this index
+ * actually serves queries." A cold native index that loaded its COUNT but not its
+ * SERVING structure passes those proxies and silently returns `[]`.
+ *
+ * This classifier reads ONLY the provider's OPTIONAL, honest `isReady()` signal
+ * (see {@link import('../plugin.js').VectorIndexProvider.isReady},
+ * {@link import('../plugin.js').GraphIndexProvider.isReady},
+ * {@link import('../plugin.js').MetadataIndexProvider.isReady}). It NEVER inspects
+ * `size()` or `isInitialized`. When `isReady()` is absent, callers must fall back
+ * to a KNOWN-ITEM PROBE (a real search/lookup that must return a known-present
+ * datum) before trusting an empty result — never a `size()` proxy.
+ */
+
+/** A provider that MAY expose the honest cold-load readiness signal. */
+export interface MaybeReadyProvider {
+  isReady?: () => boolean
+}
+
+/** Three-valued honest-readiness verdict. */
+export type IndexReadiness = 'ready' | 'not-ready' | 'unknown'
+
+/**
+ * @description Classify an index provider's honest readiness.
+ * @param provider - Any index provider (vector / graph / metadata) or `null`.
+ * @returns
+ *  - `'ready'`     when `isReady() === true`  (serving structure loaded — trust it);
+ *  - `'not-ready'` when `isReady() === false` (count/manifest loaded, NOT serving — rebuild);
+ *  - `'unknown'`   when the provider exposes no `isReady()` (caller must probe / keep the JS heuristic).
+ */
+export function assessIndexReadiness(provider: unknown): IndexReadiness {
+  const p = provider as MaybeReadyProvider | null | undefined
+  if (p == null || typeof p.isReady !== 'function') return 'unknown'
+  return p.isReady() ? 'ready' : 'not-ready'
+}
diff --git a/tests/unit/get-index-status-readiness.test.ts b/tests/unit/get-index-status-readiness.test.ts
new file mode 100644
index 00000000..7f82ec5d
--- /dev/null
+++ b/tests/unit/get-index-status-readiness.test.ts
@@ -0,0 +1,59 @@
+/**
+ * @module tests/unit/get-index-status-readiness
+ * @description Pattern-A / Finding 9: getIndexStatus reported `populated: size>0`
+ * and only `migrating`, so a native index that loaded its count but not its
+ * serving structure reported populated:true — a k8s readiness probe would 200 a
+ * brain that serves []. It now folds in the honest isReady() signal + the
+ * _indexRebuildFailed / _indexDegradedIds degraded states (mirroring
+ * validateIndexConsistency / checkHealth).
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { Brainy, NounType } from '../../src/index.js'
+
+describe('getIndexStatus honest readiness (Finding 9)', () => {
+  let brain: any
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
+    await brain.init()
+    await brain.add({ data: 'x', type: NounType.Concept })
+    await brain.flush()
+  })
+
+  it('a not-ready provider makes populated honest (false) and exposes ready:false', async () => {
+    brain.index.isReady = () => false // count present, serving structure NOT loaded
+    const status = await brain.getIndexStatus()
+    expect(status.hnswIndex.populated).toBe(false)
+    expect(status.hnswIndex.ready).toBe(false)
+    delete brain.index.isReady
+  })
+
+  it('a ready provider reports populated:true + ready:true', async () => {
+    brain.index.isReady = () => true
+    const status = await brain.getIndexStatus()
+    expect(status.hnswIndex.populated).toBe(true)
+    expect(status.hnswIndex.ready).toBe(true)
+    delete brain.index.isReady
+  })
+
+  it('rebuildFailed / rebuildError surface the init-degraded state', async () => {
+    brain._indexRebuildFailed = new Error('boom')
+    const status = await brain.getIndexStatus()
+    expect(status.rebuildFailed).toBe(true)
+    expect(status.rebuildError).toBe('boom')
+    brain._indexRebuildFailed = null
+  })
+
+  it('degradedIds surfaces the adopt-forward degraded set', async () => {
+    brain._indexDegradedIds.add('00000000-0000-4000-8000-0000000000de')
+    const status = await brain.getIndexStatus()
+    expect(status.degradedIds).toBe(1)
+    brain._indexDegradedIds.clear()
+  })
+
+  it('a JS-baseline provider (no isReady) omits ready and falls back to size>0', async () => {
+    const status = await brain.getIndexStatus()
+    expect(status.hnswIndex.ready).toBeUndefined()
+    expect(status.hnswIndex.populated).toBe(brain.index.size() > 0)
+  })
+})
diff --git a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts
new file mode 100644
index 00000000..46d318b4
--- /dev/null
+++ b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts
@@ -0,0 +1,95 @@
+/**
+ * @module tests/unit/graph/graph-fastpath-honest-readiness
+ * @description Pattern-A / Finding 2: getVerbsBySource/ByTarget gated the graph
+ * fast path on `graphIndex.isInitialized` (a proxy that reads true once the
+ * manifest/count loaded even if the source→target adjacency did NOT → the fast
+ * path then returned a silent []). The honest gate skips the fast path when the
+ * provider reports isReady()===false, falling to the correct canonical shard
+ * scan; and a one-shot probe self-heals a no-isReady provider whose adjacency
+ * did not cold-load.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { Brainy, NounType, VerbType } from '../../../src/index.js'
+
+describe('graph fast-path honest readiness (Finding 2)', () => {
+  let brain: any
+  let storage: any
+  let a: string
+  let b: string
+  let c: string
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
+    await brain.init()
+    a = await brain.add({ data: 'a', type: NounType.Concept })
+    b = await brain.add({ data: 'b', type: NounType.Concept })
+    c = await brain.add({ data: 'c', type: NounType.Concept })
+    await brain.relate({ from: a, to: b, type: VerbType.Contains })
+    await brain.relate({ from: a, to: c, type: VerbType.Contains })
+    await brain.flush()
+    storage = brain.storage
+    // Warm + wire the storage graph index (lazy) so we can stub it.
+    await storage.getVerbsBySource(a)
+  })
+
+  it('not-ready provider → shard scan returns the REAL edges, not a silent []', async () => {
+    const gi = storage.graphIndex
+    // Simulate a cold native provider: count/manifest loaded (isInitialized) but
+    // the source→target adjacency is NOT (isReady false; fast-path lookup empty).
+    gi.isReady = () => false
+    const origBySource = gi.getVerbIdsBySource.bind(gi)
+    gi.getVerbIdsBySource = async () => [] // cold adjacency — old fast path trusted this
+    storage._graphFastPathProbed = true // isolate the GATE (probe skips isReady-capable anyway)
+    try {
+      const verbs = await storage.getVerbsBySource(a)
+      // Honest gate skipped the cold fast path → canonical shard scan → real edges.
+      expect(verbs.length).toBe(2)
+    } finally {
+      delete gi.isReady
+      gi.getVerbIdsBySource = origBySource
+    }
+  })
+
+  it('ready provider → fast path is used and an empty result is trusted', async () => {
+    const gi = storage.graphIndex
+    gi.isReady = () => true
+    let fastPathCalls = 0
+    const origBySource = gi.getVerbIdsBySource.bind(gi)
+    gi.getVerbIdsBySource = async (...args: any[]) => { fastPathCalls++; return origBySource(...args) }
+    storage._graphFastPathProbed = true
+    try {
+      // `c` has no OUTGOING edges — a genuinely empty result via the fast path.
+      const verbs = await storage.getVerbsBySource(c)
+      expect(verbs).toEqual([])
+      expect(fastPathCalls).toBeGreaterThan(0) // fast path was taken, not a shard scan
+    } finally {
+      delete gi.isReady
+      gi.getVerbIdsBySource = origBySource
+    }
+  })
+
+  it('no-isReady provider whose adjacency did not cold-load → probe rebuilds once', async () => {
+    const gi = storage.graphIndex
+    if ('isReady' in gi) delete gi.isReady // force the "unknown" (no honest signal) path
+    // Force the probe to see a cold adjacency: a known edge resolves to no ints.
+    const origBySource = gi.getVerbIdsBySource.bind(gi)
+    let cold = true
+    gi.getVerbIdsBySource = async (...args: any[]) => (cold ? [] : origBySource(...args))
+    let rebuilds = 0
+    const origRebuild = gi.rebuild.bind(gi)
+    gi.rebuild = async (...args: any[]) => { rebuilds++; cold = false; return origRebuild(...args) }
+    storage._graphFastPathProbed = false // re-arm the one-shot probe
+    try {
+      await storage.getVerbsBySource(a)
+      expect(rebuilds).toBe(1)                    // probe detected cold adjacency + rebuilt once
+      expect(storage._graphFastPathProbed).toBe(true) // latched — no second rebuild
+      rebuilds = 0
+      await storage.getVerbsByTarget(b)
+      expect(rebuilds).toBe(0)                    // probe does not re-run
+    } finally {
+      gi.getVerbIdsBySource = origBySource
+      gi.rebuild = origRebuild
+    }
+  })
+})
diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts
new file mode 100644
index 00000000..49ca6426
--- /dev/null
+++ b/tests/unit/vector-cold-read-guard.test.ts
@@ -0,0 +1,115 @@
+/**
+ * @module tests/unit/vector-cold-read-guard
+ * @description Pattern-A / Finding 1: a pure semantic find({ query }) has no
+ * filter, so verifyMetadataLive never fires — nothing guarded the vector index.
+ * A cold native vector index that loaded its COUNT but not its serving structure
+ * returned a silent []. verifyVectorLive() closes that: honest isReady() first,
+ * else a known-vector self-match probe; self-heal (rebuild) or throw
+ * VectorIndexNotReadyError — never a silent empty result.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js'
+
+const V = (): number[] => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001)
+
+describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semantic find', () => {
+  let brain: any
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
+    await brain.init()
+    await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'active' } })
+    await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'archived' } })
+    await brain.flush()
+  })
+
+  it('warm brain: semantic find is correct and the guard does not rebuild', async () => {
+    const vi = brain.index
+    let rebuilds = 0
+    const origRebuild = vi.rebuild.bind(vi)
+    vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) }
+    await brain.find({ query: 'anything', searchMode: 'semantic', limit: 100 })
+    expect(rebuilds).toBe(0)
+    expect(brain._vectorVerified).toBe(true)
+    vi.rebuild = origRebuild
+  })
+
+  it('cold index: verifyVectorLive self-heals via rebuild — semantic find is correct, NOT silent []', async () => {
+    const vi = brain.index
+    const origSearch = vi.search.bind(vi)
+    const origRebuild = vi.rebuild.bind(vi)
+    let cold = true
+    brain._vectorVerified = false
+    // size()>0 (count present) but search returns nothing until a rebuild warms it.
+    vi.search = async (...a: any[]) => (cold ? [] : origSearch(...a))
+    vi.rebuild = async (...a: any[]) => { await origRebuild(...a); cold = false }
+    try {
+      const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 })
+      expect(res.length).toBeGreaterThan(0) // self-healed
+    } finally {
+      vi.search = origSearch; vi.rebuild = origRebuild
+    }
+  })
+
+  it('unrecoverably cold index: semantic find throws VectorIndexNotReadyError', async () => {
+    const vi = brain.index
+    const origSearch = vi.search.bind(vi)
+    const origRebuild = vi.rebuild.bind(vi)
+    brain._vectorVerified = false
+    vi.search = async () => []      // always cold; rebuild can't fix it
+    vi.rebuild = async () => {}
+    try {
+      await expect(
+        brain.find({ query: 'x', searchMode: 'semantic', limit: 100 })
+      ).rejects.toBeInstanceOf(VectorIndexNotReadyError)
+    } finally {
+      vi.search = origSearch; vi.rebuild = origRebuild
+    }
+  })
+
+  it('native provider reporting isReady()===false rebuilds, then serves', async () => {
+    const vi = brain.index
+    const origRebuild = vi.rebuild.bind(vi)
+    let ready = false
+    brain._vectorVerified = false
+    vi.isReady = () => ready
+    vi.rebuild = async (...a: any[]) => { await origRebuild(...a); ready = true }
+    try {
+      const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 })
+      expect(ready).toBe(true) // rebuild ran because isReady() was false
+      expect(res).toBeDefined()
+    } finally {
+      delete vi.isReady; vi.rebuild = origRebuild
+    }
+  })
+
+  it('a text-only query does not trigger the vector guard', async () => {
+    brain._vectorVerified = false
+    await brain.find({ query: 'active', searchMode: 'text', limit: 5 })
+    expect(brain._vectorVerified).toBe(false) // executeVectorSearch never called
+  })
+
+  // Regression: the probe must check "returns ANY hit", not an exact self-match —
+  // HNSW is approximate and get() may re-hydrate the vector, so a healthy
+  // many-entity index would false-positive under an exact-self check, wrongly
+  // rebuild, and throw VectorIndexNotReadyError on working data.
+  it('a healthy many-entity brain with distinct vectors serves semantic find, never throws/rebuilds', async () => {
+    const many = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
+    await many.init()
+    for (let i = 0; i < 25; i++) {
+      const v = Array.from({ length: 384 }, (_, j) => Math.sin((i * 7 + j) * 0.13) + 0.001)
+      await many.add({ vector: v, type: NounType.Concept, metadata: { n: i } })
+    }
+    await many.flush()
+    let rebuilds = 0
+    const vi = (many as any).index
+    const origRebuild = vi.rebuild.bind(vi)
+    vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) }
+    const res = await many.find({ query: 'x', searchMode: 'semantic', limit: 10 })
+    expect(res.length).toBeGreaterThan(0)
+    expect(rebuilds).toBe(0)
+    expect((many as any)._vectorVerified).toBe(true)
+    vi.rebuild = origRebuild
+    await many.close()
+  })
+})

From 7b75f932d47d6e36f40fe25d88a5a6f9974db3cd Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 13:26:41 -0700
Subject: [PATCH 31/82] chore(release): 8.2.8

---
 CHANGELOG.md      | 5 +++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7d53ac05..295891a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.2.8](https://github.com/soulcraftlabs/brainy/compare/v8.2.7...v8.2.8) (2026-07-13)
+
+- fix: honest index readiness — no silently-empty queries on a cold index (d0f69c7)
+
+
 ### [8.2.7](https://github.com/soulcraftlabs/brainy/compare/v8.2.6...v8.2.7) (2026-07-13)
 
 - fix: restore loadBinaryBlob fault-propagation (native column-store lockstep) (b6c7039)
diff --git a/package-lock.json b/package-lock.json
index d81b67e9..b2b3ce8c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.7",
+  "version": "8.2.8",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.7",
+      "version": "8.2.8",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 5d7ed63d..095737c4 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.7",
+  "version": "8.2.8",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 6bcb54f0d992b32de2042adc135fcd366959f173 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 14:43:29 -0700
Subject: [PATCH 32/82] feat: validateIndexConsistency delegates to provider
 invariants (ADR-004 Pass 3)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

validateIndexConsistency() was blind to native providers — it only ran the JS
metadata index's own check, so a native provider whose manifest/segments/counts
had diverged still read as "healthy". Per ADR-004 §6 it now feature-detects and
aggregates each provider's optional validateInvariants() (a never-throwing,
<50ms self-report of its own cross-layer invariants), names any failing
invariant with its numbers in the recommendation, and exposes the per-provider
reports. A provider that violates the never-throw contract is surfaced as
unhealthy, never swallowed.

repairIndex() now reconciles NATIVE derived state from canonical too: it consults
each provider's invariants and calls rebuild() on any whose failing invariant
asks for heal:'rebuild' — the native counterpart of detectAndRepairCorruption().

New provider surface: optional validateInvariants(); new exported types
ProviderInvariantReport / InvariantResult / InvariantHeal. Additive, no break.
Cor implements the hook in 3.0.15 (M2); brainy builds against the shape now
(feature-detected, inert until a provider exposes it). 5 tests.
---
 src/brainy.ts                                 | 126 +++++++++++++++---
 src/index.ts                                  |   1 +
 src/plugin.ts                                 |  92 +++++++++++++
 .../validate-invariants-delegation.test.ts    |  99 ++++++++++++++
 4 files changed, 299 insertions(+), 19 deletions(-)
 create mode 100644 tests/unit/validate-invariants-delegation.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index 72cd6fa3..0a2bd460 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -190,7 +190,7 @@ import type {
   HistoryVersion
 } from './db/types.js'
 import { stableDeepEqual } from './db/stableEqual.js'
-import type { VersionedIndexProvider } from './plugin.js'
+import type { VersionedIndexProvider, ProviderInvariantReport } from './plugin.js'
 import type { Operation, TransactionFunction } from './transaction/types.js'
 
 /**
@@ -12530,33 +12530,96 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     entityCount: number
     indexEntryCount: number
     recommendation: string | null
+    /** Cross-layer (ADR-004 §6): each provider's own invariant self-report, when
+     *  it exposes validateInvariants(). Absent providers are simply not listed. */
+    providers?: ProviderInvariantReport[]
   }> {
     await this.ensureInitialized()
     const result = await this.metadataIndex.validateConsistency()
-    // Fold in a non-fatal index-rebuild failure recorded at init() so the degraded
-    // state is observable through the same health surface (not just the console).
+
+    // Cross-layer integrity (ADR-004 §6): validateConsistency() above only sees the
+    // JS metadata index — it is BLIND to a native provider whose manifest ↔
+    // segments ↔ counts have diverged. Delegate to each provider's own
+    // validateInvariants() and aggregate, so "healthy-while-broken" is impossible.
+    const providers = await this.collectProviderInvariants()
+    const brokenProviders = providers.filter((p) => !p.healthy)
+
+    let healthy = result.healthy
+    const notes: string[] = []
+    if (result.recommendation) notes.push(result.recommendation)
+
     if (this._indexRebuildFailed) {
-      return {
-        ...result,
-        healthy: false,
-        recommendation:
-          `Index rebuild failed at init() (degraded — queries may be incomplete): ` +
-          `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` +
-          (result.recommendation ? ` Also: ${result.recommendation}` : '')
-      }
+      healthy = false
+      notes.unshift(
+        `Index rebuild failed at init() (degraded — queries may be incomplete): ` +
+          `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.`
+      )
     }
     if (this._indexDegradedIds.size > 0) {
-      return {
-        ...result,
-        healthy: false,
-        recommendation:
-          `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
+      healthy = false
+      notes.unshift(
+        `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
           `failed-rollback recovery — the derived index may be incomplete for them. ` +
-          `Run repairIndex() to reconcile.` +
-          (result.recommendation ? ` Also: ${result.recommendation}` : '')
+          `Run repairIndex() to reconcile.`
+      )
+    }
+    for (const p of brokenProviders) {
+      healthy = false
+      const failing = p.invariants.filter((i) => !i.holds)
+      notes.unshift(
+        `Provider '${p.provider}' reports ${failing.length} failing invariant(s): ` +
+          failing.map((i) => `${i.name} (${i.detail}; heal=${i.heal})`).join('; ') +
+          `. Run repairIndex() to reconcile from canonical.`
+      )
+    }
+
+    return {
+      ...result,
+      healthy,
+      recommendation: notes.length > 0 ? notes.join(' ') : null,
+      ...(providers.length > 0 ? { providers } : {})
+    }
+  }
+
+  /**
+   * @description Feature-detect + call each index provider's OPTIONAL
+   * `validateInvariants()` (ADR-004 §6) and collect the reports. The contract is
+   * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a
+   * provider violates that, the throw is turned into an UNHEALTHY synthetic
+   * report (loud), never swallowed into "healthy". Providers that do not expose
+   * the hook are simply omitted (the JS baseline validates via
+   * `metadataIndex.validateConsistency()`).
+   * @returns One report per provider that exposes `validateInvariants()`.
+   */
+  private async collectProviderInvariants(): Promise<ProviderInvariantReport[]> {
+    const reports: ProviderInvariantReport[] = []
+    const providers: unknown[] = [this.metadataIndex, this.index, this.graphIndex]
+    for (const provider of providers) {
+      const fn = (provider as { validateInvariants?: () => Promise<ProviderInvariantReport> } | null)
+        ?.validateInvariants
+      if (typeof fn !== 'function') continue
+      try {
+        const report = await fn.call(provider)
+        if (report && Array.isArray(report.invariants)) reports.push(report)
+      } catch (err) {
+        reports.push({
+          provider: 'unknown',
+          healthy: false,
+          serving: false,
+          invariants: [
+            {
+              name: 'validate-invariants-threw',
+              holds: false,
+              detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`,
+              heal: 'rebuild'
+            }
+          ],
+          checkedAt: Date.now(),
+          durationMs: 0
+        })
       }
     }
-    return result
+    return reports
   }
 
   /**
@@ -15112,6 +15175,31 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         `Writes are re-enabled.`
       )
     }
+    // Cross-layer repair (ADR-004 §6): repairIndex must reconcile NATIVE derived
+    // state from canonical, not just the JS metadata index. Consult each provider's
+    // own validateInvariants() and rebuild any whose failing invariant asks for it
+    // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption().
+    for (const provider of [this.metadataIndex, this.index, this.graphIndex]) {
+      const p = provider as {
+        validateInvariants?: () => Promise<ProviderInvariantReport>
+        rebuild?: () => Promise<void>
+      } | null
+      if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') continue
+      let report: ProviderInvariantReport
+      try {
+        report = await p.validateInvariants()
+      } catch {
+        continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here
+      }
+      if (report.healthy) continue
+      if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) {
+        prodLog.warn(
+          `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` +
+            `requiring a rebuild — reconciling its derived state from canonical.`
+        )
+        await p.rebuild()
+      }
+    }
     // detectAndRepairCorruption() above rebuilt the derived indexes from
     // canonical, so any adopt-forward degraded ids and a non-fatal init
     // rebuild failure are now reconciled — clear the queryable degraded state
diff --git a/src/index.ts b/src/index.ts
index 619a3470..c64d39df 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -199,6 +199,7 @@ export type {
 // Optional provider capability for generation-aware native indexes
 export { isVersionedIndexProvider } from './plugin.js'
 export type { VersionedIndexProvider } from './plugin.js'
+export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js'
 // Optional native graph-acceleration engine (cor 3.0) — the published provider
 // contract + its columnar wire types. Brainy feature-detects an implementation
 // and falls back to its pure-TS adjacency when absent.
diff --git a/src/plugin.ts b/src/plugin.ts
index 12455b3a..c0fda10f 100644
--- a/src/plugin.ts
+++ b/src/plugin.ts
@@ -115,6 +115,62 @@ export interface BrainyPluginContext {
 // implementation then fails to compile until it provides the member.
 // ===========================================================================
 
+/**
+ * @description How a failed provider invariant should be remediated (ADR-004 §6):
+ * - `'none'`    — informational; the invariant held or nothing to do.
+ * - `'repair'`  — a targeted, cheap fix exists (e.g. re-derive a count/manifest field).
+ * - `'rebuild'` — the derived state must be rebuilt from canonical (`provider.rebuild()`).
+ */
+export type InvariantHeal = 'none' | 'repair' | 'rebuild'
+
+/**
+ * @description The result of ONE provider invariant check (ADR-004 §6). A failure
+ * (`holds === false`) NAMES what diverged, with numbers, so it is diagnosable
+ * from the report alone — never a bare boolean. `name` is a stable kebab-case id
+ * for telemetry / remediation routing.
+ */
+export interface InvariantResult {
+  /** Stable kebab-case id, e.g. `'manifest-residency'` / `'posted-count-floor'`. */
+  name: string
+  /** `true` when the invariant holds. */
+  holds: boolean
+  /** Human-readable detail; on failure, names the divergence WITH numbers. */
+  detail: string
+  /** The value the invariant expected (optional, for diagnosis). */
+  expected?: unknown
+  /** The value actually observed (optional, for diagnosis). */
+  actual?: unknown
+  /** How a failure should be remediated. Ignored when `holds === true`. */
+  heal: InvariantHeal
+}
+
+/**
+ * @description A provider's self-report of its own cross-layer invariants
+ * (ADR-004 §6 — the `validateInvariants()` hook). Contract:
+ * - It NEVER throws — a failure is DATA (`healthy: false` + a failing invariant),
+ *   not an exception.
+ * - It is BOUNDED (<50ms): residency checks + O(1) counts only, NO canonical
+ *   walks — safe to call on a live brain, repeatedly.
+ * - `serving` = can the provider answer queries right now (the `isReady()` truth);
+ *   `healthy` = do ALL invariants hold. A provider can be `serving` while an
+ *   invariant flags a latent divergence, or `healthy` but not-yet-`serving` on a
+ *   cold open.
+ */
+export interface ProviderInvariantReport {
+  /** Which provider produced this report, e.g. `'vector'` / `'graph'` / `'metadata'` / `'column'`. */
+  provider: string
+  /** `true` iff every invariant in {@link invariants} holds. */
+  healthy: boolean
+  /** `true` iff the provider can serve queries now (the `isReady()` truth). */
+  serving: boolean
+  /** Each checked invariant and its verdict. */
+  invariants: InvariantResult[]
+  /** Epoch millis when the check ran. */
+  checkedAt: number
+  /** How long the check took (must stay well under 50ms). */
+  durationMs: number
+}
+
 /**
  * The `'metadataIndex'` provider — a drop-in for `MetadataIndexManager`.
  * Brainy calls this surface via `this.metadataIndex.*` (see `brainy.ts`) and
@@ -139,6 +195,18 @@ export interface MetadataIndexProvider {
    */
   isReady?(): boolean
 
+  /**
+   * @description OPTIONAL (ADR-004 §6). The provider's self-report of its own
+   * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence).
+   * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant).
+   * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so
+   * brainy's {@link } `validateIndexConsistency()` can call it on a live brain.
+   * Absent → brainy skips this provider in the cross-layer check (feature-detected).
+   * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this
+   * provider's `rebuild()`.
+   */
+  validateInvariants?(): Promise<ProviderInvariantReport>
+
   /**
    * @description OPTIONAL. A native provider returns true from the moment its
    * `init()` detects a large epoch-drift until its background
@@ -289,6 +357,18 @@ export interface GraphIndexProvider {
    */
   isReady?(): boolean
 
+  /**
+   * @description OPTIONAL (ADR-004 §6). The provider's self-report of its own
+   * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence).
+   * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant).
+   * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so
+   * brainy's {@link } `validateIndexConsistency()` can call it on a live brain.
+   * Absent → brainy skips this provider in the cross-layer check (feature-detected).
+   * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this
+   * provider's `rebuild()`.
+   */
+  validateInvariants?(): Promise<ProviderInvariantReport>
+
   /**
    * @description OPTIONAL eager cold-load. Called once during brain init — AFTER
    * the metadata provider's `init()` (so the id-mapper is hydrated; a native int
@@ -943,6 +1023,18 @@ export interface VectorIndexProvider {
    */
   isReady?(): boolean
 
+  /**
+   * @description OPTIONAL (ADR-004 §6). The provider's self-report of its own
+   * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence).
+   * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant).
+   * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so
+   * brainy's {@link } `validateIndexConsistency()` can call it on a live brain.
+   * Absent → brainy skips this provider in the cross-layer check (feature-detected).
+   * `repairIndex()` maps any failing invariant with `heal: 'rebuild'` to this
+   * provider's `rebuild()`.
+   */
+  validateInvariants?(): Promise<ProviderInvariantReport>
+
   /**
    * @description OPTIONAL. A native provider returns true from the moment its
    * `init()` detects a large epoch-drift until its background
diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts
new file mode 100644
index 00000000..953c6b91
--- /dev/null
+++ b/tests/unit/validate-invariants-delegation.test.ts
@@ -0,0 +1,99 @@
+/**
+ * @module tests/unit/validate-invariants-delegation
+ * @description Pass 3 (ADR-004 §6): validateIndexConsistency() was blind to native
+ * providers — it only saw the JS metadata index, so a native manifest↔segments↔count
+ * divergence read as "healthy". It now feature-detects + aggregates each provider's
+ * validateInvariants(), and repairIndex() maps a failing invariant with heal:'rebuild'
+ * to that provider's rebuild(). "healthy-while-broken must be impossible."
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { Brainy, NounType } from '../../src/index.js'
+import type { ProviderInvariantReport } from '../../src/index.js'
+
+const healthyReport = (provider: string): ProviderInvariantReport => ({
+  provider,
+  healthy: true,
+  serving: true,
+  invariants: [{ name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' }],
+  checkedAt: 1,
+  durationMs: 1
+})
+
+const brokenReport = (provider: string): ProviderInvariantReport => ({
+  provider,
+  healthy: false,
+  serving: true,
+  invariants: [
+    { name: 'manifest-residency', holds: true, detail: 'ok', heal: 'none' },
+    {
+      name: 'posted-count-floor',
+      holds: false,
+      detail: 'posted 2304 < canonical 2354',
+      expected: 2354,
+      actual: 2304,
+      heal: 'rebuild'
+    }
+  ],
+  checkedAt: 1,
+  durationMs: 2
+})
+
+describe('validateIndexConsistency delegates to provider validateInvariants() (Pass 3)', () => {
+  let brain: any
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
+    await brain.init()
+    await brain.add({ data: 'x', type: NounType.Concept })
+    await brain.flush()
+  })
+
+  it('a broken provider report makes the store unhealthy and names the failing invariant', async () => {
+    brain.index.validateInvariants = async () => brokenReport('vector')
+    const v = await brain.validateIndexConsistency()
+    expect(v.healthy).toBe(false)
+    expect(v.recommendation).toMatch(/posted-count-floor/)
+    expect(v.recommendation).toMatch(/posted 2304 < canonical 2354/)
+    expect(v.recommendation).toMatch(/repairIndex\(\)/)
+    expect(v.providers?.some((p: ProviderInvariantReport) => p.provider === 'vector' && !p.healthy)).toBe(true)
+    delete brain.index.validateInvariants
+  })
+
+  it('all-healthy provider reports do not flip the store unhealthy', async () => {
+    brain.index.validateInvariants = async () => healthyReport('vector')
+    brain.graphIndex.validateInvariants = async () => healthyReport('graph')
+    const v = await brain.validateIndexConsistency()
+    expect(v.healthy).toBe(true)
+    expect(v.providers?.length).toBe(2)
+    delete brain.index.validateInvariants
+    delete brain.graphIndex.validateInvariants
+  })
+
+  it('a validateInvariants() that THROWS is surfaced as unhealthy, never swallowed', async () => {
+    brain.index.validateInvariants = async () => { throw new Error('provider blew up') }
+    const v = await brain.validateIndexConsistency()
+    expect(v.healthy).toBe(false)
+    expect(v.recommendation).toMatch(/validate-invariants-threw/)
+    delete brain.index.validateInvariants
+  })
+
+  it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => {
+    const v = await brain.validateIndexConsistency()
+    expect(v.providers).toBeUndefined()
+    expect(typeof v.healthy).toBe('boolean')
+  })
+
+  it('repairIndex() rebuilds a provider whose failing invariant asks for it', async () => {
+    let rebuilt = false
+    brain.index.validateInvariants = async () => (rebuilt ? healthyReport('vector') : brokenReport('vector'))
+    const origRebuild = brain.index.rebuild.bind(brain.index)
+    brain.index.rebuild = async (...a: any[]) => { rebuilt = true; return origRebuild(...a) }
+    await brain.repairIndex()
+    expect(rebuilt).toBe(true)
+    // After repair, the store validates healthy again.
+    const v = await brain.validateIndexConsistency()
+    expect(v.providers?.find((p: ProviderInvariantReport) => p.provider === 'vector')?.healthy).toBe(true)
+    brain.index.rebuild = origRebuild
+    delete brain.index.validateInvariants
+  })
+})

From bfa17621070ee68df1953ccbfc5a4d5fa5b25f82 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 14:52:06 -0700
Subject: [PATCH 33/82] =?UTF-8?q?feat:=20registered-blob=20family=20contra?=
 =?UTF-8?q?ct=20=E2=80=94=20declared=20index=20blobs=20are=20undeletable?=
 =?UTF-8?q?=20(ADR-004=20Pass=202)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The class-killer for the lost-main.dkann incident (ADR-004 §7). A provider can
declare a derived-index blob FAMILY (a set of members that are load-bearing
together, e.g. vector-base = main.dkann + main.slotmap + main.slotrev). Once
declared:

- deleteBinaryBlob / removeRawPrefix REFUSE to remove a declared member, throwing
  the new ProtectedArtifactError — an in-process GC / sweeper is now INCAPABLE of
  deleting a load-bearing index file (COLD != DEAD). Intentional retirement is an
  explicit unregisterDerivedFamily(name) first.
- The declaration persists to _system/derived-artifacts.json, so protection
  survives a reopen; clear() resets it with the rest of the derived footprint.
- checkDerivedFamiliesPresent() names any member missing on open (the catch for an
  EXTERNAL deleter that bypasses the in-process refusal) → rebuild from canonical.
- Transients (*.tmp.*, *.rebuild-tmp, *.rotate-tmp) are never protected; a
  namespace family protects a growing prefix (seg-*).

New StorageAdapter surface (optional): registerDerivedFamily / unregisterDerivedFamily
/ listDerivedFamilies + DerivedFamilyDeclaration; new exported errors
ProtectedArtifactError / DerivedArtifactMissingError. Enforcement is inert until a
provider declares a family (no regression). Cor declares its 6 families + does the
atomic set-swap in 3.0.15 (M2); brainy builds the contract now. 8 tests.
---
 src/coreTypes.ts                              |  60 +++++++
 src/errors/brainyError.ts                     |  56 ++++++
 src/index.ts                                  |   8 +-
 src/storage/adapters/fileSystemStorage.ts     |   8 +
 src/storage/adapters/memoryStorage.ts         |   3 +
 src/storage/baseStorage.ts                    | 169 +++++++++++++++++-
 .../storage/registered-blob-contract.test.ts  | 130 ++++++++++++++
 7 files changed, 429 insertions(+), 5 deletions(-)
 create mode 100644 tests/unit/storage/registered-blob-contract.test.ts

diff --git a/src/coreTypes.ts b/src/coreTypes.ts
index cce06e8a..4b1ebfb9 100644
--- a/src/coreTypes.ts
+++ b/src/coreTypes.ts
@@ -754,6 +754,39 @@ export interface Change {
   data?: HNSWNounWithMetadata | HNSWVerbWithMetadata
 }
 
+/**
+ * @description A declared derived-index blob FAMILY (ADR-004 §7 — the
+ * registered-blob contract). A family names the set of on-disk blobs that a
+ * derived index needs AS A SET (e.g. the vector base = `main.dkann` +
+ * `main.slotmap` + `main.slotrev`): losing ANY member corrupts the index. Once a
+ * family is declared:
+ * - its members are UNDELETABLE through the storage layer — `deleteBinaryBlob` /
+ *   `removeRawPrefix` refuse with a `ProtectedArtifactError`, so an in-process
+ *   GC/sweeper cannot remove a load-bearing file (intentional retirement =
+ *   `unregisterDerivedFamily` first);
+ * - a member missing on open is a loud `DerivedArtifactMissingError` → rebuild.
+ * `COLD ≠ DEAD`: a write-once segment or a recovery-critical archive is
+ * load-bearing even when it has not been touched in a long time.
+ */
+export interface DerivedFamilyDeclaration {
+  /** Stable family id, e.g. `'vector-base'` / `'metadata-sstables'`. */
+  name: string
+  /**
+   * The logical blob keys that make up the family. When {@link namespace} is
+   * set, each entry is a PREFIX protecting every key beneath it (for growing
+   * sets like `seg-*`); otherwise each entry is an exact member key.
+   */
+  members: string[]
+  /** When true, {@link members} are prefixes (protect all keys beneath each). */
+  namespace?: boolean
+  /**
+   * Whether a missing family can be rebuilt from the canonical records (default
+   * `true`). `false` marks an irreplaceable family (a missing member is data
+   * loss, not a rebuild).
+   */
+  rebuildable?: boolean
+}
+
 export interface StorageAdapter {
   init(): Promise<void>
 
@@ -1049,6 +1082,33 @@ export interface StorageAdapter {
    */
   getBinaryBlobPath(key: string): string | null
 
+  /**
+   * @description OPTIONAL (ADR-004 §7 registered-blob contract). Declare a
+   * derived-index blob {@link DerivedFamilyDeclaration | family} whose members
+   * become UNDELETABLE through this adapter — a subsequent `deleteBinaryBlob` /
+   * `removeRawPrefix` that would remove a declared member throws a
+   * `ProtectedArtifactError`. Providers declare their families on create; the
+   * declaration is persisted so protection survives a reopen. Idempotent per
+   * `name` (re-declaring replaces).
+   * @param family - The family to protect.
+   */
+  registerDerivedFamily?(family: DerivedFamilyDeclaration): Promise<void>
+
+  /**
+   * @description OPTIONAL. Remove a family's protection so its members can be
+   * deleted again — the explicit, auditable step for intentional retirement of a
+   * derived index (the ONLY way a declared member becomes deletable).
+   * @param name - The {@link DerivedFamilyDeclaration.name} to unregister.
+   */
+  unregisterDerivedFamily?(name: string): Promise<void>
+
+  /**
+   * @description OPTIONAL. List the currently-declared derived-index families —
+   * the source of truth for what `clear()` must wipe and what a
+   * missing-on-open check verifies.
+   */
+  listDerivedFamilies?(): Promise<DerivedFamilyDeclaration[]>
+
   /**
    * Save statistics data
    * @param statistics The statistics data to save
diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts
index c2667009..6aa1f919 100644
--- a/src/errors/brainyError.ts
+++ b/src/errors/brainyError.ts
@@ -15,6 +15,8 @@ export type BrainyErrorType =
   | 'GRAPH_INDEX_NOT_READY'
   | 'METADATA_INDEX_NOT_READY'
   | 'VECTOR_INDEX_NOT_READY'
+  | 'PROTECTED_ARTIFACT'
+  | 'DERIVED_ARTIFACT_MISSING'
   | 'MIGRATION_IN_PROGRESS'
 
 /**
@@ -307,6 +309,60 @@ export class VectorIndexNotReadyError extends BrainyError {
     }
 }
 
+/**
+ * Thrown when a delete (`deleteBinaryBlob` / `removeRawPrefix`) would remove a
+ * blob that is a declared member of a protected derived-index FAMILY (ADR-004 §7
+ * registered-blob contract). Declared derived artifacts are undeletable through
+ * the storage layer — this makes an in-process GC / sweeper INCAPABLE of removing
+ * a load-bearing index file (the lost-`main.dkann` class). Intentional retirement
+ * is the explicit `unregisterDerivedFamily(name)` step, then the delete.
+ */
+export class ProtectedArtifactError extends BrainyError {
+    /** The blob key the delete targeted. */
+    public readonly key: string
+    /** The protected family the key belongs to. */
+    public readonly family: string
+    constructor(key: string, family: string) {
+        super(
+            `Refused to delete '${key}': it is a declared member of the protected ` +
+                `derived-index family '${family}'. Declared derived artifacts are undeletable ` +
+                `through the storage layer (COLD ≠ DEAD) — unregisterDerivedFamily('${family}') ` +
+                `first if retirement is intentional.`,
+            'PROTECTED_ARTIFACT',
+            false
+        )
+        this.name = 'ProtectedArtifactError'
+        this.key = key
+        this.family = family
+    }
+}
+
+/**
+ * Raised (loudly) when a declared derived-index family is missing one or more of
+ * its members on open — i.e. a load-bearing blob was deleted OUTSIDE the write
+ * path (an external sweeper the in-process refusal cannot stop). The index must
+ * be rebuilt from canonical; "healthy-while-broken" is impossible because the
+ * missing member is named, not silently tolerated.
+ */
+export class DerivedArtifactMissingError extends BrainyError {
+    /** The family with missing members. */
+    public readonly family: string
+    /** The member keys that are absent. */
+    public readonly missing: string[]
+    constructor(family: string, missing: string[]) {
+        super(
+            `Derived-index family '${family}' is missing ${missing.length} declared ` +
+                `member(s) on open (${missing.join(', ')}) — deleted outside the write path. ` +
+                `The index must be rebuilt from canonical.`,
+            'DERIVED_ARTIFACT_MISSING',
+            false
+        )
+        this.name = 'DerivedArtifactMissingError'
+        this.family = family
+        this.missing = missing
+    }
+}
+
 /**
  * Thrown when a data-plane read or write is issued against a brain that is
  * running its one-time, automatic 7.x → 8.0 on-disk upgrade — the coordinated
diff --git a/src/index.ts b/src/index.ts
index c64d39df..b0a95fe1 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -150,7 +150,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js
 
 // Base error + typed migration-lock error — thrown by any data-plane call while a
 // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After.
-export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError } from './errors/brainyError.js'
+export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js'
 export type { BrainyErrorType } from './errors/brainyError.js'
 
 // ============= 8.0 Db API — generational MVCC =============
@@ -301,7 +301,8 @@ import type {
   HNSWNoun,
   HNSWVerb,
   HNSWConfig,
-  StorageAdapter
+  StorageAdapter,
+  DerivedFamilyDeclaration
 } from './coreTypes.js'
 
 // Export vector index implementation (the JS HNSW path)
@@ -319,7 +320,8 @@ export type {
   HNSWNoun,
   HNSWVerb,
   HNSWConfig,
-  StorageAdapter
+  StorageAdapter,
+  DerivedFamilyDeclaration
 }
 
 // Export graph types
diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index 9ff0539e..c2352d2f 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -630,6 +630,9 @@ export class FileSystemStorage extends BaseStorage {
    */
   public override async removeRawPrefix(prefix: string): Promise<void> {
     await this.ensureInitialized()
+    // Registered-blob contract: a prefix-nuke must not take out a protected
+    // family member (ADR-004 §7). Throws if the prefix intersects one.
+    await this.assertPrefixNotProtected(prefix)
     await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true })
   }
 
@@ -1250,6 +1253,11 @@ export class FileSystemStorage extends BaseStorage {
    */
   public async deleteBinaryBlob(key: string): Promise<void> {
     await this.ensureInitialized()
+    // Registered-blob contract (ADR-004 §7): refuse to delete a declared
+    // derived-index family member — an in-process GC/sweeper cannot remove a
+    // load-bearing index file. Throws ProtectedArtifactError; no-op when no
+    // families are registered.
+    await this.assertBlobKeyDeletable(key)
     try {
       await fs.promises.unlink(this.blobPath(key))
     } catch {
diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts
index 05dfb0b4..6b2564d5 100644
--- a/src/storage/adapters/memoryStorage.ts
+++ b/src/storage/adapters/memoryStorage.ts
@@ -183,6 +183,9 @@ export class MemoryStorage extends BaseStorage {
    * @param key - The blob key.
    */
   public async deleteBinaryBlob(key: string): Promise<void> {
+    // Registered-blob contract (ADR-004 §7) — parity with the filesystem adapter:
+    // a declared family member is undeletable (throws ProtectedArtifactError).
+    await this.assertBlobKeyDeletable(key)
     this.blobStore.delete(key)
   }
 
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 159a0593..8a491e3e 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -15,7 +15,8 @@ import {
   VerbMetadata,
   HNSWNounWithMetadata,
   HNSWVerbWithMetadata,
-  StatisticsData
+  StatisticsData,
+  DerivedFamilyDeclaration
 } from '../coreTypes.js'
 import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
 import { validateNounType, validateVerbType } from '../utils/typeValidation.js'
@@ -31,7 +32,7 @@ import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js'
 import { unwrapBinaryData } from './binaryDataCodec.js'
 import { prodLog } from '../utils/logger.js'
 import { isAbsentError } from '../utils/errorClassification.js'
-import { BrainyError } from '../errors/brainyError.js'
+import { BrainyError, ProtectedArtifactError, DerivedArtifactMissingError } from '../errors/brainyError.js'
 import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js'
 import {
   splitNounMetadataRecord,
@@ -251,6 +252,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   protected isInitialized = false
   /** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
   private _graphFastPathProbed = false
+  /**
+   * Registered-blob contract (ADR-004 §7): declared derived-index families, keyed
+   * by name. Members are undeletable through the blob delete seams. Loaded lazily
+   * from `_system/derived-artifacts.json` and re-persisted on every change.
+   */
+  private _derivedFamilies = new Map<string, DerivedFamilyDeclaration>()
+  /** One-shot guard for loading the persisted family registry. */
+  private _derivedFamiliesLoaded = false
+  /** Storage-root-relative path of the persisted family registry. */
+  private static readonly DERIVED_FAMILIES_KEY = '_system/derived-artifacts.json'
   protected graphIndex?: GraphAdjacencyIndex
   protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
   /**
@@ -1041,6 +1052,154 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     await this.writeObjectToPath(path, data)
   }
 
+  // ==========================================================================
+  // Registered-blob contract (ADR-004 §7) — declared derived-index families are
+  // undeletable through the blob delete seams. Shared here so every adapter that
+  // extends BaseStorage inherits the same enforcement; the concrete
+  // deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable /
+  // assertPrefixNotProtected before removing anything.
+  // ==========================================================================
+
+  /** Load the persisted family registry once (lazy). */
+  private async ensureDerivedFamiliesLoaded(): Promise<void> {
+    if (this._derivedFamiliesLoaded) return
+    const stored = await this.readRawObject(BaseStorage.DERIVED_FAMILIES_KEY).catch(() => null)
+    const families = (stored as { families?: DerivedFamilyDeclaration[] } | null)?.families
+    if (Array.isArray(families)) {
+      for (const f of families) {
+        if (f && typeof f.name === 'string' && Array.isArray(f.members)) this._derivedFamilies.set(f.name, f)
+      }
+    }
+    this._derivedFamiliesLoaded = true
+  }
+
+  /** Persist the current family registry (fsync'd via the raw-object write). */
+  private async persistDerivedFamilies(): Promise<void> {
+    await this.writeRawObject(BaseStorage.DERIVED_FAMILIES_KEY, {
+      families: [...this._derivedFamilies.values()]
+    })
+  }
+
+  public async registerDerivedFamily(family: DerivedFamilyDeclaration): Promise<void> {
+    await this.ensureInitialized()
+    await this.ensureDerivedFamiliesLoaded()
+    this._derivedFamilies.set(family.name, {
+      name: family.name,
+      members: [...family.members],
+      ...(family.namespace !== undefined ? { namespace: family.namespace } : {}),
+      ...(family.rebuildable !== undefined ? { rebuildable: family.rebuildable } : {})
+    })
+    await this.persistDerivedFamilies()
+  }
+
+  public async unregisterDerivedFamily(name: string): Promise<void> {
+    await this.ensureInitialized()
+    await this.ensureDerivedFamiliesLoaded()
+    if (this._derivedFamilies.delete(name)) await this.persistDerivedFamilies()
+  }
+
+  public async listDerivedFamilies(): Promise<DerivedFamilyDeclaration[]> {
+    await this.ensureInitialized()
+    await this.ensureDerivedFamiliesLoaded()
+    return [...this._derivedFamilies.values()]
+  }
+
+  /**
+   * @description A blob key that is a transient write-scratch file (a `*.tmp.*`
+   * temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as
+   * part of an atomic write; it is never a protected family member (ADR-004 §7).
+   */
+  private isTransientBlobKey(key: string): boolean {
+    return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key)
+  }
+
+  /**
+   * @description The name of the protected family a blob key belongs to, or
+   * `null`. A `namespace` member protects every key beneath it; a plain member
+   * protects that exact key.
+   */
+  private protectingFamilyOf(key: string): string | null {
+    for (const family of this._derivedFamilies.values()) {
+      for (const member of family.members) {
+        if (family.namespace ? key === member || key.startsWith(member) : key === member) {
+          return family.name
+        }
+      }
+    }
+    return null
+  }
+
+  /**
+   * @description Enforcement point for `deleteBinaryBlob`: refuse (throw
+   * {@link ProtectedArtifactError}) when the key is a declared family member.
+   * Transients pass through. An undeclared blob delete under an ACTIVE contract
+   * (families are registered) is logged loudly — nothing under `_blobs/` should
+   * vanish unremarked once the contract is in force.
+   */
+  protected async assertBlobKeyDeletable(key: string): Promise<void> {
+    await this.ensureDerivedFamiliesLoaded()
+    if (this._derivedFamilies.size === 0) return // contract inactive — no enforcement
+    if (this.isTransientBlobKey(key)) return
+    const family = this.protectingFamilyOf(key)
+    if (family) throw new ProtectedArtifactError(key, family)
+    prodLog.warn(
+      `[BaseStorage] deleteBinaryBlob('${key}') removes an UNDECLARED blob while the ` +
+        `registered-blob contract is active — permitted, but surfaced so no _blobs/ file ` +
+        `disappears silently.`
+    )
+  }
+
+  /**
+   * @description Enforcement point for `removeRawPrefix`: refuse when the prefix
+   * would take out a protected family member (a prefix-nuke must not remove a
+   * declared blob). Transients are ignored.
+   */
+  protected async assertPrefixNotProtected(prefix: string): Promise<void> {
+    await this.ensureDerivedFamiliesLoaded()
+    if (this._derivedFamilies.size === 0) return
+    for (const family of this._derivedFamilies.values()) {
+      for (const member of family.members) {
+        // Under the shared `_blobs/` root, member keys resolve beneath it; a
+        // prefix intersects a member when either contains the other.
+        const memberBlobPath = `_blobs/${member}`
+        if (
+          memberBlobPath.startsWith(prefix) ||
+          prefix.startsWith(memberBlobPath) ||
+          member.startsWith(prefix) ||
+          prefix.startsWith(member)
+        ) {
+          if (!this.isTransientBlobKey(member)) throw new ProtectedArtifactError(member, family.name)
+        }
+      }
+    }
+  }
+
+  /**
+   * @description Verify every declared family has all its members present on
+   * disk (ADR-004 §7 missing-on-open catch for an EXTERNAL deleter that bypasses
+   * the in-process refusal). Returns the incomplete families (name + missing
+   * members) — the caller decides how to heal (rebuild from canonical). Loud by
+   * construction: a missing load-bearing blob is named, never silently tolerated.
+   */
+  public async checkDerivedFamiliesPresent(): Promise<Array<{ name: string; missing: string[]; rebuildable: boolean }>> {
+    await this.ensureInitialized()
+    await this.ensureDerivedFamiliesLoaded()
+    const incomplete: Array<{ name: string; missing: string[]; rebuildable: boolean }> = []
+    for (const family of this._derivedFamilies.values()) {
+      if (family.namespace) continue // a growing prefix has no fixed member set to verify
+      const missing: string[] = []
+      for (const member of family.members) {
+        const blob = await this.loadBinaryBlob(member).catch(() => null)
+        if (!blob) missing.push(member)
+      }
+      if (missing.length > 0) {
+        prodLog.warn(new DerivedArtifactMissingError(family.name, missing).message)
+        incomplete.push({ name: family.name, missing, rebuildable: family.rebuildable !== false })
+      }
+    }
+    return incomplete
+  }
+
   /**
    * Delete a raw object at a storage-root-relative path (no-op if absent).
    *
@@ -1224,6 +1383,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
    */
   protected async reloadDerivedState(): Promise<void> {
     this.clearWriteCache()
+    // Registered-blob registry: clear() wipes the persisted `_system/` copy and
+    // the family members under `_blobs/`, so drop the in-memory cache too — a
+    // reset brain must not keep stale family protection or report their members
+    // "missing" on the next check.
+    this._derivedFamilies.clear()
+    this._derivedFamiliesLoaded = false
     this.nounCountsByType.fill(0)
     this.verbCountsByType.fill(0)
     this.subtypeCountsByType.clear()
diff --git a/tests/unit/storage/registered-blob-contract.test.ts b/tests/unit/storage/registered-blob-contract.test.ts
new file mode 100644
index 00000000..ef450163
--- /dev/null
+++ b/tests/unit/storage/registered-blob-contract.test.ts
@@ -0,0 +1,130 @@
+/**
+ * @module tests/unit/storage/registered-blob-contract
+ * @description Pass 2 (ADR-004 §7): declared derived-index blob FAMILIES are
+ * undeletable through the storage layer — an in-process GC/sweeper cannot remove
+ * a load-bearing index file (the lost-main.dkann class). Covers declare →
+ * protected-delete-throws; unregister → delete-ok; transient passthrough;
+ * namespace prefix protects children; removeRawPrefix refuses a protected
+ * intersection; persistence across reopen; missing-on-open detection.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
+import { ProtectedArtifactError } from '../../../src/index.js'
+
+describe('registered-blob family contract (Pass 2, ADR-004 §7)', () => {
+  let storage: any
+
+  beforeEach(async () => {
+    storage = new MemoryStorage()
+    await storage.init()
+  })
+
+  const seedVectorFamily = async () => {
+    await storage.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1]))
+    await storage.saveBinaryBlob('_system/vector-index/main.slotmap', Buffer.from([2]))
+    await storage.saveBinaryBlob('_system/vector-index/main.slotrev', Buffer.from([3]))
+    await storage.registerDerivedFamily({
+      name: 'vector-base',
+      members: [
+        '_system/vector-index/main.dkann',
+        '_system/vector-index/main.slotmap',
+        '_system/vector-index/main.slotrev'
+      ]
+    })
+  }
+
+  it('a declared member is undeletable (throws ProtectedArtifactError)', async () => {
+    await seedVectorFamily()
+    await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf(
+      ProtectedArtifactError
+    )
+    // The blob is still there.
+    expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).not.toBeNull()
+  })
+
+  it('unregistering the family makes its members deletable again', async () => {
+    await seedVectorFamily()
+    await storage.unregisterDerivedFamily('vector-base')
+    await expect(storage.deleteBinaryBlob('_system/vector-index/main.dkann')).resolves.toBeUndefined()
+    expect(await storage.loadBinaryBlob('_system/vector-index/main.dkann')).toBeNull()
+  })
+
+  it('an undeclared blob is still deletable (contract does not lock everything)', async () => {
+    await seedVectorFamily()
+    await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([9]))
+    await expect(storage.deleteBinaryBlob('graph-lsm/source/sstable-1')).resolves.toBeUndefined()
+  })
+
+  it('a transient (*.tmp.*) is deletable even if it sits under a protected namespace', async () => {
+    await storage.registerDerivedFamily({
+      name: 'vector-ns',
+      members: ['_system/vector-index/'],
+      namespace: true
+    })
+    await storage.saveBinaryBlob('_system/vector-index/main.dkann.tmp.123', Buffer.from([1]))
+    await expect(
+      storage.deleteBinaryBlob('_system/vector-index/main.dkann.tmp.123')
+    ).resolves.toBeUndefined()
+  })
+
+  it('a namespace family protects a growing child key', async () => {
+    await storage.registerDerivedFamily({
+      name: 'vector-segments',
+      members: ['_system/vector-index/'],
+      namespace: true
+    })
+    await storage.saveBinaryBlob('_system/vector-index/seg-42', Buffer.from([7]))
+    await expect(storage.deleteBinaryBlob('_system/vector-index/seg-42')).rejects.toBeInstanceOf(
+      ProtectedArtifactError
+    )
+  })
+
+  it('checkDerivedFamiliesPresent() names a member deleted outside the write path', async () => {
+    await seedVectorFamily()
+    // Simulate an EXTERNAL deleter (bypasses the in-process refusal): drop a
+    // member straight from the underlying store.
+    ;(storage as any).blobStore.delete('_system/vector-index/main.slotmap')
+    const incomplete = await storage.checkDerivedFamiliesPresent()
+    expect(incomplete).toHaveLength(1)
+    expect(incomplete[0].name).toBe('vector-base')
+    expect(incomplete[0].missing).toContain('_system/vector-index/main.slotmap')
+    expect(incomplete[0].rebuildable).toBe(true)
+  })
+
+  it('no families registered → deleteBinaryBlob behaves exactly as before (no enforcement)', async () => {
+    await storage.saveBinaryBlob('some/blob', Buffer.from([1]))
+    await expect(storage.deleteBinaryBlob('some/blob')).resolves.toBeUndefined()
+  })
+})
+
+describe('registered-blob family protection survives a reopen (FileSystemStorage)', () => {
+  let dir: string
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-regblob-'))
+  })
+  afterEach(() => fs.rmSync(dir, { recursive: true, force: true }))
+
+  it('a family declared in one session protects members after reopen', async () => {
+    const s1: any = new FileSystemStorage(dir)
+    await s1.init()
+    await s1.saveBinaryBlob('_system/vector-index/main.dkann', Buffer.from([1]))
+    await s1.registerDerivedFamily({ name: 'vector-base', members: ['_system/vector-index/main.dkann'] })
+
+    // Reopen — the registry is loaded from _system/derived-artifacts.json.
+    const s2: any = new FileSystemStorage(dir)
+    await s2.init()
+    expect(await s2.listDerivedFamilies()).toHaveLength(1)
+    await expect(s2.deleteBinaryBlob('_system/vector-index/main.dkann')).rejects.toBeInstanceOf(
+      ProtectedArtifactError
+    )
+    // removeRawPrefix nuking the vector dir is also refused.
+    await expect(s2.removeRawPrefix('_blobs/_system/vector-index')).rejects.toBeInstanceOf(
+      ProtectedArtifactError
+    )
+  })
+})

From ec5b93339ab98f9454131a60d8a0b65e01bdd66f Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 15:15:01 -0700
Subject: [PATCH 34/82] perf: parallel + id-only canonical enumeration
 (heal-cost dominant term)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The canonical enumeration walk (getNounsWithPagination) hydrated each entity's
vector + metadata ONE-AT-A-TIME inside the shard loop — every enumeration paid
N x per-op-latency serially, and every index heal enumerates canonical, so this
was the dominant multiplier in the measured multi-minute heals (cortex heal-cost
decomposition).

- Hydration is now 16-way bounded-concurrency (matching the wave cor's native
  rebuild uses): heal wall-clock becomes ~2xN/16 x per-op instead of N x per-op.
  Order, cursor resume (skipped nouns still never read), filters, peek/hasMore
  and totalCount are all preserved — pages are byte-identical to the serial walk.
- New getNounIdsWithPagination(): the id-only opt-out for callers that own their
  IO schedule (an index heal). Unfiltered = ids straight from the shard paths,
  ZERO per-entity reads; filtered hydrates metadata only (16-way). Same
  cursor/offset/nextCursor contract, so it is page-compatible with the hydrating
  walk.

Regression: pagination-parallel-hydration.test.ts pins paged==big-page identity,
ids==items order, zero-read id-only, and filter parity. 103 storage tests green.
---
 src/storage/baseStorage.ts                    | 174 ++++++++++++++++--
 .../pagination-parallel-hydration.test.ts     |  94 ++++++++++
 2 files changed, 250 insertions(+), 18 deletions(-)
 create mode 100644 tests/unit/storage/pagination-parallel-hydration.test.ts

diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 8a491e3e..5fdf129f 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -262,6 +262,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   private _derivedFamiliesLoaded = false
   /** Storage-root-relative path of the persisted family registry. */
   private static readonly DERIVED_FAMILIES_KEY = '_system/derived-artifacts.json'
+  /**
+   * Bounded concurrency for hydrating enumerated nouns during a pagination walk
+   * (readCanonicalObject + getNounMetadata per item). A canonical enumeration —
+   * which every index heal performs — otherwise pays N×per-op-latency serially;
+   * 16-way matches the wave the native rebuild uses so heal wall-clock is
+   * ~2×N/16×per-op instead of N×per-op. Bounded so a huge dataset can't spawn a
+   * read per entity at once.
+   */
+  private static readonly HYDRATE_CONCURRENCY = 16
   protected graphIndex?: GraphAdjacencyIndex
   protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
   /**
@@ -1995,39 +2004,63 @@ export abstract class BaseStorage extends BaseStorageAdapter {
           .map((p) => ({ path: p, id: idFromVectorPath(p) }))
           .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
 
-        for (const { path: nounPath, id: nounId } of entries) {
-          if (collected.length >= peekCount) break
-          // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id.
-          if (cursor && shard === cursor.shard && nounId <= cursor.id) continue
+        // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
+        // id BEFORE hydrating — a cheap id compare (from the path), so skipped
+        // nouns are never read.
+        const toHydrate =
+          cursor && shard === cursor.shard
+            ? entries.filter((e) => e.id > cursor.id)
+            : entries
 
-          try {
-            const noun = await this.readCanonicalObject(nounPath)
-            if (!noun) continue
-            const deserialized = this.deserializeNoun(noun)
-            const metadata = await this.getNounMetadata(deserialized.id)
-            if (!metadata) continue
+        // Hydrate in bounded-concurrency batches (16-way) instead of one-at-a-time.
+        // Every canonical enumeration — and every index heal enumerates canonical —
+        // otherwise pays N×per-op-latency SERIALLY (the dominant heal-time term).
+        // Order is preserved (the batch is a slice of the sorted entries and its
+        // results are consumed in order), so offset windows / cursor resume stay
+        // deterministic. peekCount stops the walk; the final batch over-hydrates by
+        // at most BASE_STORAGE_HYDRATE_CONCURRENCY entries (bounded, acceptable).
+        for (
+          let i = 0;
+          i < toHydrate.length && collected.length < peekCount;
+          i += BaseStorage.HYDRATE_CONCURRENCY
+        ) {
+          const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
+          const hydrated = await Promise.all(
+            batch.map(async ({ path: nounPath }) => {
+              try {
+                const noun = await this.readCanonicalObject(nounPath)
+                if (!noun) return null
+                const deserialized = this.deserializeNoun(noun)
+                const metadata = await this.getNounMetadata(deserialized.id)
+                if (!metadata) return null
+                return { deserialized, metadata }
+              } catch (error) {
+                // Skip nouns that fail to load
+                return null
+              }
+            })
+          )
+
+          for (const h of hydrated) {
+            if (collected.length >= peekCount) break
+            if (!h) continue
+            const { deserialized, metadata } = h
 
             // Apply type filter
             if (filter?.nounType && metadata.noun) {
               const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
-              if (!types.includes(metadata.noun)) {
-                continue
-              }
+              if (!types.includes(metadata.noun)) continue
             }
 
             // Apply service filter
             if (filter?.service) {
               const services = Array.isArray(filter.service) ? filter.service : [filter.service]
-              if (metadata.service && !services.includes(metadata.service)) {
-                continue
-              }
+              if (metadata.service && !services.includes(metadata.service)) continue
             }
 
             // Combine noun + metadata via the canonical hydration helper —
             // reserved fields top-level, ONLY custom fields in `metadata`.
             collected.push({ noun: this.hydrateNounWithMetadata(deserialized, metadata), shard })
-          } catch (error) {
-            // Skip nouns that fail to load
           }
         }
       } catch (error) {
@@ -2067,6 +2100,111 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     }
   }
 
+  /**
+   * @description Enumerate noun IDS ONLY, without hydrating each entity's vector
+   * and metadata — the opt-out for callers (e.g. an index heal) that own their
+   * own IO schedule and index straight from a stream. Shares the exact
+   * shard-walk, cursor, offset and `nextCursor` contract of
+   * {@link getNounsWithPagination}, so the two are page-compatible.
+   *
+   * - UNFILTERED (the heal case): ids come straight from the shard paths — ZERO
+   *   per-entity reads. A full enumeration is O(entries listed), not O(N reads).
+   * - FILTERED: the type/service filter needs metadata, so only the metadata is
+   *   hydrated (16-way bounded concurrency), never the full noun.
+   *
+   * @param options - `limit`/`offset`/`cursor`/`filter` — same semantics as
+   *   {@link getNounsWithPagination}.
+   * @returns The page of ids plus `totalCount` / `hasMore` / `nextCursor`.
+   */
+  public async getNounIdsWithPagination(options: {
+    limit: number
+    offset?: number
+    cursor?: string
+    filter?: {
+      nounType?: string | string[]
+      service?: string | string[]
+      metadata?: Record<string, any>
+    }
+  }): Promise<{ ids: string[]; totalCount: number; hasMore: boolean; nextCursor?: string }> {
+    await this.ensureInitialized()
+
+    const { limit, offset = 0, filter } = options
+    const cursor = this.decodeNounWalkCursor(options.cursor)
+    const collected: Array<{ id: string; shard: number }> = []
+    const peekCount = cursor ? limit + 1 : offset + limit + 1
+    const startShard = cursor ? cursor.shard : 0
+
+    for (let shard = startShard; shard < 256 && collected.length < peekCount; shard++) {
+      const shardHex = shard.toString(16).padStart(2, '0')
+      const shardDir = `entities/nouns/${shardHex}`
+      try {
+        const nounFiles = await this.listCanonicalObjects(shardDir)
+        const entries = nounFiles
+          .filter((p) => p.includes('/vectors.json'))
+          .map((p) => idFromVectorPath(p))
+          .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
+        const toWalk =
+          cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
+
+        if (!filter) {
+          // Unfiltered — ids straight from the paths, no reads at all.
+          for (const id of toWalk) {
+            if (collected.length >= peekCount) break
+            collected.push({ id, shard })
+          }
+        } else {
+          // Filtered — hydrate metadata ONLY (16-way) to apply the filter.
+          for (
+            let i = 0;
+            i < toWalk.length && collected.length < peekCount;
+            i += BaseStorage.HYDRATE_CONCURRENCY
+          ) {
+            const batch = toWalk.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
+            const metas = await Promise.all(
+              batch.map(async (id) => {
+                try {
+                  return { id, metadata: await this.getNounMetadata(id) }
+                } catch {
+                  return null
+                }
+              })
+            )
+            for (const m of metas) {
+              if (collected.length >= peekCount) break
+              if (!m || !m.metadata) continue
+              const metadata = m.metadata
+              if (filter.nounType && metadata.noun) {
+                const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
+                if (!types.includes(metadata.noun)) continue
+              }
+              if (filter.service) {
+                const services = Array.isArray(filter.service) ? filter.service : [filter.service]
+                if (metadata.service && !services.includes(metadata.service)) continue
+              }
+              collected.push({ id: m.id, shard })
+            }
+          }
+        }
+      } catch (error) {
+        // Skip shards with no data
+      }
+    }
+
+    const windowStart = cursor ? 0 : offset
+    const pagePairs = collected.slice(windowStart, windowStart + limit)
+    const ids = pagePairs.map((p) => p.id)
+    const hasMore = collected.length > windowStart + limit
+    const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length)
+
+    let nextCursor: string | undefined = undefined
+    if (hasMore && pagePairs.length > 0) {
+      const lastPair = pagePairs[pagePairs.length - 1]
+      nextCursor = this.encodeNounWalkCursor(lastPair.shard, lastPair.id)
+    }
+
+    return { ids, totalCount, hasMore, nextCursor }
+  }
+
   /**
    * @description Encode a noun-walk resume cursor — the `(shard, nounId)` of the
    * last returned noun — as an opaque, version-tagged token (`cn1:` prefix lets
diff --git a/tests/unit/storage/pagination-parallel-hydration.test.ts b/tests/unit/storage/pagination-parallel-hydration.test.ts
new file mode 100644
index 00000000..ada324bb
--- /dev/null
+++ b/tests/unit/storage/pagination-parallel-hydration.test.ts
@@ -0,0 +1,94 @@
+/**
+ * @module tests/unit/storage/pagination-parallel-hydration
+ * @description The canonical enumeration walk (getNounsWithPagination) hydrated
+ * each item's vector + metadata ONE-AT-A-TIME — N×per-op-latency serially, the
+ * dominant term in an index heal (cortex heal-cost decomposition). It now
+ * hydrates 16-way, and a new getNounIdsWithPagination returns ids WITHOUT
+ * hydration (zero per-entity reads when unfiltered). Both must preserve the exact
+ * pagination contract: same order, cursor continuation, filters, totalCount.
+ */
+import { describe, it, expect, beforeEach, vi } from 'vitest'
+import { Brainy, NounType } from '../../../src/index.js'
+
+describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => {
+  let brain: any
+  let storage: any
+  const N = 30
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, dimensions: 384 })
+    await brain.init()
+    for (let i = 0; i < N; i++) {
+      await brain.add({
+        data: `n${i}`,
+        type: i % 3 === 0 ? NounType.Task : NounType.Concept,
+        metadata: { i }
+      })
+    }
+    await brain.flush()
+    storage = brain.storage
+  })
+
+  /** Page the whole dataset through a small limit via cursor and collect ordered ids. */
+  const pageAll = async (fn: (opts: any) => Promise<any>, key: 'items' | 'ids') => {
+    const out: string[] = []
+    let cursor: string | undefined
+    for (let guard = 0; guard < 1000; guard++) {
+      const page = await fn({ limit: 4, cursor })
+      const batch = key === 'items' ? page.items.map((n: any) => n.id) : page.ids
+      out.push(...batch)
+      if (!page.hasMore) break
+      cursor = page.nextCursor
+    }
+    return out
+  }
+
+  it('parallel hydration yields the SAME ordered pages as one big page', async () => {
+    const big = await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
+    const bigIds = big.items.map((n: any) => n.id)
+    // At least the N we added (a brain also has its VFS root entity).
+    expect(bigIds.length).toBeGreaterThanOrEqual(N)
+    expect(big.totalCount).toBe(bigIds.length)
+
+    const paged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
+    expect(paged).toEqual(bigIds) // identical order, no dupes, no gaps across pages
+  })
+
+  it('getNounIdsWithPagination returns exactly the same ids, in the same order', async () => {
+    const idsPaged = await pageAll((o) => storage.getNounIdsWithPagination(o), 'ids')
+    const itemsPaged = await pageAll((o) => storage.getNounsWithPagination(o), 'items')
+    expect(idsPaged).toEqual(itemsPaged)
+    expect(new Set(idsPaged).size).toBe(idsPaged.length) // every id exactly once
+    expect(idsPaged.length).toBeGreaterThanOrEqual(N)
+  })
+
+  it('id-only enumeration does ZERO per-entity hydration reads when unfiltered', async () => {
+    const readSpy = vi.spyOn(storage as any, 'readCanonicalObject')
+    await storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
+    expect(readSpy).not.toHaveBeenCalled()
+    readSpy.mockRestore()
+
+    // The hydrating walk, by contrast, DOES read each entity.
+    const readSpy2 = vi.spyOn(storage as any, 'readCanonicalObject')
+    await storage.getNounsWithPagination({ limit: 1000, offset: 0 })
+    expect(readSpy2.mock.calls.length).toBeGreaterThan(0)
+    readSpy2.mockRestore()
+  })
+
+  it('a type filter matches between the hydrating and id-only walks', async () => {
+    const taskItems = await storage.getNounsWithPagination({
+      limit: 1000,
+      offset: 0,
+      filter: { nounType: NounType.Task }
+    })
+    const taskIds = await storage.getNounIdsWithPagination({
+      limit: 1000,
+      offset: 0,
+      filter: { nounType: NounType.Task }
+    })
+    const expected = Math.ceil(N / 3) // every 3rd is a Task
+    expect(taskItems.items.length).toBe(expected)
+    expect(new Set(taskIds.ids)).toEqual(new Set(taskItems.items.map((n: any) => n.id)))
+  })
+})

From 7692c6f4efab12c54be8c2a307e01b4bdb80c405 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 15:18:09 -0700
Subject: [PATCH 35/82] docs: RELEASES.md entry for 8.3.0 (heal-cost + ADR-004
 Pass 2/3)

Consumer summary of the 8.3.0 minor: 16-way parallel canonical enumeration +
id-only getNounIdsWithPagination (index-heal speedup, standalone); the ADR-004
cross-layer integrity contract (validateInvariants delegation + repairIndex
native rebuild); and the registered-blob family contract (declared index blobs
undeletable). The two contract pieces are inert until a native provider
implements the matching hooks.
---
 RELEASES.md | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index a90dcce4..446a2c09 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,39 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.3.0 — 2026-07-13 (faster index heals + the cross-layer integrity contract)
+
+Three additive changes. The first is an immediate, standalone performance win; the other two are the
+brainy side of the write/index-spine integrity contract (ADR-004), inert until a native accelerator
+that implements the matching hooks is present — so this release changes nothing for a JS-only brain
+beyond the speedup.
+
+- **Canonical enumeration is up to ~16× faster — the dominant term in an index heal.** The paginated
+  entity walk (`getNounsWithPagination`) hydrated each entity's vector + metadata one-at-a-time; since
+  every index rebuild enumerates canonical storage, that serial per-item latency dominated multi-minute
+  heals. Hydration is now 16-way bounded-concurrency, with the pagination contract (order, cursor
+  resume, filters, totalCount) byte-identical to before. New **`getNounIdsWithPagination()`** returns
+  ids without hydrating anything (zero per-entity reads when unfiltered) for callers that own their own
+  IO schedule.
+
+- **Cross-layer integrity — `validateIndexConsistency()` is no longer blind to native providers.** It
+  only ran the JS metadata index's own check, so a native provider whose manifest/segments/counts had
+  diverged still read as "healthy". It now feature-detects and aggregates each provider's optional
+  `validateInvariants()` self-report, names every failing invariant with its numbers, and exposes the
+  per-provider reports. `repairIndex()` now reconciles native derived state from canonical too
+  (rebuilding any provider whose failing invariant asks for it). New exported types
+  `ProviderInvariantReport` / `InvariantResult` / `InvariantHeal`.
+
+- **Registered-blob families — declared index files are undeletable through the storage layer.** A
+  provider can declare a derived-index blob *family* (a set of members that are load-bearing together);
+  once declared, `deleteBinaryBlob` / `removeRawPrefix` refuse to remove a member (new exported
+  `ProtectedArtifactError`), so a stray in-process sweeper cannot delete a load-bearing index file. The
+  declaration persists across reopen; `checkDerivedFamiliesPresent()` names any member missing on open.
+  New optional `StorageAdapter` surface (`registerDerivedFamily` / `unregisterDerivedFamily` /
+  `listDerivedFamilies` + `DerivedFamilyDeclaration`) and exported `DerivedArtifactMissingError`.
+
+No breaking API change (all additions are optional/new). Each change ships with regression tests.
+
 ## v8.2.8 — 2026-07-13 (honest index readiness — no more silently-empty queries on a cold index)
 
 Closes the last of the three spine anti-patterns: the "dishonest readiness proxy," where `size() > 0`

From c40a89e6492890aa86f31ca21f57d231e6974d0c Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Jul 2026 15:21:52 -0700
Subject: [PATCH 36/82] chore(release): 8.3.0

---
 CHANGELOG.md      | 8 ++++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 295891a2..97c5d059 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.3.0](https://github.com/soulcraftlabs/brainy/compare/v8.2.8...v8.3.0) (2026-07-13)
+
+- docs: RELEASES.md entry for 8.3.0 (heal-cost + ADR-004 Pass 2/3) (7692c6f)
+- perf: parallel + id-only canonical enumeration (heal-cost dominant term) (ec5b933)
+- feat: registered-blob family contract — declared index blobs are undeletable (ADR-004 Pass 2) (bfa1762)
+- feat: validateIndexConsistency delegates to provider invariants (ADR-004 Pass 3) (6bcb54f)
+
+
 ### [8.2.8](https://github.com/soulcraftlabs/brainy/compare/v8.2.7...v8.2.8) (2026-07-13)
 
 - fix: honest index readiness — no silently-empty queries on a cold index (d0f69c7)
diff --git a/package-lock.json b/package-lock.json
index b2b3ce8c..9b0c625b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.8",
+  "version": "8.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.2.8",
+      "version": "8.3.0",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 095737c4..d939ca3c 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.2.8",
+  "version": "8.3.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 1d26988963cc8d21d45a314091f1b3f9e51a0cdc Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Tue, 14 Jul 2026 10:11:41 -0700
Subject: [PATCH 37/82] docs: cite the cross-layer integrity contract
 generically in comments and notes

---
 CHANGELOG.md                                        |  6 +++---
 src/coreTypes.ts                                    |  4 ++--
 src/errors/brainyError.ts                           |  2 +-
 src/plugin.ts                                       | 12 ++++++------
 src/storage/adapters/memoryStorage.ts               |  2 +-
 tests/unit/storage/registered-blob-contract.test.ts |  4 ++--
 tests/unit/validate-invariants-delegation.test.ts   |  2 +-
 7 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 97c5d059..b3821c3b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,10 +4,10 @@ All notable changes to this project will be documented in this file. See [standa
 
 ### [8.3.0](https://github.com/soulcraftlabs/brainy/compare/v8.2.8...v8.3.0) (2026-07-13)
 
-- docs: RELEASES.md entry for 8.3.0 (heal-cost + ADR-004 Pass 2/3) (7692c6f)
+- docs: RELEASES.md entry for 8.3.0 (heal-cost + cross-layer integrity contract) (7692c6f)
 - perf: parallel + id-only canonical enumeration (heal-cost dominant term) (ec5b933)
-- feat: registered-blob family contract — declared index blobs are undeletable (ADR-004 Pass 2) (bfa1762)
-- feat: validateIndexConsistency delegates to provider invariants (ADR-004 Pass 3) (6bcb54f)
+- feat: registered-blob family contract — declared index blobs are undeletable (bfa1762)
+- feat: validateIndexConsistency delegates to provider invariants (6bcb54f)
 
 
 ### [8.2.8](https://github.com/soulcraftlabs/brainy/compare/v8.2.7...v8.2.8) (2026-07-13)
diff --git a/src/coreTypes.ts b/src/coreTypes.ts
index 4b1ebfb9..c345624b 100644
--- a/src/coreTypes.ts
+++ b/src/coreTypes.ts
@@ -755,7 +755,7 @@ export interface Change {
 }
 
 /**
- * @description A declared derived-index blob FAMILY (ADR-004 §7 — the
+ * @description A declared derived-index blob FAMILY (the
  * registered-blob contract). A family names the set of on-disk blobs that a
  * derived index needs AS A SET (e.g. the vector base = `main.dkann` +
  * `main.slotmap` + `main.slotrev`): losing ANY member corrupts the index. Once a
@@ -1083,7 +1083,7 @@ export interface StorageAdapter {
   getBinaryBlobPath(key: string): string | null
 
   /**
-   * @description OPTIONAL (ADR-004 §7 registered-blob contract). Declare a
+   * @description OPTIONAL (registered-blob contract). Declare a
    * derived-index blob {@link DerivedFamilyDeclaration | family} whose members
    * become UNDELETABLE through this adapter — a subsequent `deleteBinaryBlob` /
    * `removeRawPrefix` that would remove a declared member throws a
diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts
index 6aa1f919..a58236e3 100644
--- a/src/errors/brainyError.ts
+++ b/src/errors/brainyError.ts
@@ -311,7 +311,7 @@ export class VectorIndexNotReadyError extends BrainyError {
 
 /**
  * Thrown when a delete (`deleteBinaryBlob` / `removeRawPrefix`) would remove a
- * blob that is a declared member of a protected derived-index FAMILY (ADR-004 §7
+ * blob that is a declared member of a protected derived-index FAMILY (the
  * registered-blob contract). Declared derived artifacts are undeletable through
  * the storage layer — this makes an in-process GC / sweeper INCAPABLE of removing
  * a load-bearing index file (the lost-`main.dkann` class). Intentional retirement
diff --git a/src/plugin.ts b/src/plugin.ts
index c0fda10f..df7c7224 100644
--- a/src/plugin.ts
+++ b/src/plugin.ts
@@ -116,7 +116,7 @@ export interface BrainyPluginContext {
 // ===========================================================================
 
 /**
- * @description How a failed provider invariant should be remediated (ADR-004 §6):
+ * @description How a failed provider invariant should be remediated:
  * - `'none'`    — informational; the invariant held or nothing to do.
  * - `'repair'`  — a targeted, cheap fix exists (e.g. re-derive a count/manifest field).
  * - `'rebuild'` — the derived state must be rebuilt from canonical (`provider.rebuild()`).
@@ -124,7 +124,7 @@ export interface BrainyPluginContext {
 export type InvariantHeal = 'none' | 'repair' | 'rebuild'
 
 /**
- * @description The result of ONE provider invariant check (ADR-004 §6). A failure
+ * @description The result of ONE provider invariant check. A failure
  * (`holds === false`) NAMES what diverged, with numbers, so it is diagnosable
  * from the report alone — never a bare boolean. `name` is a stable kebab-case id
  * for telemetry / remediation routing.
@@ -146,7 +146,7 @@ export interface InvariantResult {
 
 /**
  * @description A provider's self-report of its own cross-layer invariants
- * (ADR-004 §6 — the `validateInvariants()` hook). Contract:
+ * (the `validateInvariants()` hook). Contract:
  * - It NEVER throws — a failure is DATA (`healthy: false` + a failing invariant),
  *   not an exception.
  * - It is BOUNDED (<50ms): residency checks + O(1) counts only, NO canonical
@@ -196,7 +196,7 @@ export interface MetadataIndexProvider {
   isReady?(): boolean
 
   /**
-   * @description OPTIONAL (ADR-004 §6). The provider's self-report of its own
+   * @description OPTIONAL. The provider's self-report of its own
    * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence).
    * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant).
    * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so
@@ -358,7 +358,7 @@ export interface GraphIndexProvider {
   isReady?(): boolean
 
   /**
-   * @description OPTIONAL (ADR-004 §6). The provider's self-report of its own
+   * @description OPTIONAL. The provider's self-report of its own
    * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence).
    * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant).
    * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so
@@ -1024,7 +1024,7 @@ export interface VectorIndexProvider {
   isReady?(): boolean
 
   /**
-   * @description OPTIONAL (ADR-004 §6). The provider's self-report of its own
+   * @description OPTIONAL. The provider's self-report of its own
    * cross-layer invariants (manifest ↔ segments ↔ counts residency/coherence).
    * MUST NOT throw — a failure is DATA (`healthy: false` + a failing invariant).
    * MUST be bounded (<50ms): residency + O(1) counts only, NO canonical walks, so
diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts
index 6b2564d5..41987ea0 100644
--- a/src/storage/adapters/memoryStorage.ts
+++ b/src/storage/adapters/memoryStorage.ts
@@ -183,7 +183,7 @@ export class MemoryStorage extends BaseStorage {
    * @param key - The blob key.
    */
   public async deleteBinaryBlob(key: string): Promise<void> {
-    // Registered-blob contract (ADR-004 §7) — parity with the filesystem adapter:
+    // Registered-blob contract — parity with the filesystem adapter:
     // a declared family member is undeletable (throws ProtectedArtifactError).
     await this.assertBlobKeyDeletable(key)
     this.blobStore.delete(key)
diff --git a/tests/unit/storage/registered-blob-contract.test.ts b/tests/unit/storage/registered-blob-contract.test.ts
index ef450163..6ad51ae7 100644
--- a/tests/unit/storage/registered-blob-contract.test.ts
+++ b/tests/unit/storage/registered-blob-contract.test.ts
@@ -1,6 +1,6 @@
 /**
  * @module tests/unit/storage/registered-blob-contract
- * @description Pass 2 (ADR-004 §7): declared derived-index blob FAMILIES are
+ * @description Declared derived-index blob FAMILIES are
  * undeletable through the storage layer — an in-process GC/sweeper cannot remove
  * a load-bearing index file (the lost-main.dkann class). Covers declare →
  * protected-delete-throws; unregister → delete-ok; transient passthrough;
@@ -15,7 +15,7 @@ import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
 import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
 import { ProtectedArtifactError } from '../../../src/index.js'
 
-describe('registered-blob family contract (Pass 2, ADR-004 §7)', () => {
+describe('registered-blob family contract', () => {
   let storage: any
 
   beforeEach(async () => {
diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts
index 953c6b91..45e12ccd 100644
--- a/tests/unit/validate-invariants-delegation.test.ts
+++ b/tests/unit/validate-invariants-delegation.test.ts
@@ -1,6 +1,6 @@
 /**
  * @module tests/unit/validate-invariants-delegation
- * @description Pass 3 (ADR-004 §6): validateIndexConsistency() was blind to native
+ * @description validateIndexConsistency() was blind to native
  * providers — it only saw the JS metadata index, so a native manifest↔segments↔count
  * divergence read as "healthy". It now feature-detects + aggregates each provider's
  * validateInvariants(), and repairIndex() maps a failing invariant with heal:'rebuild'

From 366f9a91f579cd7ea75689fd5ed496d7ea056a67 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Tue, 14 Jul 2026 10:11:53 -0700
Subject: [PATCH 38/82] fix: full-removal canonical deletes + family-scoped
 migration gate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two production-reported spine fixes:

- Canonical delete is a FULL removal: remove()/removeMany() deleted the
  metadata leg but never the canonical vectors.json leg or the <id>/
  directory, leaving ghost rows that inflated enumerated counts forever,
  read as damage scars, and caused duplicate readdir entries for
  re-created VFS paths (the stale-unpost path). The delete operation now
  routes through storage.deleteNoun — both legs + the entity container —
  with a full two-leg before-image rollback; deleteVerb symmetric; the
  blind 'no metadata file' catch that masked real faults is gone. The
  generation log still holds the delete's before-image, so asOf() history
  is unchanged. repairIndex() gains a conservative, loud orphan-prune
  sweep (containers with no metadata content leg) + count recompute for
  stores damaged by earlier versions.

- Family-scoped migration gate: every read blocked on the whole-brain
  migration lock even when the migrating index was irrelevant. The gate
  now scopes to the index families a read actually consults — canonical
  reads never wait; find() waits only on its query shape's families;
  graph traversals wait only on the graph family. Writes keep the
  conservative whole-brain wait. A read that needs the migrating family
  still blocks (bounded) with the retryable MigrationInProgressError.
---
 src/brainy.ts                                 | 171 +++++++++++++++---
 src/storage/adapters/fileSystemStorage.ts     | 104 ++++++++++-
 src/storage/baseStorage.ts                    |  66 ++++---
 .../operations/StorageOperations.ts           |  44 ++++-
 tests/integration/delete-full-removal.test.ts | 161 +++++++++++++++++
 .../readdir-no-duplicate-replace.test.ts      |  56 ++++++
 .../migration-gate-family-scoped.test.ts      |  94 ++++++++++
 tests/unit/migration-lock.test.ts             |  27 ++-
 8 files changed, 657 insertions(+), 66 deletions(-)
 create mode 100644 tests/integration/delete-full-removal.test.ts
 create mode 100644 tests/integration/readdir-no-duplicate-replace.test.ts
 create mode 100644 tests/unit/brainy/migration-gate-family-scoped.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index 0a2bd460..9344ff0c 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -367,6 +367,15 @@ class InsertPreconditionExistsSignal extends Error {
   }
 }
 
+/**
+ * @description The derived-index families a read may depend on. A read that
+ * consults none of them (a canonical-storage read: `get`, an entity
+ * enumeration, a VFS content/dir read) is index-independent and must never
+ * block on another family's one-time migration. Used by the family-scoped
+ * migration gate ({@link Brainy.awaitMigrationLock}).
+ */
+export type IndexFamily = 'vector' | 'metadata' | 'graph'
+
 /**
  * The main Brainy class - Clean, Beautiful, Powerful
  * REAL IMPLEMENTATION - No stubs, no mocks
@@ -1439,7 +1448,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * re-initializing — using a closed Brainy is a consumer bug, not a lazy-init
    * opportunity.
    */
-  private async ensureInitialized(opts?: { bypassMigrationLock?: boolean }): Promise<void> {
+  private async ensureInitialized(opts?: {
+    bypassMigrationLock?: boolean
+    needs?: IndexFamily[]
+  }): Promise<void> {
     if (this.closed) {
       throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.')
     }
@@ -1449,12 +1461,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // Coordinated migration LOCK (#18): every data-plane read and write funnels
     // through here, so this is the single choke point that holds operations while
     // a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no
-    // op touches a half-built index. Observability (`health`/`checkHealth`) and
-    // the lock-clearing path (`stampBrainFormat`, which does not route through
-    // here) opt out so an operator can always watch progress and cor can stamp.
+    // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the
+    // derived-index families this operation actually consults, so a read served
+    // entirely from canonical storage (`needs: []`) or from a healthy family
+    // never blocks on an UNRELATED family's migration. `needs` omitted = the
+    // conservative whole-brain wait (writes, and any read not yet classified).
+    // Observability (`health`/`checkHealth`) and the lock-clearing path
+    // (`stampBrainFormat`, which does not route through here) opt out entirely so
+    // an operator can always watch progress and a native provider can stamp.
     // A brain that never migrates pays one boolean check (see awaitMigrationLock).
     if (!opts?.bypassMigrationLock) {
-      await this.awaitMigrationLock()
+      await this.awaitMigrationLock(opts?.needs)
     }
   }
 
@@ -2188,7 +2205,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    *
    */
   async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
-    await this.ensureInitialized()
+    // Canonical read: a get resolves an entity by id straight from storage and
+    // consults no derived index — it must not wait on any family's migration.
+    await this.ensureInitialized({ needs: [] })
     this.warnIfReadsDegraded('get')
 
     // Id normalization (8.0): a caller may read by their natural key — resolve
@@ -2247,7 +2266,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * ```
    */
   async batchGet(ids: string[], options?: GetOptions): Promise<Map<string, Entity<T>>> {
-    await this.ensureInitialized()
+    // Canonical read (see get): resolves by id from storage, no derived index.
+    await this.ensureInitialized({ needs: [] })
 
     // Id normalization (8.0): resolve each id to its canonical UUID so callers
     // can batch-read by natural key. resolveEntityId is idempotent on real
@@ -4271,7 +4291,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   async related(
     paramsOrId?: string | RelatedParams
   ): Promise<Relation<T>[]> {
-    await this.ensureInitialized()
+    // Graph read: relationship traversal consults only the graph adjacency
+    // family, so it waits on a graph migration but not on vector/metadata.
+    await this.ensureInitialized({ needs: ['graph'] })
     await this.verifyGraphAdjacencyLive()
 
     // Handle string ID shorthand: related(id) -> related({ from: id })
@@ -5810,7 +5832,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * })
    */
   async find(query: string | FindParams<T>): Promise<Result<T>[]> {
-    await this.ensureInitialized()
+    // Init only here — the migration gate is applied family-scoped below, once
+    // the query params reveal which index families this read actually consults
+    // (a string query may parse to a where / connected / vector shape). The lazy
+    // loader and cold-read probes below already defer to a migrating provider.
+    await this.ensureInitialized({ needs: [] })
 
     // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true)
     // This is a production-safe, concurrency-controlled lazy load
@@ -5849,6 +5875,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // Zero-config validation (static import for performance)
     validateFindParams(params)
 
+    // Family-scoped migration gate: wait only on the index families THIS query
+    // consults, so a filter or graph query served from a healthy family never
+    // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once
+    // params are final (after natural-language parse + connected-id resolution)
+    // and before the aggregate path, which carries no gate of its own.
+    await this.awaitMigrationLock(this.queryIndexFamilies(params))
+
     // Aggregate query path — early return when params.aggregate is set
     if (params.aggregate) {
       return this.findAggregate(params)
@@ -12530,14 +12563,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     entityCount: number
     indexEntryCount: number
     recommendation: string | null
-    /** Cross-layer (ADR-004 §6): each provider's own invariant self-report, when
+    /** Cross-layer: each provider's own invariant self-report, when
      *  it exposes validateInvariants(). Absent providers are simply not listed. */
     providers?: ProviderInvariantReport[]
   }> {
     await this.ensureInitialized()
     const result = await this.metadataIndex.validateConsistency()
 
-    // Cross-layer integrity (ADR-004 §6): validateConsistency() above only sees the
+    // Cross-layer integrity: validateConsistency() above only sees the
     // JS metadata index — it is BLIND to a native provider whose manifest ↔
     // segments ↔ counts have diverged. Delegate to each provider's own
     // validateInvariants() and aggregate, so "healthy-while-broken" is impossible.
@@ -12583,7 +12616,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
 
   /**
    * @description Feature-detect + call each index provider's OPTIONAL
-   * `validateInvariants()` (ADR-004 §6) and collect the reports. The contract is
+   * `validateInvariants()` and collect the reports. The contract is
    * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a
    * provider violates that, the throw is turned into an UNHEALTHY synthetic
    * report (loud), never swallowed into "healthy". Providers that do not expose
@@ -12673,7 +12706,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       limit?: number
     }
   ): Promise<string[]> {
-    await this.ensureInitialized()
+    // Graph read (see related): adjacency traversal only.
+    await this.ensureInitialized({ needs: ['graph'] })
     await this.verifyGraphAdjacencyLive()
 
     const direction = options?.direction || 'both'
@@ -14844,6 +14878,58 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     )
   }
 
+  /**
+   * @description The provider instance backing a given index {@link IndexFamily}.
+   * The single place the family label maps to the concrete provider, so the
+   * scoped migration gate and any future family-aware routing agree.
+   */
+  private providerForFamily(family: IndexFamily): unknown {
+    switch (family) {
+      case 'vector':
+        return this.index
+      case 'metadata':
+        return this.metadataIndex
+      case 'graph':
+        return this.graphIndex
+    }
+  }
+
+  /**
+   * @description Whether the migration LOCK should hold for an operation that
+   * depends on the given index families:
+   * - `needs === undefined` → WHOLE-BRAIN: any migrating provider counts (the
+   *   conservative default for writes and unclassified reads — a write touches
+   *   every index).
+   * - `needs === []` → NEVER: a canonical-storage read consults no derived index,
+   *   so a migration elsewhere is irrelevant; it serves immediately.
+   * - `needs = [families]` → SCOPED: only those families' providers count, so a
+   *   read served from a healthy family is not blocked by an unrelated family's
+   *   one-time migration.
+   */
+  private neededFamiliesMigrating(needs?: IndexFamily[]): boolean {
+    if (needs === undefined) return this.anyProviderMigrating()
+    for (const family of needs) {
+      if (this.providerIsMigrating(this.providerForFamily(family))) return true
+    }
+    return false
+  }
+
+  /**
+   * @description The index families a {@link find} query consults, derived from
+   * its params — the read-side input to the family-scoped migration gate. A
+   * vector / near / semantic query needs `vector`; a `where` / type / subtype /
+   * service filter or an aggregate needs `metadata`; a `connected` traversal
+   * needs `graph`. A query combining shapes needs the union. Mirrors find()'s
+   * own criteria split so the gate and the executor never disagree.
+   */
+  private queryIndexFamilies(params: FindParams<T>): IndexFamily[] {
+    const needs: IndexFamily[] = []
+    if ((params.query && params.query.trim() !== '') || params.vector || params.near) needs.push('vector')
+    if (params.where || params.type || params.subtype || params.service || params.aggregate) needs.push('metadata')
+    if (params.connected) needs.push('graph')
+    return needs
+  }
+
   /**
    * @description Rich migration progress relayed verbatim from whichever provider
    * exposes the OPTIONAL `migrationStatus(): { phase?, index?, percent?, … } | null`.
@@ -14872,11 +14958,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   /**
    * @description Block-and-queue on the coordinated migration LOCK (#18). Called
    * at the {@link ensureInitialized} choke point (and once inside `init()` before
-   * VFS bootstrap), so every data-plane read and write — and startup itself —
-   * waits here while a native provider runs its one-time 7.x → 8.0
-   * rebuild-from-canonical. The caller gets the correct answer, never a partial
-   * read of a half-built index and never a lost write. A brain that is not
-   * migrating pays a single boolean check and returns immediately.
+   * VFS bootstrap), so a data-plane operation waits here while a native provider
+   * runs its one-time 7.x → 8.0 rebuild-from-canonical. The caller gets the
+   * correct answer, never a partial read of a half-built index and never a lost
+   * write. A brain that is not migrating pays a single boolean check and returns
+   * immediately.
+   *
+   * FAMILY-SCOPED: `needs` names the index families the caller depends on, so the
+   * wait holds ONLY for a migration of one of those families. A read served
+   * entirely from canonical storage (`needs: []`) or from a healthy family never
+   * blocks on an unrelated family's migration. `needs` omitted = the conservative
+   * whole-brain wait (writes touch every index; startup wants all of them ready).
+   * @param needs - Index families this operation consults, or `undefined` for the
+   *   whole-brain wait.
    *
    * IMPORTANT: this timeout bounds THE CALLER'S WAIT, not the migration. The
    * migration itself is unbounded — the native provider rebuilds a billion-scale
@@ -14890,8 +14984,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * for a very large brain, runs the offline migrator. The block/release lines
    * log once per window; the flags reset on release so a later upgrade re-logs.
    */
-  private async awaitMigrationLock(): Promise<void> {
-    if (!this.anyProviderMigrating()) return // fast path — the overwhelming common case
+  private async awaitMigrationLock(needs?: IndexFamily[]): Promise<void> {
+    if (!this.neededFamiliesMigrating(needs)) return // fast path — the overwhelming common case
 
     const startedAt = Date.now()
     const timeoutMs = this.config.migrationWaitTimeoutMs ?? 30_000
@@ -14904,7 +14998,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       )
     }
 
-    while (this.anyProviderMigrating()) {
+    while (this.neededFamiliesMigrating(needs)) {
       const remaining = timeoutMs - (Date.now() - startedAt)
       if (remaining <= 0) {
         const status = this.providerMigrationStatus()
@@ -15092,6 +15186,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    */
   private async ensureMetadataConsistencyProbed(): Promise<void> {
     if (this._metadataConsistencyProbed) return
+    // Defer while the metadata provider runs its one-time in-place migration:
+    // probing (and self-healing via rebuild) an index the provider is mid-rebuild
+    // would collide with the provider that owns it. Mirrors the vector deference
+    // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next
+    // read runs the probe. (The family-scoped find() gate waits on the metadata
+    // family separately before any actual filter read.)
+    if (this.providerIsMigrating(this.metadataIndex)) return
     this._metadataConsistencyProbed = true
     const provider = this.metadataIndex as {
       probeConsistency?: () => Promise<boolean>
@@ -15161,6 +15262,32 @@ export class Brainy<T = any> implements BrainyInterface<T> {
 
   async repairIndex(): Promise<void> {
     await this.ensureInitialized()
+
+    // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete
+    // defect: a delete that removed the metadata (content) leg but left the
+    // vector leg + the entity directory (a "ghost"), or left an empty directory
+    // (a "scar"). These are not live entities (getNoun needs the content leg)
+    // yet inflate enumerated counts and confuse locator resolution. Feature-
+    // detected (filesystem-only — key/prefix stores have no orphan containers);
+    // recompute counts afterward so the totals stop counting the ghosts.
+    const pruner = this.storage as {
+      pruneOrphanedEntities?: () => Promise<{ nouns: string[]; verbs: string[] }>
+      rebuildTypeCounts?: () => Promise<void>
+      rebuildSubtypeCounts?: () => Promise<void>
+    }
+    if (typeof pruner.pruneOrphanedEntities === 'function') {
+      const orphans = await pruner.pruneOrphanedEntities()
+      if (orphans.nouns.length + orphans.verbs.length > 0) {
+        await pruner.rebuildTypeCounts?.()
+        await pruner.rebuildSubtypeCounts?.()
+        prodLog.warn(
+          `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` +
+            `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` +
+            `partial delete, and recomputed counts.`
+        )
+      }
+    }
+
     await this.metadataIndex.detectAndRepairCorruption()
     // Lift a failed-rollback write-quarantine: force a full rebuild so the
     // derived indexes are provably reconciled with canonical, then clear the
@@ -15175,7 +15302,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         `Writes are re-enabled.`
       )
     }
-    // Cross-layer repair (ADR-004 §6): repairIndex must reconcile NATIVE derived
+    // Cross-layer repair: repairIndex must reconcile NATIVE derived
     // state from canonical, not just the JS metadata index. Consult each provider's
     // own validateInvariants() and rebuild any whose failing invariant asks for it
     // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption().
diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index c2352d2f..a9e3014c 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -502,6 +502,104 @@ export class FileSystemStorage extends BaseStorage {
     this.writeBarrierDeleteDirs?.add(path.dirname(pathStr))
   }
 
+  /**
+   * @description Remove the entity container directory after both canonical legs
+   * were deleted (full-removal delete). `objectLegPath` is one leg's
+   * storage-root-relative path (e.g. `entities/nouns/<shard>/<id>/vectors.json`);
+   * its parent is the `<id>/` entity directory. `rmdir` removes it ONLY if empty
+   * — both legs are gone by this point, so it should be. A non-empty dir means an
+   * unexpected leftover (a leg whose delete faulted — already surfaced upstream —
+   * or foreign data): we do NOT recursively nuke it (loud errors, never quiet
+   * losses); we log and leave it for the orphan repair sweep (`repairIndex`).
+   */
+  protected override async removeCanonicalContainer(objectLegPath: string): Promise<void> {
+    const relDir = path.dirname(objectLegPath)
+    const absDir = path.join(this.rootDir, relDir)
+    try {
+      await fs.promises.rmdir(absDir)
+      // The dir removal is durable once its PARENT (the shard dir) is fsync'd.
+      this.writeBarrierDeleteDirs?.add(path.dirname(relDir))
+    } catch (error: any) {
+      if (error?.code === 'ENOENT') return // container already gone — fine
+      if (error?.code === 'ENOTEMPTY' || error?.code === 'EEXIST') {
+        console.warn(
+          `[FileSystemStorage] entity container ${relDir} not empty after delete — ` +
+            `leaving it for the orphan repair sweep (brain.repairIndex()).`
+        )
+        return
+      }
+      throw error
+    }
+  }
+
+  /**
+   * @description Prune orphaned entity directories left by the pre-8.3.1
+   * partial-delete defect. A delete used to remove the metadata (content) leg
+   * but leave the `vectors.json` leg + the `<id>/` directory (a "ghost"), or —
+   * on the direct storage path — remove both legs but leave an empty directory
+   * (a "scar"). Neither is a live entity (`getNoun` needs the metadata content
+   * leg) yet each lingers on disk, inflating enumerated counts and confusing
+   * locator resolution.
+   *
+   * CONSERVATIVE by design: removes ONLY dirs with NO metadata content leg — a
+   * directory that still holds its content is left untouched. LOUD: logs every
+   * removed orphan. Operator-invoked via {@link Brainy.repairIndex}; never runs
+   * automatically. Returns the pruned container ids so the caller can recompute
+   * counts.
+   */
+  public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
+    await this.ensureInitialized()
+    const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
+
+    for (const [kind, root] of [
+      ['nouns', 'entities/nouns'],
+      ['verbs', 'entities/verbs']
+    ] as const) {
+      const rootAbs = path.join(this.rootDir, root)
+      let shards: string[]
+      try {
+        shards = await fs.promises.readdir(rootAbs)
+      } catch (error: any) {
+        if (error?.code === 'ENOENT') continue // no entities of this kind yet
+        throw error
+      }
+
+      for (const shard of shards) {
+        const shardAbs = path.join(rootAbs, shard)
+        let entries: import('fs').Dirent[]
+        try {
+          entries = await fs.promises.readdir(shardAbs, { withFileTypes: true })
+        } catch (error: any) {
+          if (error?.code === 'ENOENT') continue
+          throw error
+        }
+
+        for (const entry of entries) {
+          if (!entry.isDirectory()) continue
+          const idAbs = path.join(shardAbs, entry.name)
+          let legs: string[]
+          try {
+            legs = await fs.promises.readdir(idAbs)
+          } catch (error: any) {
+            if (error?.code === 'ENOENT') continue
+            throw error
+          }
+          // A live entity has its metadata content leg. No content leg → a
+          // vector-only ghost or an empty scar → prune the whole container.
+          if (legs.some((f) => f.startsWith('metadata.json'))) continue
+          await fs.promises.rm(idAbs, { recursive: true, force: true })
+          pruned[kind].push(entry.name)
+          console.warn(
+            `[FileSystemStorage] pruned orphaned ${kind === 'nouns' ? 'noun' : 'verb'} ` +
+              `container ${root}/${shard}/${entry.name} (no metadata content leg)`
+          )
+        }
+      }
+    }
+
+    return pruned
+  }
+
   /**
    * Primitive operation: List objects under path prefix
    * All metadata operations use this internally via base class routing
@@ -631,7 +729,7 @@ export class FileSystemStorage extends BaseStorage {
   public override async removeRawPrefix(prefix: string): Promise<void> {
     await this.ensureInitialized()
     // Registered-blob contract: a prefix-nuke must not take out a protected
-    // family member (ADR-004 §7). Throws if the prefix intersects one.
+    // family member. Throws if the prefix intersects one.
     await this.assertPrefixNotProtected(prefix)
     await fs.promises.rm(path.join(this.rootDir, prefix), { recursive: true, force: true })
   }
@@ -1240,7 +1338,7 @@ export class FileSystemStorage extends BaseStorage {
       // errors, never quiet losses. Fault-propagation restored lockstep with
       // cortex 3.0.13, whose two column-store call sites now handle the throw
       // (they mark the field unavailable + throw a named error) instead of
-      // relying on null-on-error (CORTEX-BILLION-SCALE-SPINE).
+      // relying on null-on-error.
       if (isAbsentError(err)) return null
       throw err
     }
@@ -1253,7 +1351,7 @@ export class FileSystemStorage extends BaseStorage {
    */
   public async deleteBinaryBlob(key: string): Promise<void> {
     await this.ensureInitialized()
-    // Registered-blob contract (ADR-004 §7): refuse to delete a declared
+    // Registered-blob contract: refuse to delete a declared
     // derived-index family member — an in-process GC/sweeper cannot remove a
     // load-bearing index file. Throws ProtectedArtifactError; no-op when no
     // families are registered.
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 5fdf129f..f5efdcb6 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -253,7 +253,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   /** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
   private _graphFastPathProbed = false
   /**
-   * Registered-blob contract (ADR-004 §7): declared derived-index families, keyed
+   * Registered-blob contract: declared derived-index families, keyed
    * by name. Members are undeletable through the blob delete seams. Loaded lazily
    * from `_system/derived-artifacts.json` and re-persisted on every change.
    */
@@ -842,6 +842,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     return this.deleteObjectFromPath(path)
   }
 
+  /**
+   * @description Remove the container that held a canonical entity's leg files,
+   * called after both legs are deleted so a delete leaves NOTHING behind — no
+   * orphan "scar" directory. `objectLegPath` is any one of the entity's leg
+   * paths (e.g. its `vectors.json`); the container is that path's parent.
+   *
+   * Default: a no-op. Key/prefix-addressed stores (in-memory, cloud object
+   * stores) have no empty-container concept — deleting the leg keys already
+   * removes the entity entirely. The filesystem adapter overrides this to
+   * `rmdir` the emptied entity directory.
+   *
+   * @param objectLegPath - Storage-root-relative path of one of the entity's legs.
+   * @protected
+   */
+  protected async removeCanonicalContainer(objectLegPath: string): Promise<void> {
+    void objectLegPath
+  }
+
   /**
    * @description List canonical objects under a storage-root-relative prefix.
    *
@@ -1062,7 +1080,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   }
 
   // ==========================================================================
-  // Registered-blob contract (ADR-004 §7) — declared derived-index families are
+  // Registered-blob contract — declared derived-index families are
   // undeletable through the blob delete seams. Shared here so every adapter that
   // extends BaseStorage inherits the same enforcement; the concrete
   // deleteBinaryBlob / removeRawPrefix call assertBlobKeyDeletable /
@@ -1116,7 +1134,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   /**
    * @description A blob key that is a transient write-scratch file (a `*.tmp.*`
    * temp, a `*.rebuild-tmp`, a `*.rotate-tmp`). Its OWNER renames/removes it as
-   * part of an atomic write; it is never a protected family member (ADR-004 §7).
+   * part of an atomic write; it is never a protected family member.
    */
   private isTransientBlobKey(key: string): boolean {
     return /\.rebuild-tmp$|\.rotate-tmp$|\.tmp(\.|$)/.test(key)
@@ -1185,7 +1203,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
   /**
    * @description Verify every declared family has all its members present on
-   * disk (ADR-004 §7 missing-on-open catch for an EXTERNAL deleter that bypasses
+   * disk (missing-on-open catch for an EXTERNAL deleter that bypasses
    * the in-process refusal). Returns the incomplete families (name + missing
    * members) — the caller decides how to heal (rebuild from canonical). Loud by
    * construction: a missing load-bearing blob is named, never silently tolerated.
@@ -1601,16 +1619,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   public async deleteNoun(id: string): Promise<void> {
     await this.ensureInitialized()
 
-    // Delete both the vector file and metadata file (2-file system)
-    await this.deleteNoun_internal(id)
+    // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
+    // entity's container, so a delete leaves nothing behind — no orphan/scar
+    // directory to inflate the enumerated count or ghost resolveLocator. The
+    // generation store holds the immutable before-image, so asOf() still
+    // reconstructs the deleted entity until retention expires.
+    await this.deleteNoun_internal(id) // vectors.json leg
 
-    // Delete metadata file (if it exists)
-    try {
-      await this.deleteNounMetadata(id)
-    } catch (error) {
-      // Ignore if metadata file doesn't exist
-      prodLog.debug(`No metadata file to delete for noun ${id}`)
-    }
+    // Metadata leg + count decrement. A genuine "already absent" is a no-op
+    // downstream (readCanonicalObject / deleteObjectFromPath both treat ENOENT as
+    // success); a REAL fault must surface loudly (loud errors, never quiet
+    // losses) and must never silently skip the count decrement — so this is NO
+    // LONGER wrapped in a blind catch that masked faults as "file didn't exist".
+    await this.deleteNounMetadata(id)
+
+    // Remove the now-empty entity container (a no-op for key/prefix stores).
+    await this.removeCanonicalContainer(getNounVectorPath(id))
   }
 
   /**
@@ -2915,16 +2939,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   public async deleteVerb(id: string): Promise<void> {
     await this.ensureInitialized()
 
-    // Delete both the vector file and metadata file (2-file system)
-    await this.deleteVerb_internal(id)
-
-    // Delete metadata file (if it exists)
-    try {
-      await this.deleteVerbMetadata(id)
-    } catch (error) {
-      // Ignore if metadata file doesn't exist
-      prodLog.debug(`No metadata file to delete for verb ${id}`)
-    }
+    // FULL removal (see deleteNoun): both canonical legs + the verb container.
+    // The blind catch that masked real faults as "no metadata file" is gone — a
+    // genuine absence is already a no-op downstream; a real fault surfaces loudly.
+    await this.deleteVerb_internal(id) // vectors.json leg
+    await this.deleteVerbMetadata(id)  // metadata leg + count decrement
+    await this.removeCanonicalContainer(getVerbVectorPath(id))
   }
   /**
    * Get graph index (lazy initialization with concurrent access protection)
diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts
index ca513cd4..bd753552 100644
--- a/src/transaction/operations/StorageOperations.ts
+++ b/src/transaction/operations/StorageOperations.ts
@@ -104,13 +104,26 @@ export class SaveNounOperation implements Operation {
 }
 
 /**
- * Delete noun metadata with rollback support
+ * Delete a noun — FULL canonical removal, with rollback support.
+ *
+ * Despite the historical name, this removes the WHOLE entity: both canonical
+ * legs (metadata + vector) AND the entity's container. Previously it deleted
+ * only the metadata leg via `deleteNounMetadata`, leaving the canonical
+ * `vectors.json` leg and the `<id>/` directory orphaned on disk — a "ghost"
+ * that reads as absent (getNoun needs both legs) yet inflates the enumerated
+ * count and can never be told apart from a damage scar. Routing through
+ * `storage.deleteNoun` removes both legs + the container in one place.
+ *
+ * Immutability is preserved: this cleans only the live-HEAD projection; the
+ * generation store retains the before-image so `asOf()` still reconstructs the
+ * deleted entity until retention expires.
  *
  * Rollback strategy:
- * - Restore deleted metadata
+ * - Restore BOTH legs from the before-image (vector leg raw, metadata leg via
+ *   the count-aware save so deleteNoun()'s decrement is reversed).
  */
 export class DeleteNounMetadataOperation implements Operation {
-  readonly name = 'DeleteNounMetadata'
+  readonly name = 'DeleteNoun'
 
   constructor(
     private readonly storage: StorageAdapter,
@@ -118,21 +131,34 @@ export class DeleteNounMetadataOperation implements Operation {
   ) {}
 
   async execute(): Promise<RollbackAction> {
-    // Get metadata before deletion (for rollback)
+    // Capture the FULL before-image (both legs) so the undo restores the whole
+    // entity — a metadata-only rollback would leave the vector leg unrestored.
+    const previousNoun = await this.storage.getNoun(this.id)
     const previousMetadata = await this.storage.getNounMetadata(this.id)
 
-    if (!previousMetadata) {
+    if (!previousNoun && !previousMetadata) {
       // Nothing to delete - no rollback needed
       return async () => {}
     }
 
-    // Delete metadata
-    await this.storage.deleteNounMetadata(this.id)
+    // Full removal: both canonical legs + the entity container + count decrement.
+    await this.storage.deleteNoun(this.id)
 
     // Return rollback action
     return async () => {
-      // Restore deleted metadata
-      await this.storage.saveNounMetadata(this.id, previousMetadata)
+      // Restore the vector leg, then the metadata leg through the count-aware
+      // save so deleteNoun()'s decrement is reversed.
+      if (previousNoun) {
+        await this.storage.saveNoun({
+          id: previousNoun.id,
+          vector: previousNoun.vector,
+          connections: previousNoun.connections || new Map(),
+          level: previousNoun.level || 0
+        })
+      }
+      if (previousMetadata) {
+        await this.storage.saveNounMetadata(this.id, previousMetadata)
+      }
     }
   }
 }
diff --git a/tests/integration/delete-full-removal.test.ts b/tests/integration/delete-full-removal.test.ts
new file mode 100644
index 00000000..83675a13
--- /dev/null
+++ b/tests/integration/delete-full-removal.test.ts
@@ -0,0 +1,161 @@
+/**
+ * @module tests/integration/delete-full-removal
+ * @description Canonical noun/verb deletes are FULL removals: both legs
+ * (metadata + vectors) AND the entity's `<id>/` container are removed, so a
+ * delete leaves NOTHING behind. Regression for the ghost/scar defect where
+ * remove() deleted the vector INDEX entry + the canonical metadata leg but never
+ * the canonical vectors.json leg or the directory — leaving an orphan that reads
+ * as absent (getNoun needs both legs) yet inflated the enumerated count and
+ * confused locator resolution. Also covers the operator repair sweep
+ * (brain.repairIndex()) that prunes orphans left by the pre-fix behavior.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+
+/** All entity-directory names (the `<id>` dirs) under entities/<kind>. */
+function entityDirs(root: string, kind: 'nouns' | 'verbs'): string[] {
+  const base = path.join(root, 'entities', kind)
+  if (!fs.existsSync(base)) return []
+  const ids: string[] = []
+  for (const shard of fs.readdirSync(base)) {
+    const shardDir = path.join(base, shard)
+    if (!fs.statSync(shardDir).isDirectory()) continue
+    for (const id of fs.readdirSync(shardDir)) {
+      if (fs.statSync(path.join(shardDir, id)).isDirectory()) ids.push(id)
+    }
+  }
+  return ids
+}
+
+/** Absolute path of one entity's `<id>` directory (or null if not present). */
+function entityDir(root: string, kind: 'nouns' | 'verbs', id: string): string | null {
+  const base = path.join(root, 'entities', kind)
+  if (!fs.existsSync(base)) return null
+  for (const shard of fs.readdirSync(base)) {
+    const candidate = path.join(base, shard, id)
+    if (fs.existsSync(candidate)) return candidate
+  }
+  return null
+}
+
+describe('canonical delete is a full removal (no ghost/scar directory)', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-del-'))
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('remove() deletes BOTH legs and the container — nothing left on disk', async () => {
+    const ids: string[] = []
+    for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } }))
+    await brain.flush()
+
+    // (A filesystem brain also has its VFS-root noun, so don't assume an exact set.)
+    const before = entityDirs(dir, 'nouns')
+    expect(before).toEqual(expect.arrayContaining(ids))
+
+    await brain.remove(ids[1])
+    await brain.flush()
+
+    // getNoun is null AND the on-disk container is fully gone (no orphan), and
+    // EXACTLY one directory disappeared (the removed entity's).
+    expect(await brain.get(ids[1])).toBeNull()
+    expect(entityDir(dir, 'nouns', ids[1])).toBeNull()
+    const after = entityDirs(dir, 'nouns')
+    expect(after).toHaveLength(before.length - 1)
+    expect(after).toEqual(expect.arrayContaining([ids[0], ids[2]]))
+    expect(after).not.toContain(ids[1])
+  })
+
+  it('the enumerated count is honest after a delete (no monotonic inflation)', async () => {
+    const ids: string[] = []
+    for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `n${i}`, type: 'document', metadata: { i } }))
+    await brain.flush()
+
+    const before = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 })
+
+    await brain.remove(ids[0])
+    await brain.remove(ids[1])
+    await brain.flush()
+
+    // The count drops by EXACTLY the two removed entities — no ghost lingers to
+    // hold the total up (the pre-fix defect left it monotonic).
+    const after = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 })
+    expect(after.totalCount).toBe(before.totalCount - 2)
+    const remaining = after.items.map((n: any) => n.id)
+    expect(remaining).toEqual(expect.arrayContaining([ids[2], ids[3]]))
+    expect(remaining).not.toContain(ids[0])
+    expect(remaining).not.toContain(ids[1])
+  })
+})
+
+describe('repairIndex() prunes orphan containers left by the pre-fix partial delete', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-orphan-'))
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('a vector-only "ghost" (metadata leg deleted out-of-band) is pruned', async () => {
+    const ids: string[] = []
+    for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `g${i}`, type: 'document', metadata: { i } }))
+    await brain.flush()
+
+    // Simulate the pre-fix defect on ids[0]: remove ONLY its metadata leg,
+    // leaving vectors.json + the directory (the exact ghost shape).
+    const ghostDir = entityDir(dir, 'nouns', ids[0])!
+    for (const f of fs.readdirSync(ghostDir)) {
+      if (f.startsWith('metadata.json')) fs.rmSync(path.join(ghostDir, f))
+    }
+    // The ghost dir (vectors.json, no metadata content leg) still sits on disk.
+    expect(entityDir(dir, 'nouns', ids[0])).not.toBeNull()
+
+    await brain.repairIndex()
+
+    // The sweep removes the ghost container; the two healthy entities survive.
+    expect(entityDir(dir, 'nouns', ids[0])).toBeNull()
+    expect(entityDir(dir, 'nouns', ids[1])).not.toBeNull()
+    expect(entityDir(dir, 'nouns', ids[2])).not.toBeNull()
+  })
+
+  it('an empty "scar" directory is pruned', async () => {
+    const id = await brain.add({ data: 'lonely', type: 'document', metadata: {} })
+    await brain.flush()
+    const scarDir = entityDir(dir, 'nouns', id)!
+    for (const f of fs.readdirSync(scarDir)) fs.rmSync(path.join(scarDir, f)) // empty the dir, keep it
+    expect(fs.existsSync(scarDir)).toBe(true)
+
+    await brain.repairIndex()
+
+    expect(fs.existsSync(scarDir)).toBe(false)
+  })
+
+  it('a healthy entity (both legs present) is NEVER pruned', async () => {
+    const id = await brain.add({ data: 'keep me', type: 'document', metadata: { keep: true } })
+    await brain.flush()
+
+    await brain.repairIndex()
+
+    expect(entityDir(dir, 'nouns', id)).not.toBeNull()
+    expect(await brain.get(id)).not.toBeNull()
+  })
+})
diff --git a/tests/integration/readdir-no-duplicate-replace.test.ts b/tests/integration/readdir-no-duplicate-replace.test.ts
new file mode 100644
index 00000000..8043bd40
--- /dev/null
+++ b/tests/integration/readdir-no-duplicate-replace.test.ts
@@ -0,0 +1,56 @@
+/**
+ * @module tests/integration/readdir-no-duplicate-replace
+ * @description A re-created path must NOT show up twice in readdir. The reported
+ * defect (memory-vm) was readdir serving DUPLICATE entries for re-created paths,
+ * rooted in a delete that left a ghost (partial removal) whose next delete read
+ * null metadata and skipped unposting the stale Contains edge. With full-removal
+ * deletes there is no ghost, so the edge is always unposted and the path appears
+ * exactly once after any number of delete→recreate cycles.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+
+describe('readdir shows a re-created path exactly once (no duplicate on replace)', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-readdir-'))
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('delete then re-create the same file path — readdir lists it once', async () => {
+    await brain.vfs.mkdir('/d', { recursive: true })
+    await brain.vfs.writeFile('/d/x.txt', 'v1')
+    expect(await brain.vfs.readdir('/d')).toEqual(['x.txt'])
+
+    await brain.vfs.unlink('/d/x.txt')
+    expect(await brain.vfs.readdir('/d')).toEqual([])
+
+    await brain.vfs.writeFile('/d/x.txt', 'v2')
+    const listing = (await brain.vfs.readdir('/d')) as string[]
+    expect(listing).toEqual(['x.txt']) // exactly once — no ghost duplicate
+    expect(listing.filter((n) => n === 'x.txt')).toHaveLength(1)
+  })
+
+  it('survives several delete→recreate cycles without accumulating duplicates', async () => {
+    await brain.vfs.mkdir('/c', { recursive: true })
+    for (let i = 0; i < 4; i++) {
+      await brain.vfs.writeFile('/c/f.txt', `gen ${i}`)
+      await brain.vfs.unlink('/c/f.txt')
+    }
+    await brain.vfs.writeFile('/c/f.txt', 'final')
+    const listing = (await brain.vfs.readdir('/c')) as string[]
+    expect(listing).toEqual(['f.txt'])
+    expect(await brain.vfs.readFile('/c/f.txt')).toBeDefined()
+  })
+})
diff --git a/tests/unit/brainy/migration-gate-family-scoped.test.ts b/tests/unit/brainy/migration-gate-family-scoped.test.ts
new file mode 100644
index 00000000..b71c3899
--- /dev/null
+++ b/tests/unit/brainy/migration-gate-family-scoped.test.ts
@@ -0,0 +1,94 @@
+/**
+ * @module tests/unit/brainy/migration-gate-family-scoped
+ * @description The coordinated migration LOCK is FAMILY-SCOPED: a read waits only
+ * on the index families it actually consults. A one-time native migration of ONE
+ * family (its `isMigrating()` held) must NOT block a read served entirely from
+ * canonical storage (get / VFS content + dir) or from a HEALTHY family — only a
+ * read that needs the migrating family blocks. Regression for the whole-brain
+ * gate that hung getStats / readdir / readFile behind an unrelated family's
+ * migration until the wait timed out.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { Brainy } from '../../../src/brainy.js'
+import { MigrationInProgressError } from '../../../src/errors/brainyError.js'
+
+const seed = async () => {
+  const brain = new Brainy({
+    storage: { type: 'memory' },
+    dimensions: 384,
+    requireSubtype: false,
+    silent: true,
+    // Short so a genuine block resolves fast; a real block would otherwise wait
+    // the 30 s default — this test asserts the wait does NOT happen for scoped
+    // reads, and DOES (bounded) for the one that needs the migrating family.
+    migrationWaitTimeoutMs: 300
+  })
+  await brain.init()
+  for (let i = 0; i < 5; i++) {
+    await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } })
+  }
+  await brain.vfs.writeFile('/notes/hello.txt', 'canonical bytes — no index needed')
+  await brain.flush()
+  return brain
+}
+
+/** Force a provider to report an in-flight (stuck) migration. */
+const jam = (provider: unknown) => {
+  ;(provider as { isMigrating: () => boolean }).isMigrating = () => true
+}
+
+describe('migration LOCK is family-scoped', () => {
+  beforeEach(() => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+  })
+
+  it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => {
+    const brain = await seed()
+    const childId = (
+      (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
+    )[0].entityId
+
+    jam((brain as any).index) // vector provider migrating; graph + metadata healthy
+
+    // None of these consult the vector family → they serve immediately.
+    await expect(brain.getStats()).resolves.toBeDefined()
+    await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) // graph traversal
+    await expect(brain.vfs.readFile('/notes/hello.txt')).resolves.toBeDefined() // canonical blob
+    await expect(brain.get(childId)).resolves.not.toBeNull() // canonical by id
+    await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() // metadata filter
+  })
+
+  it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => {
+    const brain = await seed()
+    jam((brain as any).index)
+
+    // A semantic query consults the vector index — it must wait, and (bounded by
+    // migrationWaitTimeoutMs) surface the retryable MigrationInProgressError
+    // rather than serve a half-built result.
+    await expect(brain.find({ query: 'doc' })).rejects.toBeInstanceOf(MigrationInProgressError)
+  })
+
+  it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => {
+    const brain = await seed()
+    const childId = (
+      (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
+    )[0].entityId
+
+    jam((brain as any).graphIndex) // graph provider migrating; vector + metadata healthy
+
+    // Canonical + vector + metadata are unaffected.
+    await expect(brain.get(childId)).resolves.not.toBeNull()
+    await expect(brain.find({ query: 'doc' })).resolves.toBeDefined()
+    await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined()
+
+    // A graph traversal needs the migrating family → it blocks (bounded).
+    await expect(brain.related({ from: childId })).rejects.toBeInstanceOf(MigrationInProgressError)
+  })
+
+  it('with no migration in flight, every read serves (the fast path is a no-op)', async () => {
+    const brain = await seed()
+    await expect(brain.getStats()).resolves.toBeDefined()
+    await expect(brain.find({ query: 'doc' })).resolves.toBeDefined()
+    await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1)
+  })
+})
diff --git a/tests/unit/migration-lock.test.ts b/tests/unit/migration-lock.test.ts
index 81d44a9f..f0fbbe4c 100644
--- a/tests/unit/migration-lock.test.ts
+++ b/tests/unit/migration-lock.test.ts
@@ -2,11 +2,16 @@
  * Migration LOCK (#18) — coordinated, automatic 7.x → 8.0 auto-upgrade.
  *
  * Exercises the real block-and-queue behavior of `awaitMigrationLock` at the
- * `ensureInitialized` choke point: while ANY index provider reports
- * `isMigrating() === true`, data-plane reads and writes WAIT (no operation
- * touches a half-built index); observability (`getIndexStatus`, `health`) and
- * the lock-clearing `stampBrainFormat` stay exempt; and a lock that outlives the
- * configured window surfaces a retryable `MigrationInProgressError`.
+ * `ensureInitialized` choke point. The gate is FAMILY-SCOPED: a write (which
+ * touches every index) WAITS while ANY provider reports `isMigrating() === true`,
+ * and a read WAITS only when the migrating provider is one of the index families
+ * that read consults (so a read served from canonical storage or a healthy family
+ * is never blocked by an unrelated family's migration — see
+ * `tests/unit/brainy/migration-gate-family-scoped.test.ts`). Observability
+ * (`getIndexStatus`, `health`) and the lock-clearing `stampBrainFormat` stay
+ * exempt; a lock that outlives the configured window surfaces a retryable
+ * `MigrationInProgressError`. Here the graph provider holds the lock, so the
+ * read cases use a graph traversal (`related`) — the family that actually waits.
  *
  * A native (cortex) provider owns the real migration; here we simulate the lock
  * by feature-detect-injecting `isMigrating()` onto a live provider, exactly as
@@ -84,11 +89,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => {
     expect(id).toBeTruthy()
   })
 
-  it('blocks a read while migrating, then releases when the flag clears (poll path)', async () => {
+  it('blocks a graph read while the graph provider migrates, then releases when the flag clears (poll path)', async () => {
+    const anchor = await brain.add({ data: 'anchor', type: NounType.Concept })
     let migrating = true
     brain.graphIndex.isMigrating = () => migrating
 
-    const p = brain.find({ type: NounType.Concept }) // a read → gated
+    const p = brain.related({ from: anchor }) // a GRAPH read → gated on the graph migration
     let resolved = false
     p.then(() => {
       resolved = true
@@ -109,9 +115,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => {
       migrationWaitTimeoutMs: 120
     })
     await shortBrain.init()
-    setMigrating(shortBrain, true) // never clears
+    const anchor = await shortBrain.add({ data: 'anchor', type: NounType.Concept })
+    setMigrating(shortBrain, true) // graph lock never clears
 
-    await expect(shortBrain.find({ type: NounType.Concept })).rejects.toBeInstanceOf(
+    // A graph read needs the migrating family → it waits out the window and
+    // surfaces the retryable error rather than serving a half-built traversal.
+    await expect(shortBrain.related({ from: anchor })).rejects.toBeInstanceOf(
       MigrationInProgressError
     )
     try {

From c0c68ac6a6249676bc31156f8085792985359a1e Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Tue, 14 Jul 2026 10:12:00 -0700
Subject: [PATCH 39/82] docs: RELEASES.md entry for 8.3.1 (full-removal deletes
 + family-scoped gate)

---
 RELEASES.md | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/RELEASES.md b/RELEASES.md
index 446a2c09..0db09513 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,10 +10,47 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.3.1 — 2026-07-14 (full-removal deletes + family-scoped migration gate)
+
+Two production-reported fixes in the write/index spine, plus an operator repair path. No API changes;
+all behavior changes make previously-wrong states honest.
+
+- **Deleting an entity now removes it completely — no more "ghost" leftovers on disk.** A canonical
+  noun delete removed the metadata (content) leg but left the entity's `vectors.json` and its `<id>/`
+  directory behind. Consequences observed in a production deployment: deleted rows were
+  indistinguishable on disk from damage scars, enumerated counts inflated monotonically with every
+  delete (the leftovers were counted forever), and locator-style reads hit unreadable ghost rows.
+  `remove()`/`removeMany()`/`deleteNoun`/`deleteVerb` now remove **both legs and the entity
+  container**, with a full two-leg before-image rollback inside the transaction. The generation log
+  still holds the delete's before-image, so `asOf()` time-travel reconstructs deleted entities exactly
+  as before — this is live-HEAD hygiene, not a history change. This also fixes **duplicate `readdir`
+  entries for re-created VFS paths** at the root: with no ghost state, a delete always unposts its
+  index rows, so a delete→recreate cycle lists the path exactly once (regression-tested across
+  repeated cycles).
+
+- **`repairIndex()` prunes ghost/scar directories left by earlier versions.** Stores that deleted
+  entities under ≤8.3.0 may hold orphaned entity directories (a vector-only leg, or an empty dir).
+  `brain.repairIndex()` now sweeps them: it removes only containers with **no metadata content leg**
+  (never a directory that still holds content), logs every removal, and recomputes type/subtype
+  counts afterward so totals stop counting ghosts.
+
+- **Reads no longer hang behind an unrelated index migration (family-scoped gate).** During a native
+  provider's one-time background migration, *every* read — including plain `get()`, VFS
+  `readdir`/`readFile`, and metadata-only `find({ where })` — blocked on the whole-brain migration
+  lock until timeout, even when the migrating index was irrelevant to the read. The gate is now
+  scoped to the index families a read actually consults: canonical reads (`get`, `batchGet`, VFS
+  content) never wait; a `find` waits only on the families its query shape needs (vector for
+  semantic, metadata for `where`/type, graph for `connected`); graph traversals wait only on the
+  graph family. Writes and unclassified operations keep the conservative whole-brain wait. A read
+  that *does* need the migrating family still blocks (bounded by `migrationWaitTimeoutMs`) and
+  surfaces the retryable `MigrationInProgressError` — never a partial result.
+
+No breaking API change. Each fix ships with regression tests.
+
 ## v8.3.0 — 2026-07-13 (faster index heals + the cross-layer integrity contract)
 
 Three additive changes. The first is an immediate, standalone performance win; the other two are the
-brainy side of the write/index-spine integrity contract (ADR-004), inert until a native accelerator
+brainy side of the write/index-spine integrity contract, inert until a native accelerator
 that implements the matching hooks is present — so this release changes nothing for a JS-only brain
 beyond the speedup.
 

From d9fa3be648adc438061ecb43f33576ef1c207cb5 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Tue, 14 Jul 2026 10:14:38 -0700
Subject: [PATCH 40/82] chore(release): 8.3.1

---
 CHANGELOG.md      | 7 +++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b3821c3b..76ff1e87 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.3.1](https://github.com/soulcraftlabs/brainy/compare/v8.3.0...v8.3.1) (2026-07-14)
+
+- docs: RELEASES.md entry for 8.3.1 (full-removal deletes + family-scoped gate) (c0c68ac)
+- fix: full-removal canonical deletes + family-scoped migration gate (366f9a9)
+- docs: cite the cross-layer integrity contract generically in comments and notes (1d26988)
+
+
 ### [8.3.0](https://github.com/soulcraftlabs/brainy/compare/v8.2.8...v8.3.0) (2026-07-13)
 
 - docs: RELEASES.md entry for 8.3.0 (heal-cost + cross-layer integrity contract) (7692c6f)
diff --git a/package-lock.json b/package-lock.json
index 9b0c625b..fccaaecb 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.0",
+  "version": "8.3.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.3.0",
+      "version": "8.3.1",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index d939ca3c..224770c9 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.0",
+  "version": "8.3.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 2e2ba9c9aacb322c1b7c8269645f012117070c49 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Tue, 14 Jul 2026 11:51:44 -0700
Subject: [PATCH 41/82] =?UTF-8?q?fix:=20honest=20counters=20=E2=80=94=20re?=
 =?UTF-8?q?moval=20never=20re-reads=20the=20removed=20record=20+=20repairI?=
 =?UTF-8?q?ndex=20recounts=20and=20persists=20all=20rollups?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Persisted entity totals inflated permanently: a delete's count decrement
was sourced from re-reading the record being removed, so a null read
(replace race, or a ghost left by a pre-8.3.1 partial delete) silently
skipped the decrement while the paired add had counted — and
Math.max(persistedTotal, scanned) meant no disk cleanup could ever
lower the reported total. Reported by a production deployment with a
write→delete→re-create repro minting drift within hours.

- The caller's pre-delete read now rides through the entire delete path
  (remove()/removeMany()/plan → DeleteNoun/DeleteVerb operations →
  deleteNoun/deleteVerb → deleteNounMetadata/deleteVerbMetadata): a
  null internal read falls back to the known prior record instead of
  skipping the decrement. StorageAdapter.deleteNoun/deleteVerb gain an
  optional priorMetadata parameter (additive).
- rebuildTypeCounts() rebuilt only the type-statistics arrays and
  computed the total just to LOG it — the persisted scalar
  (counts.json) survived every rebuild. One canonical walk now rebuilds
  every counter rollup (scalar totals + per-type maps + type
  statistics) and persists them; repairIndex() runs the recount
  unconditionally, since counters can be inflated over clean shelves.
---
 src/brainy.ts                                 |  27 +++-
 src/coreTypes.ts                              |  22 +++-
 src/storage/baseStorage.ts                    |  69 ++++++++--
 .../operations/StorageOperations.ts           |  25 +++-
 tests/integration/counter-recount.test.ts     | 122 ++++++++++++++++++
 5 files changed, 237 insertions(+), 28 deletions(-)
 create mode 100644 tests/integration/counter-recount.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index 9344ff0c..a9d2b656 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -3093,9 +3093,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           )
         }
 
-        // Operation 3: Delete noun metadata
+        // Operation 3: Delete noun (full removal). The pre-read metadata rides
+        // along so the count decrement never depends on re-reading the record
+        // being removed (a null re-read must not silently skip it).
         tx.addOperation(
-          new DeleteNounMetadataOperation(this.storage, id)
+          new DeleteNounMetadataOperation(this.storage, id, metadata)
         )
 
         // Operations 4+: Delete all related verbs atomically
@@ -6871,8 +6873,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
                 )
               }
 
+              // Pre-read metadata rides along: the count decrement must not
+              // depend on re-reading the record being removed (see remove()).
               tx.addOperation(
-                new DeleteNounMetadataOperation(this.storage, id)
+                new DeleteNounMetadataOperation(this.storage, id, metadata)
               )
 
               for (const verb of allVerbs) {
@@ -9265,7 +9269,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     if (metadata) {
       plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata))
     }
-    plan.operations.push(new DeleteNounMetadataOperation(this.storage, id))
+    // Pre-read metadata rides along: the count decrement must not depend on
+    // re-reading the record being removed (see remove()).
+    plan.operations.push(new DeleteNounMetadataOperation(this.storage, id, metadata))
     for (const verb of cascade.values()) {
       plan.operations.push(
         // Endpoint ints resolve at EXECUTE time — a cascade verb (or its
@@ -15278,15 +15284,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     if (typeof pruner.pruneOrphanedEntities === 'function') {
       const orphans = await pruner.pruneOrphanedEntities()
       if (orphans.nouns.length + orphans.verbs.length > 0) {
-        await pruner.rebuildTypeCounts?.()
-        await pruner.rebuildSubtypeCounts?.()
         prodLog.warn(
           `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` +
             `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` +
-            `partial delete, and recomputed counts.`
+            `partial delete.`
         )
       }
     }
+    // SANCTIONED RECOUNT — unconditional, not gated on orphans found: the
+    // persisted counters can be inflated over perfectly clean shelves (deletes
+    // whose decrement was skipped by the removed-record re-read), and
+    // Math.max(totalNounCount, scanned) means an inflated scalar can never
+    // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup
+    // (scalar totals + per-type maps + type-statistics arrays) from one
+    // canonical walk and persists them.
+    await pruner.rebuildTypeCounts?.()
+    await pruner.rebuildSubtypeCounts?.()
 
     await this.metadataIndex.detectAndRepairCorruption()
     // Lift a failed-rollback write-quarantine: force a full rebuild so the
diff --git a/src/coreTypes.ts b/src/coreTypes.ts
index c345624b..b20d128d 100644
--- a/src/coreTypes.ts
+++ b/src/coreTypes.ts
@@ -848,7 +848,17 @@ export interface StorageAdapter {
    */
   getNounsByNounType(nounType: string): Promise<HNSWNounWithMetadata[]>
 
-  deleteNoun(id: string): Promise<void>
+  /**
+   * Delete a noun — FULL canonical removal (both legs + the entity container).
+   *
+   * @param id The entity id.
+   * @param priorMetadata OPTIONAL already-known metadata of the entity being
+   *   removed (the caller's pre-delete read). The count decrement must never
+   *   REQUIRE re-reading the record being removed: when the internal read
+   *   returns `null` (replace race, or a ghost left by an earlier version) the
+   *   decrement falls back to this record instead of being silently skipped.
+   */
+  deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void>
 
   /**
    * Save verb - Pure HNSW verb with core fields only
@@ -905,7 +915,15 @@ export interface StorageAdapter {
    */
   getVerbsByType(type: string): Promise<HNSWVerbWithMetadata[]>
 
-  deleteVerb(id: string): Promise<void>
+  /**
+   * Delete a verb — FULL canonical removal (both legs + the container).
+   *
+   * @param id The relationship id.
+   * @param priorMetadata OPTIONAL already-known metadata of the edge being
+   *   removed (the caller's pre-delete read); keeps the count decrement honest
+   *   when the internal read returns `null` (see `deleteNoun`).
+   */
+  deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise<void>
 
   /**
    * Save metadata
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index f5efdcb6..db8ffaa6 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -1616,7 +1616,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   /**
    * Delete a noun from storage
    */
-  public async deleteNoun(id: string): Promise<void> {
+  public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void> {
     await this.ensureInitialized()
 
     // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
@@ -1631,7 +1631,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     // success); a REAL fault must surface loudly (loud errors, never quiet
     // losses) and must never silently skip the count decrement — so this is NO
     // LONGER wrapped in a blind catch that masked faults as "file didn't exist".
-    await this.deleteNounMetadata(id)
+    // `priorMetadata` (the caller's pre-delete read) keeps the decrement honest
+    // even when the canonical read inside returns null (replace race / ghost).
+    await this.deleteNounMetadata(id, priorMetadata)
 
     // Remove the now-empty entity container (a no-op for key/prefix stores).
     await this.removeCanonicalContainer(getNounVectorPath(id))
@@ -2936,14 +2938,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   /**
    * Delete a verb from storage
    */
-  public async deleteVerb(id: string): Promise<void> {
+  public async deleteVerb(id: string, priorMetadata?: VerbMetadata | null): Promise<void> {
     await this.ensureInitialized()
 
     // FULL removal (see deleteNoun): both canonical legs + the verb container.
     // The blind catch that masked real faults as "no metadata file" is gone — a
     // genuine absence is already a no-op downstream; a real fault surfaces loudly.
+    // `priorMetadata` keeps the count decrement honest on a null internal read.
     await this.deleteVerb_internal(id) // vectors.json leg
-    await this.deleteVerbMetadata(id)  // metadata leg + count decrement
+    await this.deleteVerbMetadata(id, priorMetadata) // metadata leg + count decrement
     await this.removeCanonicalContainer(getVerbVectorPath(id))
   }
   /**
@@ -3596,19 +3599,32 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   }
 
   /**
-   * Delete noun metadata from storage (ID-first, O(1) delete)
+   * Delete noun metadata from storage (ID-first, O(1) delete).
+   *
+   * @param id - The entity id.
+   * @param priorRecord - OPTIONAL already-known metadata of the entity being
+   *   removed (e.g. the pre-delete read `remove()` performs, or a captured
+   *   before-image). The count decrement must never REQUIRE re-reading the
+   *   thing being removed: when the canonical read here returns `null` (a
+   *   replace race, or a partial-delete ghost from an earlier version) the
+   *   decrement falls back to this record instead of being silently skipped —
+   *   the skip permanently inflated the persisted totals (adds counted, paired
+   *   removals not decremented), and `Math.max(totalNounCount, scanned)` made
+   *   the inflation unfixable by any disk cleanup.
    */
-  public async deleteNounMetadata(id: string): Promise<void> {
+  public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise<void> {
     await this.ensureInitialized()
 
     // Direct O(1) delete with ID-first path. Read the canonical record BEFORE
     // removing it: the per-type and subtype decrements are sourced from the
     // entity's own metadata (`noun` type, `subtype`, `visibility`) rather than an
     // id-keyed cache, keeping type-statistics honest across deletes — symmetric
-    // with the increments in `saveNounMetadata_internal()`.
+    // with the increments in `saveNounMetadata_internal()`. A null read falls
+    // back to the caller-provided prior record (see @param priorRecord).
     const path = getNounMetadataPath(id)
-    const record = await this.readCanonicalObject(path)
+    const read = await this.readCanonicalObject(path)
     await this.deleteCanonicalObject(path)
+    const record = read ?? priorRecord
 
     const priorType = record?.noun as NounType | undefined
     // 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
@@ -3782,16 +3798,20 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   /**
    * Delete verb metadata from storage (ID-first, O(1) delete)
    */
-  public async deleteVerbMetadata(id: string): Promise<void> {
+  public async deleteVerbMetadata(id: string, priorRecord?: VerbMetadata | null): Promise<void> {
     await this.ensureInitialized()
 
     // Direct O(1) delete with ID-first path. Read the canonical record BEFORE
     // removing it so every decrement is sourced from the edge's own metadata
     // (`verb` type, `subtype`, `visibility`) rather than an id-keyed cache —
-    // symmetric with the increments in `saveVerbMetadata_internal()`.
+    // symmetric with the increments in `saveVerbMetadata_internal()`. A null
+    // read falls back to the caller-provided prior record: the decrement must
+    // never REQUIRE re-reading the thing being removed (a silent skip minted
+    // permanent counter inflation — see deleteNounMetadata).
     const path = getVerbMetadataPath(id)
-    const record = await this.readCanonicalObject(path)
+    const read = await this.readCanonicalObject(path)
     await this.deleteCanonicalObject(path)
+    const record = read ?? priorRecord
 
     const priorVerb = record?.verb as VerbType | undefined
     // Symmetric count decrement (previously OMITTED — verb deletes touched neither the
@@ -4228,6 +4248,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT)
     this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT)
 
+    // The SAME walk also rebuilds the user-facing scalar totals + per-type maps
+    // persisted in counts.json (totalNounCount / totalVerbCount / entityCounts /
+    // verbCounts). Previously only the type-statistics arrays were rebuilt and
+    // the total was computed just to LOG it — so a drifted persisted scalar
+    // (deletes whose decrement was skipped) survived every "rebuild" forever,
+    // and Math.max(totalNounCount, scanned) made the inflation unfixable by any
+    // disk cleanup. This method is now the SANCTIONED RECOUNT: one canonical
+    // walk, every counter rollup rebuilt and persisted from it.
+    const countedNouns = new Map<string, number>()
+    const countedVerbs = new Map<string, number>()
+
     // Scan noun shards
     for (let shard = 0; shard < 256; shard++) {
       const shardHex = shard.toString(16).padStart(2, '0')
@@ -4248,6 +4279,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
                 if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
                   this.nounCountsByType[typeIndex]++
                 }
+                countedNouns.set(metadata.noun, (countedNouns.get(metadata.noun) || 0) + 1)
               }
             }
           } catch (error) {
@@ -4279,6 +4311,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
                 if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
                   this.verbCountsByType[typeIndex]++
                 }
+                countedVerbs.set(metadata.verb, (countedVerbs.get(metadata.verb) || 0) + 1)
               }
             }
           } catch (error) {
@@ -4295,7 +4328,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
     const totalVerbs = this.verbCountsByType.reduce((sum, count) => sum + count, 0)
     const totalNouns = this.nounCountsByType.reduce((sum, count) => sum + count, 0)
-    prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`)
+
+    // The sanctioned recount half: replace the user-facing scalar totals + the
+    // per-type maps with the walk's truth and PERSIST them (counts.json), so an
+    // inflated persisted counter is actually corrected — not merely out-voted
+    // in memory until the next reopen rehydrates the stale file.
+    this.entityCounts = countedNouns
+    this.verbCounts = countedVerbs
+    this.totalNounCount = totalNouns
+    this.totalVerbCount = totalVerbs
+    this.countCache.clear()
+    await this.persistCounts()
+
+    prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`)
   }
 
   /**
diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts
index bd753552..316f1ac0 100644
--- a/src/transaction/operations/StorageOperations.ts
+++ b/src/transaction/operations/StorageOperations.ts
@@ -127,22 +127,33 @@ export class DeleteNounMetadataOperation implements Operation {
 
   constructor(
     private readonly storage: StorageAdapter,
-    private readonly id: string
+    private readonly id: string,
+    /**
+     * OPTIONAL already-known metadata of the entity being removed (the caller's
+     * pre-delete read). Removal must never REQUIRE re-reading the thing being
+     * removed: if the reads here return null (replace race, or a ghost left by
+     * an earlier version), the count decrement downstream falls back to this
+     * record instead of being silently skipped — the skip minted permanent
+     * counter inflation (adds counted, paired removals not decremented).
+     */
+    private readonly priorMetadata?: NounMetadata | null
   ) {}
 
   async execute(): Promise<RollbackAction> {
     // Capture the FULL before-image (both legs) so the undo restores the whole
     // entity — a metadata-only rollback would leave the vector leg unrestored.
+    // A null metadata read falls back to the caller's pre-delete read.
     const previousNoun = await this.storage.getNoun(this.id)
-    const previousMetadata = await this.storage.getNounMetadata(this.id)
+    const previousMetadata = (await this.storage.getNounMetadata(this.id)) ?? this.priorMetadata ?? null
 
     if (!previousNoun && !previousMetadata) {
       // Nothing to delete - no rollback needed
       return async () => {}
     }
 
-    // Full removal: both canonical legs + the entity container + count decrement.
-    await this.storage.deleteNoun(this.id)
+    // Full removal: both canonical legs + the entity container + count decrement
+    // (the prior record keeps the decrement honest on a null canonical read).
+    await this.storage.deleteNoun(this.id, previousMetadata)
 
     // Return rollback action
     return async () => {
@@ -268,9 +279,9 @@ export class DeleteVerbMetadataOperation implements Operation {
       return async () => {}
     }
 
-    // Delete verb (metadata + vector)
-    // Note: StorageAdapter has deleteVerb but not deleteVerbMetadata
-    await this.storage.deleteVerb(this.id)
+    // Delete verb (metadata + vector). The pre-read rides along so the count
+    // decrement never depends on re-reading the record being removed.
+    await this.storage.deleteVerb(this.id, previousMetadata)
 
     // Return rollback action
     return async () => {
diff --git a/tests/integration/counter-recount.test.ts b/tests/integration/counter-recount.test.ts
new file mode 100644
index 00000000..310ea4d5
--- /dev/null
+++ b/tests/integration/counter-recount.test.ts
@@ -0,0 +1,122 @@
+/**
+ * @module tests/integration/counter-recount
+ * @description Counter honesty. Two laws under test:
+ * (1) REMOVAL NEVER REQUIRES RE-READING THE REMOVED RECORD — the count
+ *     decrement falls back to the caller's pre-delete read when the canonical
+ *     metadata re-read returns null (replace race / ghost), instead of being
+ *     silently skipped. The skip minted permanent inflation: adds counted,
+ *     paired removals not decremented, and Math.max(totalNounCount, scanned)
+ *     pinned the inflated scalar forever.
+ * (2) THE SANCTIONED RECOUNT — repairIndex() unconditionally recomputes and
+ *     PERSISTS every counter rollup (scalar totals + per-type maps +
+ *     type-statistics) from one canonical walk, so an already-inflated brain
+ *     is permanently corrected (survives reopen).
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+
+/** Absolute path of one entity's `<id>` directory (or null if not present). */
+function entityDir(root: string, id: string): string | null {
+  const base = path.join(root, 'entities', 'nouns')
+  if (!fs.existsSync(base)) return null
+  for (const shard of fs.readdirSync(base)) {
+    const candidate = path.join(base, shard, id)
+    if (fs.existsSync(candidate)) return candidate
+  }
+  return null
+}
+
+describe('counter honesty — removal without re-reading + the sanctioned recount', () => {
+  let dir: string
+  let brain: any
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-recount-'))
+    brain = await open()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('the counter returns to baseline across add→remove→re-create cycles (no drift)', async () => {
+    const baseline = await brain.storage.getNounCount()
+    for (let i = 0; i < 4; i++) {
+      const id = await brain.add({ data: `cycle ${i}`, type: 'document', metadata: { i } })
+      expect(await brain.storage.getNounCount()).toBe(baseline + 1)
+      await brain.remove(id)
+      expect(await brain.storage.getNounCount()).toBe(baseline)
+    }
+  })
+
+  it('deleteNoun decrements from the provided prior record when the canonical re-read is null', async () => {
+    const id = await brain.add({ data: 'to be ghosted', type: 'document', metadata: { g: 1 } })
+    await brain.flush()
+    const baseline = await brain.storage.getNounCount()
+
+    // Capture the pre-delete read (what remove() holds), then simulate the
+    // replace-race / ghost shape: the metadata leg vanishes before the delete's
+    // internal re-read.
+    const prior = await brain.storage.getNounMetadata(id)
+    expect(prior).not.toBeNull()
+    const eDir = entityDir(dir, id)!
+    for (const f of fs.readdirSync(eDir)) {
+      if (f.startsWith('metadata.json')) fs.rmSync(path.join(eDir, f))
+    }
+
+    // Without the prior record this decrement used to be silently skipped.
+    await brain.storage.deleteNoun(id, prior)
+    expect(await brain.storage.getNounCount()).toBe(baseline - 1)
+    // Full removal still holds: nothing left on disk.
+    expect(entityDir(dir, id)).toBeNull()
+  })
+
+  it('repairIndex() recounts an inflated persisted scalar over CLEAN shelves — and it survives reopen', async () => {
+    for (let i = 0; i < 3; i++) {
+      await brain.add({ data: `real ${i}`, type: 'document', metadata: { i } })
+    }
+    await brain.flush()
+    const honest = await brain.storage.getNounCount()
+    // Pagination's totalCount has its own baseline: it enumerates EVERYTHING
+    // (including the internal VFS root), while the user-facing scalar counts
+    // only public entities — so the two legitimately differ by the internals.
+    const pageHonest = (await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 })).totalCount
+
+    // Simulate the historical drift: an inflated persisted scalar (deletes whose
+    // decrement was skipped). Persist it so a reopen rehydrates the lie.
+    ;(brain.storage as any).totalNounCount = honest + 52
+    await (brain.storage as any).persistCounts()
+    await brain.close()
+    brain = await open()
+    expect(await brain.storage.getNounCount()).toBe(honest + 52) // the lie survived reopen
+
+    // The sanctioned recount — unconditional in repairIndex (no orphans needed).
+    await brain.repairIndex()
+    expect(await brain.storage.getNounCount()).toBe(honest)
+
+    // Permanently: the corrected counter survives another reopen.
+    await brain.close()
+    brain = await open()
+    expect(await brain.storage.getNounCount()).toBe(honest)
+
+    // And the paginated totalCount (the Math.max consumer) is honest too —
+    // back to ITS baseline, no longer pinned high by the inflated scalar.
+    const page = await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 })
+    expect(page.totalCount).toBe(pageHonest)
+  })
+})

From 0932ecde37f2c11aa554d3f6d9847432afb2f3fc Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Tue, 14 Jul 2026 11:51:51 -0700
Subject: [PATCH 42/82] docs: RELEASES.md entry for 8.3.2 (honest counters)

---
 RELEASES.md | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index 0db09513..ca57a1ef 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,36 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.3.2 — 2026-07-14 (honest counters — the recount + removal-without-re-reading)
+
+Completes 8.3.1's delete-hygiene story at the counter layer, from a production proof chain reported
+by a downstream deployment: persisted entity totals were permanently **inflated** — deletes whose
+count decrement was silently skipped — and because paginated `totalCount` serves
+`Math.max(persistedTotal, scanned)`, the inflated number always won and **no disk cleanup could ever
+lower it**.
+
+- **A removal's count decrement no longer requires re-reading the record being removed.** The
+  decrement was sourced from re-reading the entity's metadata inside the delete; if that read
+  returned `null` (a replace race, or a ghost left by a pre-8.3.1 partial delete) the decrement was
+  silently skipped while the paired add had counted — minting drift on every write→delete→re-create
+  cycle. The caller's pre-delete read now rides through the whole delete path
+  (`remove()`/`removeMany()` → the delete operation → `deleteNoun`/`deleteVerb` →
+  `deleteNounMetadata`/`deleteVerbMetadata`, both sides symmetric): a null internal read falls back
+  to the known prior record instead of skipping. The `StorageAdapter` signatures gain an optional
+  `priorMetadata` parameter (additive; existing adapters unaffected).
+
+- **`repairIndex()` is the sanctioned counter recount — unconditional, and it actually persists.**
+  `rebuildTypeCounts()` previously rebuilt only the type-statistics arrays and computed the total
+  *just to log it* — the persisted scalar (`counts.json`) survived every "rebuild" untouched, so an
+  already-inflated brain could never be corrected. One canonical walk now rebuilds **every** counter
+  rollup — scalar totals, per-type maps, and type statistics — and persists them, and `repairIndex()`
+  runs it unconditionally (not only when orphan directories are found: counters can be inflated over
+  perfectly clean shelves). Brains with delete history should run `brain.repairIndex()` once after
+  upgrading; the correction survives reopen.
+
+No API breaks (optional-parameter additions only). Regression tests cover the drift cycle, the
+null-read decrement fallback, and the persisted recount across reopen.
+
 ## v8.3.1 — 2026-07-14 (full-removal deletes + family-scoped migration gate)
 
 Two production-reported fixes in the write/index spine, plus an operator repair path. No API changes;

From 1a3a493c2760faf4b1be3ecd0fb84fa011e1e111 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Tue, 14 Jul 2026 11:57:00 -0700
Subject: [PATCH 43/82] chore(release): 8.3.2

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76ff1e87..ea8f83e6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.3.2](https://github.com/soulcraftlabs/brainy/compare/v8.3.1...v8.3.2) (2026-07-14)
+
+- docs: RELEASES.md entry for 8.3.2 (honest counters) (0932ecd)
+- fix: honest counters — removal never re-reads the removed record + repairIndex recounts and persists all rollups (2e2ba9c)
+
+
 ### [8.3.1](https://github.com/soulcraftlabs/brainy/compare/v8.3.0...v8.3.1) (2026-07-14)
 
 - docs: RELEASES.md entry for 8.3.1 (full-removal deletes + family-scoped gate) (c0c68ac)
diff --git a/package-lock.json b/package-lock.json
index fccaaecb..d1e8be63 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.1",
+  "version": "8.3.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.3.1",
+      "version": "8.3.2",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 224770c9..9839bf0a 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.1",
+  "version": "8.3.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From af8c1795bd12a8d9066e64804b01026f237d1d9e Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 09:25:21 -0700
Subject: [PATCH 44/82] =?UTF-8?q?fix:=20VFS=20rename=20moves=20the=20conta?=
 =?UTF-8?q?inment=20edge=20=E2=80=94=20no=20ghost=20in=20the=20old=20direc?=
 =?UTF-8?q?tory?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A cross-directory rename added the new parent's Contains edge but skipped
removing the old one (a 'not critical' shortcut), leaving the moved entity
a child of BOTH directories: readdir(oldDir) kept listing it, re-creating
the old path showed the name twice, and tree-walking consumers saw the
file in two places. Reported live by a production deployment (37 stale
containment ghosts; blocks tree-sync integrations).

- rename() now removes the old parent's vfs containment edge(s) by edge
  id resolved from the graph's own adjacency (the removal law: removal
  never requires reading the removed thing), and a move to the root gets
  its containment edge (previously skipped -> orphaned from readdir('/')).
- New vfs.repairContainment(): reconciles every VFS entity's containment
  edges against canonical metadata.path — removes stale/duplicate vfs
  edges, restores missing ones, never touches user knowledge edges. Wired
  into repairIndex() so the one operator ritual heals existing ghosts.
- Regression: tests/integration/vfs-rename-containment.test.ts (7).
---
 src/brainy.ts                                 |  15 ++
 src/vfs/VirtualFileSystem.ts                  | 111 ++++++++++++-
 .../vfs-rename-containment.test.ts            | 157 ++++++++++++++++++
 3 files changed, 278 insertions(+), 5 deletions(-)
 create mode 100644 tests/integration/vfs-rename-containment.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index a9d2b656..ef48c216 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -15301,6 +15301,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     await pruner.rebuildTypeCounts?.()
     await pruner.rebuildSubtypeCounts?.()
 
+    // VFS containment reconciliation: heal "cosmetic ghost" edges left by
+    // pre-fix renames (an entity Contains-linked from BOTH its old and new
+    // directory — readdir listed it in two places) and duplicate edges from
+    // concurrent writers. Canonical metadata.path is the truth; only VFS
+    // containment edges are touched. Loud per repair.
+    if (this._vfsInitialized && this._vfs) {
+      const containment = await this._vfs.repairContainment()
+      if (containment.removed + containment.restored > 0) {
+        prodLog.warn(
+          `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` +
+            `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).`
+        )
+      }
+    }
+
     await this.metadataIndex.detectAndRepairCorruption()
     // Lift a failed-rollback write-quarantine: force a full rebuild so the
     // derived indexes are provably reconciled with canonical, then clear the
diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts
index 013f04f3..00bddefb 100644
--- a/src/vfs/VirtualFileSystem.ts
+++ b/src/vfs/VirtualFileSystem.ts
@@ -1954,15 +1954,27 @@ export class VirtualFileSystem implements IVirtualFileSystem {
     const newParentPath = this.getParentPath(newPath)
 
     if (oldParentPath !== newParentPath) {
-      // Remove from old parent
+      // Remove the OLD parent's containment edge(s) — by edge id, resolved from
+      // the graph's own adjacency (the removal law: a removal never requires
+      // reading the thing being removed). This step used to be skipped as "not
+      // critical", which left the moved entity a child of BOTH directories:
+      // readdir(oldDir) kept listing it, re-creating the old path showed the
+      // name twice, and tree-walking consumers saw the file in two places.
       if (oldParentPath) {
         const oldParentId = await this.pathResolver.resolve(oldParentPath)
-        // unrelate takes the relation ID, not params - need to find and remove relation
-        // For now, skip unrelate as it's not critical for rename
+        const staleEdges = await this.brain.related({
+          from: oldParentId,
+          to: entityId,
+          type: VerbType.Contains
+        })
+        for (const edge of staleEdges) {
+          await this.brain.unrelate(edge.id)
+        }
       }
 
-      // Add to new parent
-      if (newParentPath && newParentPath !== '/') {
+      // Add to the new parent. The root ('/') is a REAL parent — skipping it
+      // orphaned a move-to-root out of readdir('/') entirely.
+      if (newParentPath) {
         const newParentId = await this.pathResolver.resolve(newParentPath)
         await this.brain.relate({
           from: newParentId,
@@ -2003,6 +2015,95 @@ export class VirtualFileSystem implements IVirtualFileSystem {
     this.triggerWatchers(newPath, 'rename')
   }
 
+  /**
+   * Reconcile every VFS entity's containment edges against its canonical
+   * `metadata.path` — the path is the truth (maintained by write/rename); the
+   * `Contains` edges are a projection of it. Heals the "cosmetic ghost" class
+   * left by pre-fix renames that added the new parent's edge without removing
+   * the old one (an entity listed in TWO directories; a re-created old path
+   * showing its name twice), plus duplicate edges from the same parent left by
+   * concurrent writers.
+   *
+   * CONSERVATIVE by design: only VFS containment edges (subtype
+   * `'vfs-contains'` or `metadata.isVFS`) are ever touched — a user's own
+   * knowledge-graph `Contains` edge between the same entities is never
+   * removed. An entity whose expected parent path has no entity is logged
+   * loudly and left alone (never orphaned further). Operator-invoked via
+   * `brain.repairIndex()`.
+   *
+   * The canonical pagination walk (not an index query) is deliberate: this is
+   * a repair op — the projections are the thing under suspicion, so the walk
+   * reads the source of truth.
+   *
+   * @returns Counts of stale edges removed and missing expected edges restored.
+   */
+  async repairContainment(): Promise<{ removed: number; restored: number }> {
+    await this.ensureInitialized()
+
+    // Pass 1: canonical walk → every VFS entity's id + path.
+    const idByPath = new Map<string, string>()
+    const vfsEntities: Array<{ id: string; path: string }> = []
+    let cursor: string | undefined
+    for (;;) {
+      const page = await (this.brain as any).storage.getNounsWithPagination({ limit: 500, cursor })
+      for (const noun of page.items) {
+        const meta = (noun as any).metadata ?? noun
+        const p = meta?.path
+        if (meta?.vfsType && typeof p === 'string') {
+          idByPath.set(p, noun.id)
+          vfsEntities.push({ id: noun.id, path: p })
+        }
+      }
+      if (!page.hasMore) break
+      cursor = page.nextCursor
+    }
+
+    let removed = 0
+    let restored = 0
+    for (const { id, path } of vfsEntities) {
+      if (path === '/') continue // the root has no parent
+      const expectedParentId = idByPath.get(this.getParentPath(path))
+      if (!expectedParentId) {
+        console.warn(
+          `[VFS] repairContainment: no entity found for parent of ${path} — leaving its edges untouched.`
+        )
+        continue
+      }
+
+      const incoming = await this.brain.related({ to: id, type: VerbType.Contains })
+      let expectedSeen = false
+      for (const edge of incoming) {
+        const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true
+        if (!isVfsEdge) continue // never touch user knowledge edges
+        if (edge.from === expectedParentId && !expectedSeen) {
+          expectedSeen = true // keep exactly one correct edge
+          continue
+        }
+        // Stale parent (a pre-fix rename ghost) or a duplicate of the correct
+        // edge (concurrent-writer artifact) — remove it, loudly.
+        await this.brain.unrelate(edge.id)
+        removed++
+        console.warn(
+          `[VFS] repairContainment: removed ${edge.from === expectedParentId ? 'duplicate' : 'stale'} ` +
+            `containment edge ${edge.from} -> ${id} (${path})`
+        )
+      }
+      if (!expectedSeen) {
+        await this.brain.relate({
+          from: expectedParentId,
+          to: id,
+          type: VerbType.Contains,
+          subtype: 'vfs-contains',
+          metadata: { isVFS: true }
+        })
+        restored++
+        console.warn(`[VFS] repairContainment: restored missing containment edge for ${path}`)
+      }
+    }
+
+    return { removed, restored }
+  }
+
   /**
    * Copy a file or directory to a new path.
    *
diff --git a/tests/integration/vfs-rename-containment.test.ts b/tests/integration/vfs-rename-containment.test.ts
new file mode 100644
index 00000000..0169bf01
--- /dev/null
+++ b/tests/integration/vfs-rename-containment.test.ts
@@ -0,0 +1,157 @@
+/**
+ * @module tests/integration/vfs-rename-containment
+ * @description A cross-directory rename must MOVE the containment edge, not
+ * accumulate one per parent. The pre-fix rename added the new parent's
+ * Contains edge but skipped removing the old one ("not critical") — leaving
+ * the entity a child of BOTH directories: readdir(oldDir) kept listing it,
+ * re-creating the old path showed the name twice (the duplicate-readdir /
+ * "cosmetic ghost" field report), and tree-walking consumers saw the file in
+ * two places. Also covers the repair sweep (repairIndex → vfs.repairContainment)
+ * that heals ghosts left by earlier versions, and move-to-root (whose edge
+ * used to be skipped entirely).
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy, VerbType } from '../../src/index.js'
+
+describe('VFS rename moves the containment edge (no ghost in the old directory)', () => {
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 })
+    await brain.init()
+    await brain.vfs.mkdir('/a', { recursive: true })
+    await brain.vfs.mkdir('/b', { recursive: true })
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+  })
+
+  it('cross-directory move: old dir stops listing it, new dir lists it exactly once', async () => {
+    await brain.vfs.writeFile('/a/x.txt', 'v1')
+    await brain.vfs.rename('/a/x.txt', '/b/x.txt')
+
+    expect(await brain.vfs.readdir('/a')).toEqual([])
+    expect(await brain.vfs.readdir('/b')).toEqual(['x.txt'])
+    expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('v1')
+  })
+
+  it('re-creating the old path lists each name exactly once in each directory', async () => {
+    await brain.vfs.writeFile('/a/x.txt', 'moved away')
+    await brain.vfs.rename('/a/x.txt', '/b/x.txt')
+    await brain.vfs.writeFile('/a/x.txt', 'new file at old path')
+
+    const a = (await brain.vfs.readdir('/a')) as string[]
+    const b = (await brain.vfs.readdir('/b')) as string[]
+    expect(a).toEqual(['x.txt']) // exactly once — the pre-fix ghost made this list the moved entity too
+    expect(b).toEqual(['x.txt'])
+    expect(String(await brain.vfs.readFile('/a/x.txt'))).toBe('new file at old path')
+    expect(String(await brain.vfs.readFile('/b/x.txt'))).toBe('moved away')
+  })
+
+  it('repeated moves never accumulate containment edges', async () => {
+    await brain.vfs.writeFile('/a/f.txt', 'wanderer')
+    for (let i = 0; i < 3; i++) {
+      await brain.vfs.rename('/a/f.txt', '/b/f.txt')
+      await brain.vfs.rename('/b/f.txt', '/a/f.txt')
+    }
+    expect(await brain.vfs.readdir('/a')).toEqual(['f.txt'])
+    expect(await brain.vfs.readdir('/b')).toEqual([])
+
+    // Exactly ONE containment edge exists on the entity.
+    const stat = await brain.vfs.stat('/a/f.txt')
+    const edges = await brain.related({ to: stat.entityId, type: VerbType.Contains })
+    expect(edges).toHaveLength(1)
+  })
+
+  it('move to the root gets a containment edge (used to be skipped → orphan)', async () => {
+    await brain.vfs.writeFile('/a/up.txt', 'to the top')
+    await brain.vfs.rename('/a/up.txt', '/up.txt')
+
+    const rootListing = (await brain.vfs.readdir('/')) as string[]
+    expect(rootListing).toContain('up.txt')
+    expect(await brain.vfs.readdir('/a')).toEqual([])
+    expect(String(await brain.vfs.readFile('/up.txt'))).toBe('to the top')
+  })
+})
+
+describe('repairIndex() heals pre-fix containment ghosts (vfs.repairContainment)', () => {
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 })
+    await brain.init()
+    await brain.vfs.mkdir('/a', { recursive: true })
+    await brain.vfs.mkdir('/b', { recursive: true })
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+  })
+
+  /** Reproduce the PRE-FIX defect state: file lives at /b/g.txt but a stale
+   *  vfs-contains edge from /a lingers (what old renames left behind). */
+  const synthesizeGhost = async () => {
+    await brain.vfs.writeFile('/b/g.txt', 'ghost target')
+    const fileId = (await brain.vfs.stat('/b/g.txt')).entityId
+    const aId = (await brain.vfs.stat('/a')).entityId
+    await brain.relate({
+      from: aId,
+      to: fileId,
+      type: VerbType.Contains,
+      subtype: 'vfs-contains',
+      metadata: { isVFS: true }
+    })
+    return { fileId, aId }
+  }
+
+  it('a stale old-parent edge is removed; listings become honest', async () => {
+    const { fileId } = await synthesizeGhost()
+    // The defect state is visible: /a lists a file whose path says /b.
+    expect((await brain.vfs.readdir('/a')) as string[]).toContain('g.txt')
+
+    await brain.repairIndex()
+
+    expect(await brain.vfs.readdir('/a')).toEqual([])
+    expect(await brain.vfs.readdir('/b')).toEqual(['g.txt'])
+    const edges = await brain.related({ to: fileId, type: VerbType.Contains })
+    expect(edges).toHaveLength(1)
+  })
+
+  it("a user's own knowledge Contains edge onto the file survives the repair", async () => {
+    const { fileId } = await synthesizeGhost()
+    // A knowledge-graph containment from a NON-directory entity (a collection
+    // curating the file) — same verb TYPE, not a vfs-contains edge. The repair
+    // must remove only VFS containment ghosts, never user knowledge edges.
+    // (Edges dedupe by (from,to,type), so the user edge needs its own source.)
+    const collectionId = await brain.add({
+      data: 'reading list',
+      type: 'collection',
+      metadata: { kind: 'curation' }
+    })
+    await brain.relate({ from: collectionId, to: fileId, type: VerbType.Contains, subtype: 'curates' })
+
+    await brain.repairIndex()
+
+    const edges = await brain.related({ to: fileId, type: VerbType.Contains })
+    const subtypes = edges.map((e: any) => e.subtype).sort()
+    // The stale vfs ghost edge is gone; the correct vfs edge + the user's
+    // curation edge remain.
+    expect(subtypes).toEqual(['curates', 'vfs-contains'])
+    expect(edges.find((e: any) => e.subtype === 'curates')?.from).toBe(collectionId)
+  })
+
+  it('a missing expected edge is restored (entity unreachable from its own directory)', async () => {
+    await brain.vfs.writeFile('/b/lost.txt', 'find me')
+    const fileId = (await brain.vfs.stat('/b/lost.txt')).entityId
+    // Simulate total edge loss (an older damage shape).
+    for (const e of await brain.related({ to: fileId, type: VerbType.Contains })) {
+      await brain.unrelate(e.id)
+    }
+    expect((await brain.vfs.readdir('/b')) as string[]).not.toContain('lost.txt')
+
+    await brain.repairIndex()
+
+    expect((await brain.vfs.readdir('/b')) as string[]).toContain('lost.txt')
+  })
+})

From 4fb41f9a7cc0ce28926f999e421939cac83ba045 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 09:25:36 -0700
Subject: [PATCH 45/82] =?UTF-8?q?test:=20lens-consistency=20regression=20?=
 =?UTF-8?q?=E2=80=94=20combined=20vs=20subtype-only=20vs=20canonical=20gro?=
 =?UTF-8?q?und=20truth?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Ports the fresh-brain probe that closed the type+subtype lens-drop
investigation into the permanent suite: a 7-pair corpus seeded through the
real write API (never restored bytes), every lens checked id-for-id against
an unfiltered canonical scan, warm AND after a cold reopen, plus the
type+subtype update-flip leg. Guards the under-inclusion class the egress
integrity guard structurally cannot catch.
---
 tests/integration/lens-consistency.test.ts | 138 +++++++++++++++++++++
 1 file changed, 138 insertions(+)
 create mode 100644 tests/integration/lens-consistency.test.ts

diff --git a/tests/integration/lens-consistency.test.ts b/tests/integration/lens-consistency.test.ts
new file mode 100644
index 00000000..64484a38
--- /dev/null
+++ b/tests/integration/lens-consistency.test.ts
@@ -0,0 +1,138 @@
+/**
+ * @module tests/integration/lens-consistency
+ * @description The three metadata "lenses" over one corpus must agree with
+ * canonical ground truth id-for-id, warm AND after a cold reopen:
+ *   - combined:      find({ type: T, where: { subtype: S } })
+ *   - subtype-only:  find({ where: { subtype: S } })
+ *   - type-only:     find({ type: T })
+ * Ported from the fresh-brain probe that closed the type+subtype lens-drop
+ * investigation (a restored pre-8.2.2 torn capture had entities visible to the
+ * subtype-only lens but dropped by the combined lens — "0 of 2 migrated, all
+ * gates green"). The corpus is seeded through the REAL write API — never
+ * restored bytes — which is what made the original datapoint decisive. The
+ * invariants: every lens matches an unfiltered canonical scan exactly (no
+ * missing ids, no extras) and combined ⊆ subtype-only always holds.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+
+/** The corpus: 7 (type, subtype) pairs, uneven counts, incl. the incident's 2-of-a-pair shape. */
+const CORPUS: Array<{ type: string; subtype: string; count: number }> = [
+  { type: 'proposition', subtype: 'decision', count: 2 }, // the incident shape: "0 of 2"
+  { type: 'concept', subtype: 'decision', count: 3 },
+  { type: 'task', subtype: 'decision', count: 2 },
+  { type: 'concept', subtype: 'action', count: 4 },
+  { type: 'message', subtype: 'note', count: 5 },
+  { type: 'message', subtype: 'ship', count: 3 },
+  { type: 'document', subtype: 'guide', count: 4 }
+]
+
+/** Canonical ground truth: unfiltered enumeration, post-filtered IN THE TEST. */
+async function groundTruth(
+  brain: any,
+  match: { type?: string; subtype?: string }
+): Promise<Set<string>> {
+  const ids = new Set<string>()
+  let cursor: string | undefined
+  for (;;) {
+    const page = await brain.storage.getNounsWithPagination({ limit: 500, cursor })
+    for (const noun of page.items) {
+      // Hydrated shape: `type`/`subtype` are TOP-LEVEL; `metadata` holds only
+      // custom user fields (vfsType is one — the VFS plumbing marker).
+      const n = noun as any
+      if (n.metadata?.vfsType) continue // VFS plumbing is not corpus
+      if (!n.type || !n.subtype) continue
+      if (match.type && n.type !== match.type) continue
+      if (match.subtype && n.subtype !== match.subtype) continue
+      ids.add(n.id)
+    }
+    if (!page.hasMore) break
+    cursor = page.nextCursor
+  }
+  return ids
+}
+
+const idSet = (results: Array<{ id: string }>): Set<string> => new Set(results.map((r) => r.id))
+
+/** Every lens vs ground truth, id-for-id, for every pair in the corpus. */
+async function assertAllLenses(brain: any): Promise<void> {
+  const types = [...new Set(CORPUS.map((c) => c.type))]
+  const subtypes = [...new Set(CORPUS.map((c) => c.subtype))]
+
+  for (const { type, subtype } of CORPUS) {
+    const combined = idSet(await brain.find({ type, where: { subtype }, limit: 1000 }))
+    const subtypeOnly = idSet(await brain.find({ where: { subtype }, limit: 1000 }))
+    const truthPair = await groundTruth(brain, { type, subtype })
+    const truthSubtype = await groundTruth(brain, { subtype })
+
+    expect([...combined].sort()).toEqual([...truthPair].sort()) // no drops, no extras
+    expect([...subtypeOnly].sort()).toEqual([...truthSubtype].sort())
+    for (const id of combined) expect(subtypeOnly.has(id)).toBe(true) // combined ⊆ subtype-only
+  }
+
+  for (const type of types) {
+    const typeOnly = idSet(await brain.find({ type, limit: 1000 }))
+    const truthType = await groundTruth(brain, { type })
+    expect([...typeOnly].sort()).toEqual([...truthType].sort())
+  }
+
+  // Count cross-check against the corpus definition itself.
+  for (const subtype of subtypes) {
+    const expected = CORPUS.filter((c) => c.subtype === subtype).reduce((s, c) => s + c.count, 0)
+    const got = (await brain.find({ where: { subtype }, limit: 1000 })).length
+    expect(got).toBe(expected)
+  }
+}
+
+describe('lens consistency — combined vs subtype-only vs canonical ground truth', () => {
+  let dir: string
+  let brain: any
+
+  beforeAll(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-lens-'))
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+    // Seed through the REAL write API — never restored bytes.
+    let i = 0
+    for (const { type, subtype, count } of CORPUS) {
+      for (let k = 0; k < count; k++) {
+        await brain.add({ data: `${type} ${subtype} ${i++}`, type, subtype, metadata: { k } })
+      }
+    }
+    await brain.flush()
+  })
+  afterAll(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('WARM: all lenses agree with ground truth id-for-id', async () => {
+    await assertAllLenses(brain)
+  })
+
+  it('COLD REOPEN: all lenses still agree after close + reopen from disk', async () => {
+    await brain.close()
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+    await assertAllLenses(brain)
+  })
+
+  it('after an update() flips type AND subtype, every lens tracks the move exactly', async () => {
+    // The historical cross-bucket-staleness path: change (concept, action) -> (task, review).
+    const victims = await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1 })
+    expect(victims.length).toBe(1)
+    const id = victims[0].id
+    await brain.update({ id, type: 'task', subtype: 'review' })
+
+    const oldCombined = idSet(await brain.find({ type: 'concept', where: { subtype: 'action' }, limit: 1000 }))
+    expect(oldCombined.has(id)).toBe(false) // unposted from the old buckets
+    const newCombined = idSet(await brain.find({ type: 'task', where: { subtype: 'review' }, limit: 1000 }))
+    expect(newCombined.has(id)).toBe(true) // posted to the new buckets
+    const subtypeOnly = idSet(await brain.find({ where: { subtype: 'review' }, limit: 1000 }))
+    expect(subtypeOnly.has(id)).toBe(true)
+  })
+})

From c3feafdc477eefb7613e5084b52d88681a09d53d Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 09:42:29 -0700
Subject: [PATCH 46/82] docs: RELEASES.md entry for 8.3.3 (rename containment
 fix + repair)

---
 RELEASES.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index ca57a1ef..69159560 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,34 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory)
+
+One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed
+deletes, 8.3.2 fixed counters, this fixes moves).
+
+- **A cross-directory `vfs.rename()` now MOVES the containment edge instead of accumulating one per
+  parent.** The old parent's `Contains` edge was never removed on a move, leaving the entity a child
+  of **both** directories: `readdir(oldDir)` kept listing it after the move, re-creating the old path
+  showed the same name twice, and any tree-walking consumer (sync engines, file browsers) saw the
+  file in two places. The old edge is now removed by edge id, resolved from the graph's own adjacency
+  — a removal never requires reading the thing being removed. Bonus fix in the same seam: a move **to
+  the root** now gets its containment edge (it was previously skipped, orphaning the file out of
+  `readdir('/')`).
+
+- **`repairIndex()` now also reconciles VFS containment** (new `vfs.repairContainment()`): every VFS
+  entity's containment edges are checked against its canonical `metadata.path` — stale old-parent
+  ghosts and duplicate edges are removed, a missing expected edge is restored, and user
+  knowledge-graph edges are never touched (only `vfs-contains` edges are candidates). Loud per
+  repair. Stores that performed cross-directory renames under ≤8.3.2 should run `brain.repairIndex()`
+  once after upgrading — the same single ritual now heals orphan directories, counters, **and**
+  containment edges.
+
+- Also ships a permanent lens-consistency regression suite (combined type+subtype vs subtype-only vs
+  canonical ground truth, id-for-id, warm and after a cold reopen), ported from the field
+  investigation that closed the historical lens-drop report.
+
+No API changes beyond the new optional `vfs.repairContainment()` (also invoked by `repairIndex()`).
+
 ## v8.3.2 — 2026-07-14 (honest counters — the recount + removal-without-re-reading)
 
 Completes 8.3.1's delete-hygiene story at the counter layer, from a production proof chain reported

From 92299f27be561225ddb4a2e45e23b681269fd7b7 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 09:48:01 -0700
Subject: [PATCH 47/82] chore(release): 8.3.3

---
 CHANGELOG.md      | 7 +++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ea8f83e6..fcfeadb9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15)
+
+- docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd)
+- test: lens-consistency regression — combined vs subtype-only vs canonical ground truth (4fb41f9)
+- fix: VFS rename moves the containment edge — no ghost in the old directory (af8c179)
+
+
 ### [8.3.2](https://github.com/soulcraftlabs/brainy/compare/v8.3.1...v8.3.2) (2026-07-14)
 
 - docs: RELEASES.md entry for 8.3.2 (honest counters) (0932ecd)
diff --git a/package-lock.json b/package-lock.json
index d1e8be63..c4e4f9ab 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.2",
+  "version": "8.3.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.3.2",
+      "version": "8.3.3",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 9839bf0a..ee74980b 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.2",
+  "version": "8.3.3",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 38b0041464b81657f5ac4703f1e657f624b20a7e Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 10:49:02 -0700
Subject: [PATCH 48/82] =?UTF-8?q?feat:=20generation=20fact=20log=20?=
 =?UTF-8?q?=E2=80=94=20after-image=20commit=20records,=20dual-written=20at?=
 =?UTF-8?q?=20every=20commit=20point?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Every committed generation now also appends a FACT — an after-image
commit record (what each touched entity/relationship became, or a
body-less tombstone for a removal) — to an append-only, crc32c-framed
segment log under _generations/facts/. The before-image history and the
canonical tree remain authoritative; the fact log gives consumers ONE
sequential, self-verifying stream (index heals, incremental replays)
in place of a per-entity directory walk.

- Wire format: positional msgpack facts [generation, timestamp, ops,
  meta, blobHashes]; op = [kind u8, id bin16, record | nil tombstone];
  32-byte segment header (magic, formatVersion, firstGeneration,
  zeroed+verified reserved); length+crc32c frame per fact; zero-padded
  segment names so lexicographic order == generation order; JSON
  manifest with an atomic rename flip, manifest-first rotation.
- Commit protocol: facts append+fsync BEFORE the commit point inside
  the existing durability window, so a crash can only leave the log
  AHEAD of committed truth — open() truncates back (torn tails detected
  by CRC). Absent generation = never committed; a scan can never see an
  uncommitted fact. transact() facts are durable-on-return; single-op
  facts ride the group-commit flush exactly like buffered history. A
  fact-append failure fails the write, loudly — a silent gap would be a
  lie a later replay discovers.
- New public surface: brain.scanFacts() (sequential batches with heal
  telemetry: head/segments/approx up front, per-batch generation range
  + bytes + segment id, loud abort on gaps, summary cross-check) and
  brain.factSegmentPaths() (immutable sealed segments for zero-copy
  consumers; the mutable tail excluded). Exported types CommitFact,
  FactOp, FactScanBatch, FactScanHandle.
- Storage: optional binary raw-byte primitives (appendRawBytes,
  readRawBytes, writeRawBytes, rawByteSize) on StorageAdapter —
  feature-detected; filesystem + memory adapters implement them; an
  adapter without them hosts no fact log. Fact segments are byte-copied
  (never hard-linked) into snapshots. The _generations/facts/ namespace
  is registered as a protected family (rebuildable: false): no sweeper
  or GC may delete under it.
- New crc32c (Castagnoli) utility with RFC known-answer tests.
---
 src/brainy.ts                                 |  72 ++
 src/coreTypes.ts                              |  32 +
 src/db/factLog.ts                             | 670 ++++++++++++++++++
 src/db/generationStore.ts                     | 126 ++++
 src/db/types.ts                               |  16 +
 src/index.ts                                  |   8 +
 src/storage/adapters/fileSystemStorage.ts     |  83 ++-
 src/storage/adapters/memoryStorage.ts         |  40 ++
 src/utils/crc32c.ts                           |  43 ++
 tests/integration/db-mvcc.test.ts             |  12 +-
 tests/integration/fact-log-dual-write.test.ts | 200 ++++++
 tests/unit/db/fact-log.test.ts                | 189 +++++
 tests/unit/db/generationStore.test.ts         |   6 +-
 13 files changed, 1493 insertions(+), 4 deletions(-)
 create mode 100644 src/db/factLog.ts
 create mode 100644 src/utils/crc32c.ts
 create mode 100644 tests/integration/fact-log-dual-write.test.ts
 create mode 100644 tests/unit/db/fact-log.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index ef48c216..f79ca562 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -167,6 +167,7 @@ import {
   type ImportResult
 } from './db/portableGraph.js'
 import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js'
+import type { FactScanHandle } from './db/factLog.js'
 import {
   ChangeFeed,
   type BrainyChangeEvent,
@@ -973,6 +974,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         readOnly: this.config.mode === 'reader'
       })
 
+      // The generation fact log is CANONICAL state, not a derived index — no
+      // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare
+      // its namespace as a protected family (rebuildable: false — a lost fact
+      // segment is NOT reconstructable) so the storage layer REFUSES such
+      // deletes; refusal beats trust. Feature-detected + idempotent per name.
+      if (
+        this.config.mode !== 'reader' &&
+        this.generationStore.getFactLog() &&
+        typeof this.storage.registerDerivedFamily === 'function'
+      ) {
+        await this.storage.registerDerivedFamily({
+          name: 'generation-facts',
+          members: ['_generations/facts/'],
+          namespace: true,
+          rebuildable: false
+        })
+      }
+
       // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
       // marker (`_system/brain-format.json`) into an in-memory field NOW —
       // after the store-open phase, but BEFORE any derived index or native
@@ -7434,6 +7453,59 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     await this.removeMigrationBackupSafe()
   }
 
+  /**
+   * @description Open a sequential scan over the generation FACT LOG — the
+   * append-only record of every committed generation as an AFTER-IMAGE fact
+   * (what each touched entity/relationship became, or a body-less tombstone
+   * for a removal). The scan is the streaming substrate for index heals and
+   * incremental replays: one sequential read in commit order replaces a
+   * per-entity directory walk. The handle carries heal-narration telemetry
+   * (`headGeneration` / `segmentCount` / `approxFactCount` up front; ordered
+   * batches each stamped with their generation range, byte size, and segment;
+   * a `summary()` cross-check after iteration). A detected gap or damaged
+   * segment ABORTS the scan loudly — never a silent skip.
+   *
+   * Returns `null` when this store hosts no fact log: the storage adapter
+   * lacks the binary append primitives, or the brain predates the fact log
+   * (its history began before dual-write — facts exist only from the first
+   * write after upgrade; callers fall back to the canonical enumeration walk).
+   *
+   * @param options - `fromGeneration`/`toGeneration` bound the scan (inclusive
+   *   both ends); `kinds` filters ops to `'noun'`/`'verb'`; `batchSize` caps
+   *   facts per yielded batch (default 256).
+   * @returns The scan handle, or `null` when no fact log exists.
+   * @example
+   * const scan = brain.scanFacts({ fromGeneration: 1 })
+   * if (scan) {
+   *   for await (const batch of scan.batches()) {
+   *     for (const fact of batch.facts) {
+   *       // fact.ops: [{ kind, id, record | null (tombstone) }, ...]
+   *     }
+   *   }
+   * }
+   */
+  scanFacts(options?: {
+    fromGeneration?: number
+    toGeneration?: number
+    kinds?: Array<'noun' | 'verb'>
+    batchSize?: number
+  }): FactScanHandle | null {
+    const factLog = this.generationStore?.getFactLog()
+    return factLog ? factLog.scanFacts(options) : null
+  }
+
+  /**
+   * @description The immutable, sealed fact-log segment files covering
+   * `fromGeneration` — the zero-copy handoff for consumers that map segment
+   * files directly instead of streaming {@link scanFacts} batches. The
+   * append-mutable TAIL segment is deliberately excluded (read it via
+   * `scanFacts`). Paths are storage-root-relative. Empty when no fact log
+   * exists or nothing is sealed yet.
+   */
+  factSegmentPaths(options?: { fromGeneration?: number }): string[] {
+    return this.generationStore?.getFactLog()?.segmentPaths(options) ?? []
+  }
+
   /**
    * @description Read the reified transaction log — one entry per committed
    * generation, carrying the committed generation, the commit timestamp, and
diff --git a/src/coreTypes.ts b/src/coreTypes.ts
index b20d128d..dc4b3860 100644
--- a/src/coreTypes.ts
+++ b/src/coreTypes.ts
@@ -1127,6 +1127,38 @@ export interface StorageAdapter {
    */
   listDerivedFamilies?(): Promise<DerivedFamilyDeclaration[]>
 
+  /**
+   * @description OPTIONAL binary raw-byte primitives — the substrate for
+   * append-only log-structured files (the generation fact log's CRC-framed
+   * segments). Feature-detected: an adapter that omits them simply hosts no
+   * fact log (readers fall back to canonical enumeration). Paths are
+   * storage-root-relative and used VERBATIM (no `.gz`/`.bin` suffixing —
+   * unlike the JSON object and blob primitives).
+   *
+   * Append to a raw binary file, creating it (and parent directories) when
+   * absent. Append durability is the CALLER's job via `syncRawObjects` —
+   * matching the commit protocol, which batches fsyncs at its barrier.
+   */
+  appendRawBytes?(path: string, bytes: Uint8Array): Promise<void>
+
+  /**
+   * Read a raw binary file whole. Absent → `null`; a real IO fault throws
+   * (never masked as absence).
+   */
+  readRawBytes?(path: string): Promise<Uint8Array | null>
+
+  /**
+   * Replace a raw binary file atomically (write-new → fsync → rename) — the
+   * reconcile primitive (e.g. truncating a fact-log tail back to committed
+   * truth after a crash).
+   */
+  writeRawBytes?(path: string, bytes: Uint8Array): Promise<void>
+
+  /**
+   * Byte size of a raw binary file, or `null` when absent.
+   */
+  rawByteSize?(path: string): Promise<number | null>
+
   /**
    * Save statistics data
    * @param statistics The statistics data to save
diff --git a/src/db/factLog.ts b/src/db/factLog.ts
new file mode 100644
index 00000000..b43e52d0
--- /dev/null
+++ b/src/db/factLog.ts
@@ -0,0 +1,670 @@
+/**
+ * @module db/factLog
+ * @description The generation FACT LOG — an append-only, CRC-framed record of
+ * every committed generation as an AFTER-IMAGE "fact": what each touched
+ * entity/relationship BECAME (or a body-less tombstone when it was removed).
+ * This is the dual-write half of the log-canonical transition: today the
+ * before-image history + canonical tree remain authoritative; the fact log is
+ * appended at the same commit points and reconciled to committed truth at
+ * open, so consumers (index heals, replays, scans) can read one sequential,
+ * self-verifying stream instead of walking the entity tree.
+ *
+ * ## Wire format (frozen; additive-only within a major)
+ *
+ * Fact (msgpack, POSITIONAL array — the segment header's formatVersion
+ * governs the schema):
+ *
+ *   fact := [ generation:u64, timestamp:u64, ops, meta|nil, blobHashes|nil ]
+ *   op   := [ kind:u8 (0=noun, 1=verb), id:bin16 (raw uuid bytes),
+ *             record:[metaLeg, vecLeg] | nil ]        // nil = TOMBSTONE
+ *
+ * Segment file (`_generations/facts/seg-<firstGeneration, zero-padded 20>.bfl`):
+ *
+ *   header := magic "BFACTS\0\0" (8B) | formatVersion:u32 LE |
+ *             firstGeneration:u64 LE | reserved 12B (ZEROED, verified)
+ *   frame  := length:u32 LE | crc32c:u32 LE (of payload) | payload
+ *
+ * A fact is never split across segments; a torn tail (length overrun or CRC
+ * mismatch) terminates that segment's scan — everything before it is intact.
+ * Zero-padded names make lexicographic order == generation order.
+ *
+ * ## Invariant
+ *
+ * After {@link FactLog.open}, the log contains EXACTLY the committed prefix:
+ * facts are appended BEFORE the commit point (inside the same durability
+ * window), so a crash can only leave the log AHEAD of committed truth — open
+ * truncates any fact beyond the committed generation. Absent generation =
+ * never committed; present = committed. A scan can never see an uncommitted
+ * fact.
+ *
+ * The manifest (`_generations/facts/manifest.json`, JSON — forensics stay
+ * terminal-readable) is the single source of truth for the segment SET;
+ * rotation flips it atomically (write-new → fsync → rename) BEFORE the new
+ * tail's first byte exists, so no segment file is ever unaccounted for.
+ */
+import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack'
+import { crc32c } from '../utils/crc32c.js'
+import { prodLog } from '../utils/logger.js'
+
+// Swappable msgpack implementation — defaults to the JS codec; a native
+// provider (registered via the plugin registry's 'msgpack' key) may replace
+// it. Byte-compatibility is the contract (positional arrays, bin16 ids).
+let msgpackEncode: (value: unknown) => Uint8Array = defaultEncode
+let msgpackDecode: (bytes: Uint8Array) => unknown = defaultDecode
+
+/** Replace the msgpack encode/decode implementation at runtime. */
+export function setFactCodec(impl: {
+  encode: (value: unknown) => Uint8Array
+  decode: (bytes: Uint8Array) => unknown
+}): void {
+  msgpackEncode = impl.encode
+  msgpackDecode = impl.decode
+}
+
+/** Storage-root-relative home of the fact log. */
+export const FACTS_PREFIX = '_generations/facts'
+/** The facts manifest path (JSON). */
+export const FACTS_MANIFEST_PATH = `${FACTS_PREFIX}/manifest.json`
+/** Current segment format version (header field; additive-only within a major). */
+export const FACTS_FORMAT_VERSION = 1
+/** Rotation threshold: seal the tail segment once it exceeds this many bytes. */
+const SEGMENT_ROTATE_BYTES = 8 * 1024 * 1024
+/** Segment header: magic(8) + formatVersion(4) + firstGeneration(8) + reserved(12). */
+const HEADER_BYTES = 32
+const MAGIC = new Uint8Array([0x42, 0x46, 0x41, 0x43, 0x54, 0x53, 0x00, 0x00]) // "BFACTS\0\0"
+/** Frame prefix: length(4) + crc32c(4). */
+const FRAME_PREFIX_BYTES = 8
+
+/** One write inside a fact: what the id became (or a tombstone). */
+export interface FactOp {
+  kind: 'noun' | 'verb'
+  id: string
+  /** The AFTER-IMAGE legs, or `null` for a tombstone (the id was removed). */
+  record: { metadata: unknown | null; vector: unknown | null } | null
+}
+
+/** One committed generation, as scanned back out of the log. */
+export interface CommitFact {
+  generation: number
+  timestamp: number
+  ops: FactOp[]
+  meta?: Record<string, unknown>
+  blobHashes?: string[]
+}
+
+/** The telemetry a scan batch carries (frozen shape). */
+export interface FactScanBatch {
+  facts: CommitFact[]
+  firstGeneration: number
+  lastGeneration: number
+  factCount: number
+  byteSize: number
+  segmentId: string
+}
+
+/** The telemetry a scan OPEN returns (frozen shape). */
+export interface FactScanHandle {
+  headGeneration: number
+  segmentCount: number
+  approxFactCount: number
+  /** Ordered batches; a detected gap aborts LOUDLY, never a silent skip. */
+  batches: () => AsyncGenerator<FactScanBatch>
+  /** Close telemetry — the invariant cross-check, valid after iteration ends. */
+  summary: () => { factsYielded: number; segmentsRead: number }
+}
+
+/** Manifest entry for a sealed segment. */
+interface SegmentEntry {
+  file: string
+  firstGeneration: number
+  lastGeneration: number
+  facts: number
+  bytes: number
+}
+
+/** The facts manifest (JSON on disk). */
+interface FactsManifest {
+  formatVersion: number
+  segments: SegmentEntry[]
+  /** The append target. Its true content is established by scanning (crash tolerance). */
+  tailSegment: string | null
+  updatedAt: string
+}
+
+/** The narrow byte-level storage surface the fact log rides. */
+export interface FactLogStorage {
+  appendRawBytes(path: string, bytes: Uint8Array): Promise<void>
+  readRawBytes(path: string): Promise<Uint8Array | null>
+  writeRawBytes(path: string, bytes: Uint8Array): Promise<void>
+  rawByteSize(path: string): Promise<number | null>
+  readRawObject(path: string): Promise<any | null>
+  writeRawObject(path: string, data: any): Promise<void>
+  syncRawObjects(paths: string[]): Promise<void>
+  deleteRawObject(path: string): Promise<void>
+}
+
+/** True when the storage adapter exposes every primitive the fact log needs. */
+export function storageSupportsFactLog(storage: unknown): storage is FactLogStorage {
+  const s = storage as Record<string, unknown>
+  return (
+    typeof s.appendRawBytes === 'function' &&
+    typeof s.readRawBytes === 'function' &&
+    typeof s.writeRawBytes === 'function' &&
+    typeof s.rawByteSize === 'function'
+  )
+}
+
+/** uuid string → 16 raw bytes (bin16 on the wire). */
+function uuidToBytes(id: string): Uint8Array {
+  const hex = id.replace(/-/g, '')
+  if (hex.length !== 32) {
+    // Non-uuid ids (legacy/natural keys) ride as UTF-8 with a length prefix
+    // marker impossible for uuids: we refuse instead — the write API has
+    // guaranteed uuid ids since 8.0, so anything else is a corruption signal.
+    throw new Error(`fact log: id is not a uuid: ${id}`)
+  }
+  const bytes = new Uint8Array(16)
+  for (let i = 0; i < 16; i++) {
+    bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16)
+  }
+  return bytes
+}
+
+/** 16 raw bytes → canonical lowercase uuid string. */
+function bytesToUuid(bytes: Uint8Array): string {
+  let hex = ''
+  for (let i = 0; i < 16; i++) hex += bytes[i].toString(16).padStart(2, '0')
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`
+}
+
+/** Zero-padded segment filename: lexicographic order == generation order. */
+function segmentFileName(firstGeneration: number): string {
+  return `seg-${String(firstGeneration).padStart(20, '0')}.bfl`
+}
+
+/** Build a segment header. Reserved bytes are ZEROED (and verified on open). */
+function buildHeader(firstGeneration: number): Uint8Array {
+  const header = new Uint8Array(HEADER_BYTES)
+  header.set(MAGIC, 0)
+  const view = new DataView(header.buffer)
+  view.setUint32(8, FACTS_FORMAT_VERSION, true)
+  view.setBigUint64(12, BigInt(firstGeneration), true)
+  // bytes 20..31 stay zero (reserved)
+  return header
+}
+
+/** Encode one fact into a framed record (length + crc32c + msgpack payload). */
+function encodeFrame(fact: CommitFact): Uint8Array {
+  const payload = msgpackEncode([
+    fact.generation,
+    fact.timestamp,
+    fact.ops.map((op) => [
+      op.kind === 'noun' ? 0 : 1,
+      uuidToBytes(op.id),
+      op.record === null ? null : [op.record.metadata, op.record.vector]
+    ]),
+    fact.meta ?? null,
+    fact.blobHashes && fact.blobHashes.length > 0 ? fact.blobHashes : null
+  ])
+  const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length)
+  const view = new DataView(frame.buffer)
+  view.setUint32(0, payload.length, true)
+  view.setUint32(4, crc32c(payload), true)
+  frame.set(payload, FRAME_PREFIX_BYTES)
+  return frame
+}
+
+/** Decode one msgpack payload back into a CommitFact. */
+function decodeFact(payload: Uint8Array): CommitFact {
+  const raw = msgpackDecode(payload) as unknown[]
+  const [generation, timestamp, ops, meta, blobHashes] = raw as [
+    number,
+    number,
+    Array<[number, Uint8Array, [unknown, unknown] | null]>,
+    Record<string, unknown> | null,
+    string[] | null
+  ]
+  return {
+    generation: Number(generation),
+    timestamp: Number(timestamp),
+    ops: ops.map(([kind, idBytes, record]) => ({
+      kind: kind === 0 ? ('noun' as const) : ('verb' as const),
+      id: bytesToUuid(idBytes),
+      record: record === null ? null : { metadata: record[0] ?? null, vector: record[1] ?? null }
+    })),
+    ...(meta ? { meta } : {}),
+    ...(blobHashes && blobHashes.length > 0 ? { blobHashes } : {})
+  }
+}
+
+/**
+ * Parse a segment's bytes: verify the header, then walk frames until the end
+ * or a torn tail (length overrun / CRC mismatch), which terminates the walk —
+ * everything before it is intact. Returns the decoded facts plus the byte
+ * length of the VALID prefix (header + intact frames), which reconciliation
+ * uses to cut a torn tail without re-encoding.
+ */
+function parseSegment(
+  file: string,
+  bytes: Uint8Array
+): { facts: CommitFact[]; validBytes: number } {
+  if (bytes.length < HEADER_BYTES) {
+    prodLog.warn(`[FactLog] segment ${file} shorter than its header — treating as empty`)
+    return { facts: [], validBytes: 0 }
+  }
+  for (let i = 0; i < MAGIC.length; i++) {
+    if (bytes[i] !== MAGIC[i]) {
+      throw new Error(`fact log: segment ${file} has a bad magic — not a fact segment`)
+    }
+  }
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+  const version = view.getUint32(8, true)
+  if (version !== FACTS_FORMAT_VERSION) {
+    throw new Error(
+      `fact log: segment ${file} has formatVersion ${version}; this build reads ${FACTS_FORMAT_VERSION}`
+    )
+  }
+  for (let i = 20; i < HEADER_BYTES; i++) {
+    if (bytes[i] !== 0) {
+      // Non-zero reserved bytes = a future format this build cannot verify.
+      throw new Error(`fact log: segment ${file} has non-zero reserved header bytes — unverifiable`)
+    }
+  }
+
+  const facts: CommitFact[] = []
+  let offset = HEADER_BYTES
+  while (offset + FRAME_PREFIX_BYTES <= bytes.length) {
+    const length = view.getUint32(offset, true)
+    const expectedCrc = view.getUint32(offset + 4, true)
+    const start = offset + FRAME_PREFIX_BYTES
+    const end = start + length
+    if (end > bytes.length) break // torn tail: frame length overruns the file
+    const payload = bytes.subarray(start, end)
+    if (crc32c(payload) !== expectedCrc) break // torn tail: payload CRC mismatch
+    facts.push(decodeFact(payload))
+    offset = end
+  }
+  return { facts, validBytes: offset }
+}
+
+/**
+ * The generation fact log. One instance per open store; every method assumes
+ * the single-writer discipline the generation store already enforces (calls
+ * arrive under its commit mutex).
+ */
+export class FactLog {
+  private readonly storage: FactLogStorage
+  private manifest: FactsManifest = {
+    formatVersion: FACTS_FORMAT_VERSION,
+    segments: [],
+    tailSegment: null,
+    updatedAt: new Date(0).toISOString()
+  }
+  /** Decoded facts of the TAIL segment (bounded by the rotation threshold). */
+  private tailFacts: CommitFact[] = []
+  /** Byte size of the tail segment file (valid prefix). */
+  private tailBytes = 0
+  /** Highest generation in the log (0 = empty). */
+  private head = 0
+  /** Segment paths appended since the last sync (the fsync batch). */
+  private readonly dirtySegments = new Set<string>()
+
+  constructor(storage: FactLogStorage) {
+    this.storage = storage
+  }
+
+  /** The highest committed generation the log holds (0 = empty). */
+  headGeneration(): number {
+    return this.head
+  }
+
+  /**
+   * Open the log and reconcile it to committed truth: read the manifest,
+   * establish the tail's intact content (torn-tail scan), then TRUNCATE any
+   * fact with `generation > committedGeneration` — those never committed (a
+   * crash between fact-append and the commit point). After open, the log is
+   * exactly the committed prefix.
+   */
+  async open(committedGeneration: number): Promise<void> {
+    const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null
+    if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) {
+      if (stored.formatVersion !== FACTS_FORMAT_VERSION) {
+        throw new Error(
+          `fact log: manifest formatVersion ${stored.formatVersion}; this build reads ${FACTS_FORMAT_VERSION}`
+        )
+      }
+      this.manifest = stored
+    }
+
+    // Drop sealed segments that sit ENTIRELY beyond committed truth (a crash
+    // right after a rotation whose facts never committed), newest first.
+    while (this.manifest.segments.length > 0) {
+      const last = this.manifest.segments[this.manifest.segments.length - 1]
+      if (last.firstGeneration > committedGeneration) {
+        prodLog.warn(
+          `[FactLog] dropping sealed segment ${last.file} (generations ${last.firstGeneration}..` +
+            `${last.lastGeneration} never committed)`
+        )
+        await this.storage.deleteRawObject(`${FACTS_PREFIX}/${last.file}`)
+        this.manifest.segments.pop()
+        await this.persistManifest()
+      } else if (last.lastGeneration > committedGeneration) {
+        // A sealed segment STRADDLING committed truth: cut it back.
+        await this.truncateSegmentTo(last.file, committedGeneration)
+        const cut = await this.reloadSegmentEntry(last.file)
+        this.manifest.segments[this.manifest.segments.length - 1] = cut
+        await this.persistManifest()
+        break
+      } else {
+        break
+      }
+    }
+
+    // Establish the tail: scan its intact prefix, then truncate beyond
+    // committed truth (the common crash shape: buffered single-op facts whose
+    // counter never went durable).
+    if (this.manifest.tailSegment) {
+      const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}`
+      const bytes = await this.storage.readRawBytes(tailPath)
+      if (bytes === null) {
+        // Manifest named a tail whose first byte never landed — an empty tail.
+        this.tailFacts = []
+        this.tailBytes = 0
+      } else {
+        const { facts, validBytes } = parseSegment(this.manifest.tailSegment, bytes)
+        const kept = facts.filter((f) => f.generation <= committedGeneration)
+        if (kept.length !== facts.length || validBytes !== bytes.length) {
+          const dropped = facts.length - kept.length
+          if (dropped > 0) {
+            prodLog.warn(
+              `[FactLog] truncating ${dropped} uncommitted fact(s) beyond generation ` +
+                `${committedGeneration} from the tail (never committed)`
+            )
+          }
+          await this.rewriteTail(kept)
+        } else {
+          this.tailFacts = facts
+          this.tailBytes = validBytes
+        }
+      }
+    }
+
+    this.head = this.computeHead()
+  }
+
+  /**
+   * Append one committed generation's fact. NOT durable until {@link sync} —
+   * the caller batches durability at its commit barrier (transact syncs in
+   * the same call; Model-B group-commit syncs at flush).
+   */
+  async append(fact: CommitFact): Promise<void> {
+    if (fact.generation <= this.head) {
+      throw new Error(
+        `fact log: non-monotonic append (generation ${fact.generation} ≤ head ${this.head})`
+      )
+    }
+    if (this.manifest.tailSegment === null) {
+      await this.startTail(fact.generation)
+    } else if (this.tailBytes >= SEGMENT_ROTATE_BYTES) {
+      await this.rotate(fact.generation)
+    }
+    const frame = encodeFrame(fact)
+    const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}`
+    await this.storage.appendRawBytes(tailPath, frame)
+    this.tailFacts.push(fact)
+    this.tailBytes += frame.length
+    this.head = fact.generation
+    this.dirtySegments.add(tailPath)
+  }
+
+  /** Fsync every segment appended since the last sync. */
+  async sync(): Promise<void> {
+    if (this.dirtySegments.size === 0) return
+    const paths = [...this.dirtySegments]
+    this.dirtySegments.clear()
+    await this.storage.syncRawObjects(paths)
+  }
+
+  /**
+   * Open a scan over committed facts. The scan runs against a MANIFEST
+   * SNAPSHOT (sealed segments + the tail's decoded facts at open) — exactly-
+   * once per fact, inclusive bounds, stable under concurrent appends. Gaps
+   * abort LOUDLY: a missing generation inside a segment's declared range is
+   * corruption, never silently skipped.
+   */
+  scanFacts(options?: {
+    fromGeneration?: number
+    toGeneration?: number
+    kinds?: Array<'noun' | 'verb'>
+    batchSize?: number
+  }): FactScanHandle {
+    const from = options?.fromGeneration ?? 1
+    const to = options?.toGeneration ?? this.head
+    const kinds = options?.kinds
+    const batchSize = Math.max(1, options?.batchSize ?? 256)
+
+    // Snapshot: the segment list + tail content as of NOW.
+    const segments = this.manifest.segments.filter(
+      (s) => s.lastGeneration >= from && s.firstGeneration <= to
+    )
+    const tailSnapshot = this.tailFacts.filter((f) => f.generation >= from && f.generation <= to)
+    const tailId = this.manifest.tailSegment ?? 'tail'
+    const approxFactCount =
+      segments.reduce((sum, s) => sum + s.facts, 0) + tailSnapshot.length
+
+    let factsYielded = 0
+    let segmentsRead = 0
+    const storage = this.storage
+
+    async function* batches(this: void): AsyncGenerator<FactScanBatch> {
+      let expectedNext = 0 // gap detection: generations are monotonic, not necessarily dense
+      const emit = (facts: CommitFact[], segmentId: string, byteSize: number): FactScanBatch => ({
+        facts,
+        firstGeneration: facts[0].generation,
+        lastGeneration: facts[facts.length - 1].generation,
+        factCount: facts.length,
+        byteSize,
+        segmentId
+      })
+      const filterOps = (fact: CommitFact): CommitFact =>
+        kinds
+          ? { ...fact, ops: fact.ops.filter((op) => kinds.includes(op.kind)) }
+          : fact
+
+      for (const entry of segments) {
+        const bytes = await storage.readRawBytes(`${FACTS_PREFIX}/${entry.file}`)
+        if (bytes === null) {
+          throw new Error(
+            `fact log: sealed segment ${entry.file} is MISSING — the log is damaged; aborting scan`
+          )
+        }
+        const { facts } = parseSegment(entry.file, bytes)
+        segmentsRead++
+        const inRange = facts.filter((f) => f.generation >= from && f.generation <= to)
+        for (const f of inRange) {
+          if (f.generation <= expectedNext - 1) {
+            throw new Error(`fact log: out-of-order fact ${f.generation} in ${entry.file} — aborting scan`)
+          }
+          expectedNext = f.generation + 1
+        }
+        for (let i = 0; i < inRange.length; i += batchSize) {
+          const slice = inRange.slice(i, i + batchSize).map(filterOps)
+          if (slice.length === 0) continue
+          factsYielded += slice.length
+          yield emit(slice, entry.file, slice.reduce((n, f) => n + encodeFrame(f).length, 0))
+        }
+      }
+
+      if (tailSnapshot.length > 0) {
+        segmentsRead++
+        for (const f of tailSnapshot) {
+          if (f.generation <= expectedNext - 1) {
+            throw new Error(`fact log: out-of-order fact ${f.generation} in the tail — aborting scan`)
+          }
+          expectedNext = f.generation + 1
+        }
+        for (let i = 0; i < tailSnapshot.length; i += batchSize) {
+          const slice = tailSnapshot.slice(i, i + batchSize).map(filterOps)
+          factsYielded += slice.length
+          yield emit(slice, tailId, slice.reduce((n, f) => n + encodeFrame(f).length, 0))
+        }
+      }
+    }
+
+    return {
+      headGeneration: this.head,
+      segmentCount: segments.length + (tailSnapshot.length > 0 ? 1 : 0),
+      approxFactCount,
+      batches,
+      summary: () => ({ factsYielded, segmentsRead })
+    }
+  }
+
+  /**
+   * The mmap fast path (capability handoff): the immutable sealed segment
+   * files covering `fromGeneration`, in order. The TAIL is deliberately NOT
+   * included — it is append-mutable; consumers read it via {@link scanFacts}.
+   */
+  segmentPaths(options?: { fromGeneration?: number }): string[] {
+    const from = options?.fromGeneration ?? 1
+    return this.manifest.segments
+      .filter((s) => s.lastGeneration >= from)
+      .map((s) => `${FACTS_PREFIX}/${s.file}`)
+  }
+
+  /**
+   * Drop every fact with `generation > keepThrough` — the in-session abort
+   * compensation: a transact appends its fact BEFORE the commit point, so a
+   * real (non-crash) abort after the append must take the fact back out. The
+   * dropped facts can only live in the TAIL (they were just appended); the
+   * rewrite is atomic and bounded by the rotation threshold.
+   */
+  async dropAbove(keepThrough: number): Promise<void> {
+    if (this.head <= keepThrough) return
+    const kept = this.tailFacts.filter((f) => f.generation <= keepThrough)
+    if (kept.length === this.tailFacts.length) {
+      throw new Error(
+        `fact log: dropAbove(${keepThrough}) found no droppable facts in the tail ` +
+          `(head ${this.head}) — the fact to drop was already sealed; the log needs reopen`
+      )
+    }
+    await this.rewriteTail(kept)
+    this.head = this.computeHead()
+  }
+
+  // -- internals -------------------------------------------------------------
+
+  private computeHead(): number {
+    if (this.tailFacts.length > 0) return this.tailFacts[this.tailFacts.length - 1].generation
+    const sealed = this.manifest.segments
+    if (sealed.length > 0) return sealed[sealed.length - 1].lastGeneration
+    return 0
+  }
+
+  /** Create the very first tail segment (manifest-first, then header bytes). */
+  private async startTail(firstGeneration: number): Promise<void> {
+    const file = segmentFileName(firstGeneration)
+    this.manifest.tailSegment = file
+    await this.persistManifest()
+    await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, buildHeader(firstGeneration))
+    this.tailFacts = []
+    this.tailBytes = HEADER_BYTES
+  }
+
+  /**
+   * Seal the tail into the manifest and start a new one. Manifest-first: the
+   * flip both seals the old tail AND names the new one atomically, so no
+   * segment file ever exists unaccounted for.
+   */
+  private async rotate(nextGeneration: number): Promise<void> {
+    const sealedFile = this.manifest.tailSegment
+    if (!sealedFile) return
+    // Seal what the tail actually holds.
+    await this.sync() // sealed segments are always fully durable
+    const entry: SegmentEntry = {
+      file: sealedFile,
+      firstGeneration: this.tailFacts[0]?.generation ?? nextGeneration,
+      lastGeneration: this.tailFacts[this.tailFacts.length - 1]?.generation ?? nextGeneration - 1,
+      facts: this.tailFacts.length,
+      bytes: this.tailBytes
+    }
+    const newFile = segmentFileName(nextGeneration)
+    this.manifest.segments.push(entry)
+    this.manifest.tailSegment = newFile
+    await this.persistManifest()
+    await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, buildHeader(nextGeneration))
+    this.tailFacts = []
+    this.tailBytes = HEADER_BYTES
+  }
+
+  /** Atomically persist the manifest (write-new → fsync → rename downstream). */
+  private async persistManifest(): Promise<void> {
+    this.manifest.updatedAt = new Date().toISOString()
+    await this.storage.writeRawObject(FACTS_MANIFEST_PATH, this.manifest)
+    await this.storage.syncRawObjects([FACTS_MANIFEST_PATH])
+  }
+
+  /** Rewrite the tail segment to hold exactly `facts` (atomic replace). */
+  private async rewriteTail(facts: CommitFact[]): Promise<void> {
+    const file = this.manifest.tailSegment
+    if (!file) return
+    const first = facts[0]?.generation ?? this.segmentFirstGenerationFromName(file)
+    const parts: Uint8Array[] = [buildHeader(first)]
+    for (const f of facts) parts.push(encodeFrame(f))
+    const total = parts.reduce((n, p) => n + p.length, 0)
+    const merged = new Uint8Array(total)
+    let offset = 0
+    for (const p of parts) {
+      merged.set(p, offset)
+      offset += p.length
+    }
+    await this.storage.writeRawBytes(`${FACTS_PREFIX}/${file}`, merged)
+    this.tailFacts = facts
+    this.tailBytes = total
+  }
+
+  /** Cut a SEALED segment back to `committedGeneration` (atomic replace). */
+  private async truncateSegmentTo(file: string, committedGeneration: number): Promise<void> {
+    const path = `${FACTS_PREFIX}/${file}`
+    const bytes = await this.storage.readRawBytes(path)
+    if (bytes === null) return
+    const { facts } = parseSegment(file, bytes)
+    const kept = facts.filter((f) => f.generation <= committedGeneration)
+    prodLog.warn(
+      `[FactLog] truncating sealed segment ${file} to generation ${committedGeneration} ` +
+        `(${facts.length - kept.length} uncommitted fact(s) dropped)`
+    )
+    const first = kept[0]?.generation ?? this.segmentFirstGenerationFromName(file)
+    const parts: Uint8Array[] = [buildHeader(first)]
+    for (const f of kept) parts.push(encodeFrame(f))
+    const total = parts.reduce((n, p) => n + p.length, 0)
+    const merged = new Uint8Array(total)
+    let offset = 0
+    for (const p of parts) {
+      merged.set(p, offset)
+      offset += p.length
+    }
+    await this.storage.writeRawBytes(path, merged)
+  }
+
+  /** Re-derive a sealed segment's manifest entry from its actual bytes. */
+  private async reloadSegmentEntry(file: string): Promise<SegmentEntry> {
+    const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`)
+    const { facts, validBytes } = bytes
+      ? parseSegment(file, bytes)
+      : { facts: [] as CommitFact[], validBytes: 0 }
+    return {
+      file,
+      firstGeneration: facts[0]?.generation ?? this.segmentFirstGenerationFromName(file),
+      lastGeneration: facts[facts.length - 1]?.generation ?? 0,
+      facts: facts.length,
+      bytes: validBytes
+    }
+  }
+
+  /** Parse the zero-padded firstGeneration back out of a segment filename. */
+  private segmentFirstGenerationFromName(file: string): number {
+    const match = /^seg-(\d{20})\.bfl$/.exec(file)
+    return match ? Number(match[1]) : 0
+  }
+}
diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts
index d7a302c9..0d813f4d 100644
--- a/src/db/generationStore.ts
+++ b/src/db/generationStore.ts
@@ -45,6 +45,7 @@ import type {
   GenerationStorage,
   TxLogEntry
 } from './types.js'
+import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js'
 
 /**
  * The byte-identical before-images of every id a commit touches, read UNDER
@@ -121,6 +122,16 @@ export interface GenerationStoreOpenResult {
 export class GenerationStore {
   private readonly storage: GenerationStorage
 
+  /**
+   * The generation FACT LOG (dual-write transition) — an append-only,
+   * CRC-framed record of every committed generation as an AFTER-IMAGE fact.
+   * `null` when the storage layer lacks the binary raw-byte primitives.
+   * Appends ride the same commit protocol: a fact-append failure FAILS the
+   * write (loud — a silent fact gap would make the log a lie that a later
+   * replay discovers), and open() reconciles the log to committed truth.
+   */
+  private factLog: FactLog | null = null
+
   /** Latest reserved/observed generation (≥ {@link committed}). */
   private counter = 0
   /** Committed-transaction watermark (manifest generation). */
@@ -400,6 +411,19 @@ export class GenerationStore {
 
     this.opened = true
 
+    // Generation FACT LOG (dual-write transition): when the storage layer
+    // exposes the binary raw-byte primitives, open the after-image fact log
+    // and reconcile it to committed truth — facts are appended BEFORE the
+    // commit point, so a crash can only leave the log AHEAD; open truncates
+    // any fact beyond `committed`. Storage without the primitives simply
+    // hosts no fact log (readers fall back to canonical enumeration).
+    if (storageSupportsFactLog(this.storage)) {
+      this.factLog = new FactLog(this.storage)
+      await this.factLog.open(this.committed)
+    } else {
+      this.factLog = null
+    }
+
     // Hook single-op write batches so generation() is always meaningful.
     // Suppressed while a transact batch executes (the batch is ONE generation).
     if (!options?.readOnly) {
@@ -600,6 +624,58 @@ export class GenerationStore {
    * @returns The committed generation and its commit timestamp.
    * @throws GenerationConflictError when the CAS expectation fails.
    */
+  /**
+   * The generation fact log, or `null` when the storage layer cannot host one.
+   * Consumers scan committed facts through it (`scanFacts` / `segmentPaths`).
+   */
+  getFactLog(): FactLog | null {
+    return this.factLog
+  }
+
+  /**
+   * @description Build one commit's AFTER-IMAGE fact by reading canonical
+   * state back for every touched id — under the commit mutex, immediately
+   * after the operations applied, so canonical IS the after-image (and the
+   * reads are page-cache-warm: the operations just wrote these files). An
+   * absent id (both legs null) becomes a body-less TOMBSTONE — the delete
+   * fact needs no body, so removal never requires reading the removed thing.
+   * The fact's blobHashes are extracted from the AFTER records (the content
+   * this generation's state references), unlike the history path's
+   * before-image hashes.
+   */
+  private async buildCommitFact(args: {
+    generation: number
+    timestamp: number
+    nouns: string[]
+    verbs: string[]
+    meta?: Record<string, unknown>
+  }): Promise<CommitFact> {
+    const ops: FactOp[] = []
+    const afterRecords: GenerationRecord[] = []
+    for (const id of args.nouns) {
+      const after = await this.storage.readNounRaw(id)
+      const absent = after.metadata === null && after.vector === null
+      ops.push({ kind: 'noun', id, record: absent ? null : after })
+      if (!absent) afterRecords.push({ kind: 'noun', metadata: after.metadata, vector: after.vector })
+    }
+    for (const id of args.verbs) {
+      const after = await this.storage.readVerbRaw(id)
+      const absent = after.metadata === null && after.vector === null
+      ops.push({ kind: 'verb', id, record: absent ? null : after })
+      if (!absent) afterRecords.push({ kind: 'verb', metadata: after.metadata, vector: after.vector })
+    }
+    const blobHashes = this.storage.extractBlobHashesFromRecords
+      ? this.storage.extractBlobHashesFromRecords(afterRecords)
+      : []
+    return {
+      generation: args.generation,
+      timestamp: args.timestamp,
+      ops,
+      ...(args.meta ? { meta: args.meta } : {}),
+      ...(blobHashes.length > 0 ? { blobHashes } : {})
+    }
+  }
+
   async commitTransaction(args: {
     touched: TouchedIds
     meta?: Record<string, unknown>
@@ -733,6 +809,24 @@ export class GenerationStore {
         await this.storage.flushWriteBarrier?.()
         faultPoint('after-execute')
 
+        // Fact log (dual-write): append + fsync this generation's AFTER-IMAGE
+        // fact BEFORE the commit point, inside the same durability window —
+        // so a crash can only leave the log AHEAD (open truncates), never a
+        // committed generation without its fact. A real abort below this point
+        // compensates via dropAbove in the catch. An append failure fails the
+        // write, loudly — a silent fact gap would be a lie a replay discovers.
+        if (this.factLog) {
+          const fact = await this.buildCommitFact({
+            generation: gen,
+            timestamp,
+            nouns,
+            verbs,
+            ...(args.meta ? { meta: args.meta } : {})
+          })
+          await this.factLog.append(fact)
+          await this.factLog.sync()
+        }
+
         // -- 5. Counter + manifest rename (COMMIT POINT) ----------------------
         await this.persistCounterUnlocked()
         faultPoint('before-manifest-rename')
@@ -800,6 +894,13 @@ export class GenerationStore {
             // over-count-safe; the scrub restores exactness
           }
         }
+        // Fact-log compensation: a real (non-crash) abort after the fact was
+        // appended must take the fact back out — the generation never
+        // committed. A crash instead reaches open(), whose truncation does the
+        // same reconcile from disk.
+        if (this.factLog && this.factLog.headGeneration() >= gen) {
+          await this.factLog.dropAbove(gen - 1)
+        }
         // Return the reservation when no concurrent bump consumed a later
         // number, so a failed transaction leaves generation() unchanged.
         if (this.counter === gen) this.counter = gen - 1
@@ -991,6 +1092,14 @@ export class GenerationStore {
             this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp })
             this.pendingGens.push(gen)
             this.extendChains(gen, nouns, verbs)
+            // The adopted generation is committed — it gets its fact like any
+            // other (durability rides the group-commit flush, same as the
+            // buffered history).
+            if (this.factLog) {
+              await this.factLog.append(
+                await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs })
+              )
+            }
             prodLog.warn(
               `[GenerationStore] Recovered a failed rollback FORWARD: single-op write ` +
                 `committed as generation ${gen} because its canonical undo could not be ` +
@@ -1020,6 +1129,17 @@ export class GenerationStore {
       this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp })
       this.pendingGens.push(gen)
       this.extendChains(gen, nouns, verbs)
+      // Fact log (dual-write): the acked write's AFTER-IMAGE fact, appended
+      // now (read back warm, under the mutex — group-commit means flush-time
+      // canonical only holds the LATEST state, so each generation's after-image
+      // exists only here). Durability rides the group-commit flush, exactly
+      // like the buffered before-image history: a crash before the flush loses
+      // the fact AND the generation together — never a torn state.
+      if (this.factLog) {
+        await this.factLog.append(
+          await this.buildCommitFact({ generation: gen, timestamp, nouns, verbs })
+        )
+      }
       this.schedulePendingFlush()
       return { generation: gen, timestamp }
     })
@@ -1141,6 +1261,12 @@ export class GenerationStore {
       // ONE fsync for the whole window — the durability-batching win.
       await this.storage.syncRawObjects(stagedPaths)
 
+      // Fact log (dual-write): make the window's buffered facts durable in the
+      // same batch, BEFORE the commit point below — so a crash can only leave
+      // the log AHEAD of the counter (open truncates), never a committed
+      // generation without its durable fact.
+      await this.factLog?.sync()
+
       // Test-only crash simulation: a throwing injector here leaves the staged
       // group-commit generation dirs on disk with NO manifest advance — the
       // exact "crashed mid-flush" state recovery must DROP-WITHOUT-RESTORE
diff --git a/src/db/types.ts b/src/db/types.ts
index 4f932e76..2bfd6ac2 100644
--- a/src/db/types.ts
+++ b/src/db/types.ts
@@ -423,6 +423,22 @@ export interface GenerationStorage {
   /** Read all lines of `_system/tx-log.jsonl` (empty array if absent). */
   readTxLogLines(): Promise<string[]>
 
+  /**
+   * OPTIONAL binary raw-byte primitives — the substrate for the generation
+   * fact log's append-only CRC-framed segments. Feature-detected: a storage
+   * layer that omits them hosts no fact log (dual-write is skipped; readers
+   * fall back to canonical enumeration). Paths are used VERBATIM (no
+   * suffixing). Append durability rides `syncRawObjects` at the commit
+   * barrier, exactly like the staged history files.
+   */
+  appendRawBytes?(path: string, bytes: Uint8Array): Promise<void>
+  /** Read a raw binary file whole; absent → null; a real fault throws. */
+  readRawBytes?(path: string): Promise<Uint8Array | null>
+  /** Replace a raw binary file atomically (tmp → fsync → rename). */
+  writeRawBytes?(path: string, bytes: Uint8Array): Promise<void>
+  /** Byte size of a raw binary file, or null when absent. */
+  rawByteSize?(path: string): Promise<number | null>
+
   /**
    * OPTIONAL temporal-blob contract (implemented by blob-aware storage; the
    * generation store treats the hashes as opaque strings). Extract the
diff --git a/src/index.ts b/src/index.ts
index b0a95fe1..b9135600 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -196,6 +196,14 @@ export type {
   HistoryVersion,
   EntityHistory
 } from './db/types.js'
+// The generation fact log — sequential after-image scan surface
+// (brain.scanFacts / brain.factSegmentPaths) for index heals and replays.
+export type {
+  CommitFact,
+  FactOp,
+  FactScanBatch,
+  FactScanHandle
+} from './db/factLog.js'
 // Optional provider capability for generation-aware native indexes
 export { isVersionedIndexProvider } from './plugin.js'
 export type { VersionedIndexProvider } from './plugin.js'
diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index a9e3014c..4f246f67 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -693,6 +693,17 @@ export class FileSystemStorage extends BaseStorage {
    */
   private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set<string>(['_id_mapper'])
 
+  /**
+   * Nested path PREFIXES whose files are byte-copied into snapshots, not
+   * hard-linked — for append-in-place files below the top level. The
+   * generation fact log's tail segment is appended in place between rotations;
+   * a hard-linked tail would let post-snapshot appends reach through into the
+   * snapshot. (Sealed segments are immutable and would be link-safe, but the
+   * prefix rule keeps the discipline simple; segments are bounded by the
+   * rotation threshold, so the copy cost is small.)
+   */
+  private static readonly SNAPSHOT_BYTE_COPY_PREFIXES: string[] = ['_generations/facts/']
+
   /**
    * Top-level directories excluded from snapshots: process-local lock state
    * (writer lock, flush-request RPC files) must never travel with the data, and
@@ -870,6 +881,75 @@ export class FileSystemStorage extends BaseStorage {
     }
   }
 
+  // ==========================================================================
+  // Binary raw-byte primitives — the substrate for append-only log-structured
+  // files (the generation fact log's CRC-framed segments). Paths are used
+  // VERBATIM (no .gz/.bin suffixing). Append durability rides syncRawObjects
+  // at the commit barrier, like every other staged write.
+  // ==========================================================================
+
+  /**
+   * Append bytes to a raw binary file, creating it (and parent directories)
+   * when absent. NOT fsync'd here — the caller batches durability via
+   * `syncRawObjects` at its commit barrier.
+   */
+  public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
+    await this.ensureInitialized()
+    const fullPath = path.join(this.rootDir, rawPath)
+    await fs.promises.mkdir(path.dirname(fullPath), { recursive: true })
+    await fs.promises.appendFile(fullPath, bytes)
+  }
+
+  /**
+   * Read a raw binary file whole. Absent → `null`; a real IO fault throws —
+   * a present-but-unreadable log segment must never read as "no facts".
+   */
+  public async readRawBytes(rawPath: string): Promise<Uint8Array | null> {
+    await this.ensureInitialized()
+    try {
+      const buf: Buffer = await fs.promises.readFile(path.join(this.rootDir, rawPath))
+      return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
+    } catch (error: any) {
+      if (isAbsentError(error)) return null
+      throw error
+    }
+  }
+
+  /**
+   * Replace a raw binary file atomically: write-new → fsync → rename. The
+   * reconcile primitive (e.g. truncating a fact-log tail back to committed
+   * truth after a crash) — a crash mid-replace leaves either the old file or
+   * the new one, never a mix.
+   */
+  public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
+    await this.ensureInitialized()
+    const fullPath = path.join(this.rootDir, rawPath)
+    await fs.promises.mkdir(path.dirname(fullPath), { recursive: true })
+    const tmpPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}`
+    const handle = await fs.promises.open(tmpPath, 'w')
+    try {
+      await handle.writeFile(bytes)
+      await handle.sync()
+    } finally {
+      await handle.close()
+    }
+    await fs.promises.rename(tmpPath, fullPath)
+  }
+
+  /**
+   * Byte size of a raw binary file, or `null` when absent.
+   */
+  public async rawByteSize(rawPath: string): Promise<number | null> {
+    await this.ensureInitialized()
+    try {
+      const stat = await fs.promises.stat(path.join(this.rootDir, rawPath))
+      return stat.size
+    } catch (error: any) {
+      if (isAbsentError(error)) return null
+      throw error
+    }
+  }
+
   /**
    * Snapshot the entire store into `targetPath` as a hard-link farm
    * (Cassandra-style: instant, space-shared). Safe because every data file
@@ -919,7 +999,8 @@ export class FileSystemStorage extends BaseStorage {
       // msync/truncate reach through into the snapshot.
       if (
         FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) ||
-        FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0])
+        FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) ||
+        FileSystemStorage.SNAPSHOT_BYTE_COPY_PREFIXES.some((p) => normalized.startsWith(p))
       ) {
         await fs.promises.copyFile(sourceFile, targetFile)
         continue
diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts
index 41987ea0..bab9d4d9 100644
--- a/src/storage/adapters/memoryStorage.ts
+++ b/src/storage/adapters/memoryStorage.ts
@@ -222,6 +222,45 @@ export class MemoryStorage extends BaseStorage {
     return [...this.txLogLines]
   }
 
+  // ===========================================================================
+  // Binary raw-byte primitives — in-memory mirror of the filesystem adapter's
+  // append-only substrate (the generation fact log's segments), so memory
+  // brains dual-write facts too and the compat suite runs on both adapters.
+  // ===========================================================================
+
+  /** Raw binary files, keyed by verbatim path. */
+  private rawBytesStore: Map<string, Uint8Array> = new Map()
+
+  /** Append bytes to a raw binary file, creating it when absent. */
+  public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
+    const existing = this.rawBytesStore.get(rawPath)
+    if (!existing) {
+      this.rawBytesStore.set(rawPath, bytes.slice())
+      return
+    }
+    const merged = new Uint8Array(existing.length + bytes.length)
+    merged.set(existing, 0)
+    merged.set(bytes, existing.length)
+    this.rawBytesStore.set(rawPath, merged)
+  }
+
+  /** Read a raw binary file whole (a copy); absent → null. */
+  public async readRawBytes(rawPath: string): Promise<Uint8Array | null> {
+    const bytes = this.rawBytesStore.get(rawPath)
+    return bytes ? bytes.slice() : null
+  }
+
+  /** Replace a raw binary file (atomic by construction in memory). */
+  public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
+    this.rawBytesStore.set(rawPath, bytes.slice())
+  }
+
+  /** Byte size of a raw binary file, or null when absent. */
+  public async rawByteSize(rawPath: string): Promise<number | null> {
+    const bytes = this.rawBytesStore.get(rawPath)
+    return bytes ? bytes.length : null
+  }
+
   /**
    * Serialize the entire in-memory store to a directory in the exact layout
    * the filesystem adapter uses (uncompressed JSON objects, `_blobs/*.bin`
@@ -354,6 +393,7 @@ export class MemoryStorage extends BaseStorage {
   public async clear(): Promise<void> {
     this.objectStore.clear()
     this.blobStore.clear()
+    this.rawBytesStore.clear()
     this.txLogLines = []
     this.statistics = null
 
diff --git a/src/utils/crc32c.ts b/src/utils/crc32c.ts
new file mode 100644
index 00000000..0c2a1c14
--- /dev/null
+++ b/src/utils/crc32c.ts
@@ -0,0 +1,43 @@
+/**
+ * @module utils/crc32c
+ * @description CRC-32C (Castagnoli, polynomial 0x1EDC6F41, reflected 0x82F63B78)
+ * — the storage-industry frame checksum (ext4, iSCSI, SCTP, LSM segment files).
+ * Used to frame generation-fact segments: every appended record carries the
+ * CRC-32C of its payload, so a torn tail (crash mid-append) or bit rot is
+ * DETECTED at scan time and never silently read as data.
+ *
+ * Table-driven, dependency-free reference implementation. Native providers may
+ * substitute a hardware-accelerated (SSE4.2 / ARMv8 CRC) implementation — the
+ * polynomial is the contract, byte-identical results required.
+ */
+
+/** The 256-entry lookup table for the reflected CRC-32C polynomial. */
+const TABLE: Uint32Array = (() => {
+  const table = new Uint32Array(256)
+  for (let n = 0; n < 256; n++) {
+    let c = n
+    for (let k = 0; k < 8; k++) {
+      c = c & 1 ? 0x82f63b78 ^ (c >>> 1) : c >>> 1
+    }
+    table[n] = c >>> 0
+  }
+  return table
+})()
+
+/**
+ * Compute the CRC-32C checksum of a byte buffer.
+ *
+ * Known-answer vectors (RFC 3720 appendix / the standard test suite):
+ * - ASCII "123456789" → 0xE3069283
+ * - 32 zero bytes → 0x8A9136AA
+ *
+ * @param bytes - The payload to checksum.
+ * @returns The CRC-32C as an unsigned 32-bit integer.
+ */
+export function crc32c(bytes: Uint8Array): number {
+  let crc = 0xffffffff
+  for (let i = 0; i < bytes.length; i++) {
+    crc = TABLE[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8)
+  }
+  return (crc ^ 0xffffffff) >>> 0
+}
diff --git a/tests/integration/db-mvcc.test.ts b/tests/integration/db-mvcc.test.ts
index 764a755a..10a158ae 100644
--- a/tests/integration/db-mvcc.test.ts
+++ b/tests/integration/db-mvcc.test.ts
@@ -515,7 +515,15 @@ describe('8.0 Db API — generational MVCC', () => {
       await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 4 } }])
     ).release()
 
-    const recordsBefore = (await storage.listRawObjects('_generations')).length
+    // History record-sets only — the generation FACT LOG also lives under
+    // `_generations/` (at `facts/`) and is deliberately NOT reclaimed by
+    // history compaction (facts are the future canonical, not undo history).
+    const historyRecords = async (): Promise<number> =>
+      (await storage.listRawObjects('_generations')).filter(
+        (p: string) => !p.startsWith('_generations/facts/')
+      ).length
+
+    const recordsBefore = await historyRecords()
     expect(recordsBefore).toBeGreaterThan(0)
 
     // Compact while pinned: record-sets above the pin survive, pinned reads stay correct.
@@ -528,7 +536,7 @@ describe('8.0 Db API — generational MVCC', () => {
     const second = await brain.compactHistory()
     expect(first.removedGenerations + second.removedGenerations).toBeGreaterThan(0)
 
-    const recordsAfter = (await storage.listRawObjects('_generations')).length
+    const recordsAfter = await historyRecords()
     expect(recordsAfter).toBeLessThan(recordsBefore)
     expect(recordsAfter).toBe(0)
 
diff --git a/tests/integration/fact-log-dual-write.test.ts b/tests/integration/fact-log-dual-write.test.ts
new file mode 100644
index 00000000..9df1b908
--- /dev/null
+++ b/tests/integration/fact-log-dual-write.test.ts
@@ -0,0 +1,200 @@
+/**
+ * @module tests/integration/fact-log-dual-write
+ * @description The generation fact log end-to-end through real commits: every
+ * committed generation (single-op AND transact) appends its AFTER-IMAGE fact
+ * at the commit point; removals append body-less tombstones; an aborted
+ * transaction leaves no fact; facts survive reopen and continue monotonically;
+ * the scan surface (brain.scanFacts) carries the frozen telemetry shape; and
+ * the fact-log namespace is protected against prefix-nuking.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy, ProtectedArtifactError, type CommitFact } from '../../src/index.js'
+
+async function allFacts(brain: any): Promise<CommitFact[]> {
+  const scan = brain.scanFacts()
+  expect(scan).not.toBeNull()
+  const facts: CommitFact[] = []
+  for await (const batch of scan!.batches()) facts.push(...batch.facts)
+  return facts
+}
+
+describe('fact log dual-write (memory adapter)', () => {
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+  })
+
+  it('every single-op write appends its after-image fact; a remove appends a tombstone', async () => {
+    const id = await brain.add({ data: 'first', type: 'document', metadata: { rev: 1 } })
+    await brain.update({ id, metadata: { rev: 2 } })
+    await brain.remove(id)
+
+    const facts = await allFacts(brain)
+    // add + update + remove each committed a generation (the remove may span
+    // cascade ops but is ONE generation). Facts are monotonic.
+    const gens = facts.map((f) => f.generation)
+    expect([...gens].sort((a, b) => a - b)).toEqual(gens)
+    expect(facts.length).toBeGreaterThanOrEqual(3)
+
+    // The add fact carries the after-image of the new entity.
+    const addFact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null))
+    expect(addFact).toBeDefined()
+
+    // The remove fact carries a body-less tombstone for the id.
+    const removeFact = facts[facts.length - 1]
+    const tombstone = removeFact.ops.find((op) => op.id === id)
+    expect(tombstone).toBeDefined()
+    expect(tombstone!.record).toBeNull()
+    expect(tombstone!.kind).toBe('noun')
+  })
+
+  it('the update fact holds the NEW state (after-image, not before)', async () => {
+    const id = await brain.add({ data: 'versioned', type: 'document', metadata: { v: 'old' } })
+    await brain.update({ id, metadata: { v: 'new' } })
+
+    const facts = await allFacts(brain)
+    const updateFact = facts[facts.length - 1]
+    const op = updateFact.ops.find((o) => o.id === id)!
+    expect(op.record).not.toBeNull()
+    expect((op.record!.metadata as any).v).toBe('new')
+  })
+
+  it('a transact commits ONE fact carrying all its ops, with meta', async () => {
+    const receipt = await brain.transact(
+      [
+        { op: 'add', type: 'document', metadata: { part: 1 }, data: 'a' },
+        { op: 'add', type: 'document', metadata: { part: 2 }, data: 'b' }
+      ],
+      { meta: { source: 'batch-import' } }
+    )
+
+    const facts = await allFacts(brain)
+    const txFact = facts.find((f) => f.generation === receipt.generation)
+    expect(txFact).toBeDefined()
+    expect(txFact!.ops.filter((op) => op.kind === 'noun').length).toBeGreaterThanOrEqual(2)
+    expect(txFact!.meta).toEqual({ source: 'batch-import' })
+  })
+
+  it('an aborted transact leaves NO fact (absent = never committed)', async () => {
+    const id = await brain.add({ data: 'cas target', type: 'document', metadata: { n: 1 } })
+    const before = (await allFacts(brain)).length
+
+    await expect(
+      brain.transact([{ op: 'update', id, ifRev: 999, metadata: { n: 2 } }])
+    ).rejects.toThrow()
+
+    const after = await allFacts(brain)
+    expect(after.length).toBe(before)
+  })
+
+  it('fact generations line up with the transaction log', async () => {
+    await brain.add({ data: 'x', type: 'document', metadata: {} })
+    await brain.add({ data: 'y', type: 'document', metadata: {} })
+    await brain.flush()
+
+    const facts = await allFacts(brain)
+    const logGens = new Set((await brain.transactionLog()).map((e: any) => e.generation))
+    for (const f of facts) {
+      expect(logGens.has(f.generation)).toBe(true)
+    }
+  })
+
+  it('scan telemetry carries the frozen shape end-to-end', async () => {
+    for (let i = 0; i < 5; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
+
+    const scan = brain.scanFacts({ batchSize: 2 })!
+    expect(scan.headGeneration).toBeGreaterThanOrEqual(5)
+    expect(scan.approxFactCount).toBeGreaterThanOrEqual(5)
+    let batches = 0
+    for await (const b of scan.batches()) {
+      batches++
+      expect(b.factCount).toBe(b.facts.length)
+      expect(b.firstGeneration).toBe(b.facts[0].generation)
+      expect(b.lastGeneration).toBe(b.facts[b.facts.length - 1].generation)
+      expect(b.byteSize).toBeGreaterThan(0)
+      expect(typeof b.segmentId).toBe('string')
+    }
+    expect(batches).toBeGreaterThan(1)
+    expect(scan.summary().factsYielded).toBe(scan.approxFactCount)
+  })
+})
+
+describe('fact log dual-write (filesystem adapter — durability + protection)', () => {
+  let dir: string
+  let brain: any
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factlog-'))
+    brain = await open()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('facts survive close + reopen and appends continue monotonically', async () => {
+    const id = await brain.add({ data: 'persist me', type: 'document', metadata: { k: 1 } })
+    await brain.remove(id)
+    await brain.close()
+
+    brain = await open()
+    const facts = await allFacts(brain)
+    expect(facts.length).toBeGreaterThanOrEqual(2)
+    const headBefore = facts[facts.length - 1].generation
+
+    await brain.add({ data: 'after reopen', type: 'document', metadata: { k: 2 } })
+    const facts2 = await allFacts(brain)
+    expect(facts2[facts2.length - 1].generation).toBeGreaterThan(headBefore)
+  })
+
+  it('the fact segments exist on disk under _generations/facts/ with zero-padded names', async () => {
+    await brain.add({ data: 'on disk', type: 'document', metadata: {} })
+    await brain.flush()
+    const factsDir = path.join(dir, '_generations', 'facts')
+    const files = fs.readdirSync(factsDir)
+    // The manifest rides the store's JSON object discipline (gzip on disk).
+    expect(files.some((f) => f.startsWith('manifest.json'))).toBe(true)
+    const segs = files.filter((f) => /^seg-\d{20}\.bfl$/.test(f))
+    expect(segs.length).toBeGreaterThanOrEqual(1)
+  })
+
+  it('the fact-log namespace is PROTECTED: a prefix-nuke is refused', async () => {
+    await brain.add({ data: 'protected', type: 'document', metadata: {} })
+    await expect(brain.storage.removeRawPrefix('_generations/facts')).rejects.toBeInstanceOf(
+      ProtectedArtifactError
+    )
+    // Per-generation history cleanup remains unaffected (no false intersect).
+    await expect(brain.storage.removeRawPrefix('_generations/999999')).resolves.toBeUndefined()
+  })
+
+  it('transact facts are durable-on-return (no flush needed before reopen)', async () => {
+    const receipt = await brain.transact([
+      { op: 'add', type: 'document', metadata: { durable: true }, data: 'tx' }
+    ])
+    // Simulate an abrupt end: no flush(), no close() — reopen from disk.
+    brain = await open()
+    const facts = await allFacts(brain)
+    expect(facts.some((f) => f.generation === receipt.generation)).toBe(true)
+  })
+})
diff --git a/tests/unit/db/fact-log.test.ts b/tests/unit/db/fact-log.test.ts
new file mode 100644
index 00000000..abce2dc9
--- /dev/null
+++ b/tests/unit/db/fact-log.test.ts
@@ -0,0 +1,189 @@
+/**
+ * @module tests/unit/db/fact-log
+ * @description The generation fact log in isolation: wire-format round-trip
+ * (positional msgpack facts, bin16 uuids, body-less tombstones), crc32c
+ * framing with torn-tail detection, open-time truncation to committed truth
+ * (the log can only ever be AHEAD after a crash; open cuts it back), rotation
+ * with a manifest-first flip, exactly-once scans with the frozen telemetry
+ * shape, and the mmap segment handoff excluding the mutable tail.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+import {
+  FactLog,
+  FACTS_PREFIX,
+  type CommitFact,
+  type FactLogStorage,
+  storageSupportsFactLog
+} from '../../../src/db/factLog.js'
+import { crc32c } from '../../../src/utils/crc32c.js'
+
+const UUID = (n: number): string =>
+  `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
+
+const fact = (generation: number, overrides?: Partial<CommitFact>): CommitFact => ({
+  generation,
+  timestamp: 1_700_000_000_000 + generation,
+  ops: [
+    {
+      kind: 'noun',
+      id: UUID(generation),
+      record: { metadata: { noun: 'document', title: `doc ${generation}` }, vector: { v: [1, 2] } }
+    }
+  ],
+  ...overrides
+})
+
+describe('crc32c known-answer vectors', () => {
+  it('matches the RFC 3720 test vectors', () => {
+    expect(crc32c(new TextEncoder().encode('123456789'))).toBe(0xe3069283)
+    expect(crc32c(new Uint8Array(32))).toBe(0x8a9136aa)
+  })
+})
+
+describe('fact log — round-trip, framing, reconcile, rotation, scan', () => {
+  let storage: FactLogStorage
+  let log: FactLog
+
+  beforeEach(async () => {
+    const mem: any = new MemoryStorage()
+    await mem.init()
+    expect(storageSupportsFactLog(mem)).toBe(true)
+    storage = mem
+    log = new FactLog(storage)
+    await log.open(0)
+  })
+
+  it('facts round-trip byte-exactly: ops, tombstones, meta, blobHashes', async () => {
+    await log.append(fact(1))
+    await log.append(
+      fact(2, {
+        ops: [
+          { kind: 'verb', id: UUID(21), record: { metadata: { verb: 'contains' }, vector: null } },
+          { kind: 'noun', id: UUID(22), record: null } // TOMBSTONE
+        ],
+        meta: { source: 'test' },
+        blobHashes: ['abc123', 'abc123'] // multiset — duplicates preserved
+      })
+    )
+    await log.sync()
+
+    const scan = log.scanFacts()
+    expect(scan.headGeneration).toBe(2)
+    const all: CommitFact[] = []
+    for await (const batch of scan.batches()) all.push(...batch.facts)
+
+    expect(all).toHaveLength(2)
+    expect(all[0].generation).toBe(1)
+    expect(all[0].ops[0].id).toBe(UUID(1))
+    expect(all[0].ops[0].record?.metadata).toEqual({ noun: 'document', title: 'doc 1' })
+    expect(all[1].ops[0].kind).toBe('verb')
+    expect(all[1].ops[1].record).toBeNull() // the tombstone is body-less
+    expect(all[1].meta).toEqual({ source: 'test' })
+    expect(all[1].blobHashes).toEqual(['abc123', 'abc123'])
+    expect(scan.summary().factsYielded).toBe(2)
+  })
+
+  it('appends are monotonic — a replayed/duplicate generation throws', async () => {
+    await log.append(fact(5))
+    await expect(log.append(fact(5))).rejects.toThrow(/non-monotonic/)
+    await expect(log.append(fact(3))).rejects.toThrow(/non-monotonic/)
+    await expect(log.append(fact(6))).resolves.toBeUndefined() // gaps are fine (aborted reservations)
+  })
+
+  it('a torn tail (partial frame) is detected and ignored — intact prefix survives', async () => {
+    await log.append(fact(1))
+    await log.append(fact(2))
+    await log.sync()
+
+    // Simulate a crash mid-append: chop bytes off the tail file.
+    const tailPath = `${FACTS_PREFIX}/seg-${'1'.padStart(20, '0')}.bfl`
+    const bytes = (await storage.readRawBytes(tailPath))!
+    await storage.writeRawBytes(tailPath, bytes.subarray(0, bytes.length - 7))
+
+    const reopened = new FactLog(storage)
+    await reopened.open(2)
+    expect(reopened.headGeneration()).toBe(1) // fact 2's frame was torn → gone
+
+    const all: CommitFact[] = []
+    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([1])
+  })
+
+  it('open() truncates facts beyond committed truth (the crash-ahead shape)', async () => {
+    await log.append(fact(1))
+    await log.append(fact(2))
+    await log.append(fact(3))
+    await log.sync()
+
+    // The store's committed generation is 1 — facts 2..3 never committed.
+    const reopened = new FactLog(storage)
+    await reopened.open(1)
+    expect(reopened.headGeneration()).toBe(1)
+
+    const all: CommitFact[] = []
+    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([1])
+
+    // And appends continue cleanly from the truncated head.
+    await reopened.append(fact(2))
+    expect(reopened.headGeneration()).toBe(2)
+  })
+
+  it('a cleared store (committed=0) truncates everything', async () => {
+    await log.append(fact(1))
+    await log.append(fact(2))
+    await log.sync()
+    const reopened = new FactLog(storage)
+    await reopened.open(0)
+    expect(reopened.headGeneration()).toBe(0)
+  })
+
+  it('scan honors fromGeneration/toGeneration inclusively and filters kinds', async () => {
+    for (let g = 1; g <= 6; g++) await log.append(fact(g))
+    await log.sync()
+
+    const scan = log.scanFacts({ fromGeneration: 2, toGeneration: 4 })
+    const all: CommitFact[] = []
+    for await (const b of scan.batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([2, 3, 4])
+
+    const verbsOnly = log.scanFacts({ kinds: ['verb'] })
+    for await (const b of verbsOnly.batches()) {
+      for (const f of b.facts) expect(f.ops.every((op) => op.kind === 'verb')).toBe(true)
+    }
+  })
+
+  it('batch telemetry carries the frozen shape', async () => {
+    for (let g = 1; g <= 5; g++) await log.append(fact(g))
+    await log.sync()
+
+    const scan = log.scanFacts({ batchSize: 2 })
+    expect(scan.approxFactCount).toBe(5)
+    const batches = []
+    for await (const b of scan.batches()) batches.push(b)
+    expect(batches.length).toBe(3)
+    expect(batches[0]).toMatchObject({ firstGeneration: 1, lastGeneration: 2, factCount: 2 })
+    expect(batches[0].byteSize).toBeGreaterThan(0)
+    expect(typeof batches[0].segmentId).toBe('string')
+    expect(scan.summary()).toEqual({ factsYielded: 5, segmentsRead: 1 })
+  })
+
+  it('survives reopen: head and content come back from disk', async () => {
+    for (let g = 1; g <= 3; g++) await log.append(fact(g))
+    await log.sync()
+
+    const reopened = new FactLog(storage)
+    await reopened.open(3)
+    expect(reopened.headGeneration()).toBe(3)
+    const all: CommitFact[] = []
+    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([1, 2, 3])
+  })
+
+  it('segmentPaths excludes the mutable tail (mmap handoff = sealed only)', async () => {
+    await log.append(fact(1))
+    await log.sync()
+    expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed
+  })
+})
diff --git a/tests/unit/db/generationStore.test.ts b/tests/unit/db/generationStore.test.ts
index 8073d7ee..184d3974 100644
--- a/tests/unit/db/generationStore.test.ts
+++ b/tests/unit/db/generationStore.test.ts
@@ -249,7 +249,11 @@ describe('db/GenerationStore', () => {
       store.release(pinned)
       const result = await store.compact()
       expect(result.removedGenerations).toBeGreaterThan(0)
-      const remaining = await storage.listRawObjects(GENERATIONS_PREFIX)
+      // History record-sets only — the fact log (at `_generations/facts/`) is
+      // deliberately NOT reclaimed by history compaction.
+      const remaining = (await storage.listRawObjects(GENERATIONS_PREFIX)).filter(
+        (p: string) => !p.startsWith(`${GENERATIONS_PREFIX}/facts/`)
+      )
       expect(remaining).toEqual([])
     })
 

From 2888ae6b40124b8c1adbdf08770a2365594d5319 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 10:54:20 -0700
Subject: [PATCH 49/82] =?UTF-8?q?feat:=20entity-tree=20family=20stamp=20?=
 =?UTF-8?q?=E2=80=94=20sourceGeneration=20+=20rollup=20coherence=20at=20op?=
 =?UTF-8?q?en?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The canonical entity tree now carries a FAMILY STAMP
(_system/family-stamps/entity-tree.json, JSON — forensics stay
terminal-readable): which committed generation the tree reflects
(sourceGeneration) plus the rollup invariants (entity/relationship
counts) that verify a millions-of-files projection whole where per-file
checks cannot scale. Written at flush/close boundaries; open-time
coherence is a COMPARISON, not a walk:

- coherent / absent (legacy store) → silent
- behind → benign for the tree (it is written BY the commit; only the
  stamp is stale after a crash between commit and flush) — refreshes at
  the next flush
- incoherent (counts diverged at equal generation, or a stamp AHEAD of
  the log head) → loud, names the failing invariant; repairIndex()'s
  unconditional recount heals and RE-STAMPS so repair leaves a coherent
  stamp behind
- a fault reading the stamp is UNVERIFIABLE — never conflated with
  absence

One verifier (verifyFamilyStamp, exported) reads both member modes:
enumerated (exact byte size per member, bounded families) and rollup
(invariants, unbounded families). New exports: readFamilyStamp,
verifyFamilyStamp, ENTITY_TREE_STAMP_PATH, FamilyStamp, StampMembers,
StampVerdict.
---
 src/brainy.ts                               | 108 ++++++++++++++
 src/db/familyStamp.ts                       | 147 ++++++++++++++++++++
 src/index.ts                                |   5 +
 tests/integration/entity-tree-stamp.test.ts | 144 +++++++++++++++++++
 4 files changed, 404 insertions(+)
 create mode 100644 src/db/familyStamp.ts
 create mode 100644 tests/integration/entity-tree-stamp.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index f79ca562..b18834d8 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -168,6 +168,13 @@ import {
 } from './db/portableGraph.js'
 import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js'
 import type { FactScanHandle } from './db/factLog.js'
+import {
+  ENTITY_TREE_STAMP_PATH,
+  readFamilyStamp,
+  verifyFamilyStamp,
+  writeFamilyStamp,
+  type FamilyStamp
+} from './db/familyStamp.js'
 import {
   ChangeFeed,
   type BrainyChangeEvent,
@@ -992,6 +999,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         })
       }
 
+      // Entity-tree stamp coherence: compare the stamped sourceGeneration +
+      // rollup invariants against the log head + live counters. Loud on
+      // genuine incoherence (repairIndex heals), silent on absent/coherent,
+      // benign-behind refreshes at the next flush. Never blocks open.
+      await this.verifyEntityTreeStamp()
+
       // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
       // marker (`_system/brain-format.json`) into an in-memory field NOW —
       // after the store-open phase, but BEFORE any derived index or native
@@ -10354,11 +10367,94 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     //    Db pins and an explicit autoCompact: false.
     await this.autoCompactHistory()
 
+    // 7. Stamp the entity tree: which source generation the canonical tree
+    //    reflects + the rollup invariants that verify it whole (the counters
+    //    persisted in step 1). Written at flush boundaries — the tree tracks
+    //    every commit by construction, so the stamp is a durable checkpoint,
+    //    not a per-commit cost. Open compares stamp vs log head + rollups.
+    await this.stampEntityTree()
+
     const elapsed = Date.now() - startTime
 
     console.log(`All indexes flushed to disk in ${elapsed}ms`)
   }
 
+  /**
+   * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the
+   * committed generation the canonical tree reflects — equal by construction,
+   * since the tree is written by the commit itself) plus the rollup invariants
+   * (entity/relationship counts) that verify the tree whole where per-file
+   * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp};
+   * healed by `repairIndex()`, whose unconditional recount rebuilds the
+   * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write
+   * fault warns loudly but never fails the flush that carried real data.
+   */
+  private async stampEntityTree(): Promise<void> {
+    if (this.isReadOnly) return
+    try {
+      const [nounCount, verbCount] = await Promise.all([
+        this.storage.getNounCount(),
+        this.storage.getVerbCount()
+      ])
+      await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
+        family: 'entity-tree',
+        sourceGeneration: this.generationStore.generation(),
+        members: { mode: 'rollup', invariants: { nounCount, verbCount } }
+      })
+    } catch (error) {
+      prodLog.warn(
+        `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` +
+          `next successful flush): ${(error as Error).message}`
+      )
+    }
+  }
+
+  /**
+   * @description Open-time coherence check for the entity tree's family stamp:
+   * compare `sourceGeneration` against the log head and the stamped rollup
+   * invariants against the live counters. Verdicts:
+   * - `coherent` / `absent` (legacy store; first flush stamps) → silent.
+   * - `behind` → benign for the tree (it is written BY the commit; only the
+   *   stamp is stale — a crash landed between commit and flush). Refreshed at
+   *   the next flush.
+   * - `incoherent` → LOUD: the tree or its counters diverged from what was
+   *   stamped — `repairIndex()` recounts from canonical and re-stamps.
+   * Never blocks open; a fault reading the stamp is surfaced as unverifiable,
+   * never conflated with absence.
+   */
+  private async verifyEntityTreeStamp(): Promise<void> {
+    let stamp: FamilyStamp | null
+    try {
+      stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH)
+    } catch (error) {
+      prodLog.warn(
+        `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` +
+          `${(error as Error).message}`
+      )
+      return
+    }
+    const [nounCount, verbCount] = await Promise.all([
+      this.storage.getNounCount(),
+      this.storage.getVerbCount()
+    ])
+    const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), {
+      nounCount,
+      verbCount
+    })
+    if (verdict.state === 'incoherent') {
+      prodLog.warn(
+        `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` +
+          `The canonical tree or its counters diverged from the stamped state — run ` +
+          `brain.repairIndex() to recount from canonical and re-stamp.`
+      )
+    } else if (verdict.state === 'behind') {
+      prodLog.debug(
+        `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` +
+          `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.`
+      )
+    }
+  }
+
   /**
    * Ask the writer process serving this data directory to flush its in-memory
    * indexes to disk, so a read-only inspector can observe fresh state.
@@ -15373,6 +15469,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     await pruner.rebuildTypeCounts?.()
     await pruner.rebuildSubtypeCounts?.()
 
+    // The recount changed the rollup truth — re-stamp the entity tree so the
+    // stamp's invariants match the healed counters (repair leaves a coherent
+    // stamp, not a stale one that warns on the next open).
+    await this.stampEntityTree()
+
     // VFS containment reconciliation: heal "cosmetic ghost" edges left by
     // pre-fix renames (an entity Contains-linked from BOTH its old and new
     // directory — readdir listed it in two places) and duplicate edges from
@@ -15775,6 +15876,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       })()
     ])
 
+    // Stamp the entity tree at the close boundary (counters + counter now
+    // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT
+    // instead of benign-behind. Best-effort, never blocks the close.
+    if (this.generationStore && !this.isReadOnly) {
+      await this.stampEntityTree()
+    }
+
     // Phase 2: Close components to release resources (timers, file handles)
     // Data is already safe on disk from Phase 1
     await Promise.all([
diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts
new file mode 100644
index 00000000..728ad5bb
--- /dev/null
+++ b/src/db/familyStamp.ts
@@ -0,0 +1,147 @@
+/**
+ * @module db/familyStamp
+ * @description The generalized FAMILY STAMP — one JSON shape that declares,
+ * for any derived projection, WHICH source state it reflects and HOW to verify
+ * it is whole. The entity tree (canonical current-state files) carries the
+ * first brainy-side stamp; native index families carry the same shape. One
+ * verifier reads both member modes:
+ *
+ * - `enumerated` — bounded families: exact byte size per member file,
+ *   verified at open.
+ * - `rollup`     — unbounded families (the entity tree: millions of files):
+ *   the verified surface is a small set of rollup invariants (entity/
+ *   relationship counts) plus `sourceGeneration`.
+ *
+ * `sourceGeneration` is the generation of the source-of-truth log this
+ * projection reflects — open-time coherence becomes a COMPARISON (stamp vs
+ * log head), not a walk:
+ *
+ * - equal + invariants hold → coherent, serve.
+ * - behind → the projection missed the tail (crash between commit and stamp);
+ *   for the Stage-1 tree this is benign by construction (the tree is written
+ *   BY the commit), so the stamp refreshes; a DERIVED projection would replay
+ *   the gap instead.
+ * - invariants FAIL at equal generation → genuine incoherence: loud, and the
+ *   repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
+ *   canonical walk) heals it.
+ *
+ * Stamps are JSON on purpose — every incident gets debugged by reading a
+ * stamp in a terminal.
+ */
+
+/** Storage-root-relative directory holding family stamps. */
+export const FAMILY_STAMPS_PREFIX = '_system/family-stamps'
+
+/** The entity tree's stamp path. */
+export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json`
+
+/** One enumerated member: a file and its exact expected byte size. */
+export interface EnumeratedMember {
+  path: string
+  bytes: number
+}
+
+/** The stamp's verified surface, in one of the two member modes. */
+export type StampMembers =
+  | { mode: 'enumerated'; files: EnumeratedMember[] }
+  | { mode: 'rollup'; invariants: Record<string, number> }
+
+/** The generalized family stamp (one shape, one verifier, both engines). */
+export interface FamilyStamp {
+  /** Which projection this stamps (e.g. `'entity-tree'`). */
+  family: string
+  /** Monotonic per-family stamp generation — bumps on every committed stamp. */
+  generation: number
+  /** ISO timestamp of the stamp write. */
+  committedAt: string
+  /** The source-of-truth generation this projection reflects. */
+  sourceGeneration: number
+  /** The verified surface. */
+  members: StampMembers
+}
+
+/** The verdict of an open-time stamp verification. */
+export type StampVerdict =
+  | { state: 'coherent' }
+  | { state: 'absent' } // legacy store — first stamp writes at the next flush
+  | { state: 'behind'; stampSource: number; head: number }
+  | { state: 'incoherent'; failures: string[] }
+  | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
+
+/** The narrow storage surface stamps ride (JSON objects + fsync). */
+export interface StampStorage {
+  readRawObject(path: string): Promise<any | null>
+  writeRawObject(path: string, data: any): Promise<void>
+  syncRawObjects(paths: string[]): Promise<void>
+}
+
+/** Read a family's stamp; `null` when none was ever written. */
+export async function readFamilyStamp(
+  storage: StampStorage,
+  path: string
+): Promise<FamilyStamp | null> {
+  const stored = (await storage.readRawObject(path)) as FamilyStamp | null
+  if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null
+  return stored
+}
+
+/** Write a family's stamp durably (atomic object write + fsync). */
+export async function writeFamilyStamp(
+  storage: StampStorage,
+  path: string,
+  stamp: Omit<FamilyStamp, 'generation' | 'committedAt'> & { generation?: number }
+): Promise<void> {
+  const prior = await readFamilyStamp(storage, path)
+  const full: FamilyStamp = {
+    ...stamp,
+    generation: (prior?.generation ?? 0) + 1,
+    committedAt: new Date().toISOString()
+  }
+  await storage.writeRawObject(path, full)
+  await storage.syncRawObjects([path])
+}
+
+/**
+ * The ONE verifier, both member modes. `actual` supplies the observed rollup
+ * values (rollup mode) or file sizes (enumerated mode, keyed by path);
+ * `head` is the source-of-truth generation now.
+ */
+export function verifyFamilyStamp(
+  stamp: FamilyStamp | null,
+  head: number,
+  actual: Record<string, number>
+): StampVerdict {
+  if (stamp === null) return { state: 'absent' }
+  if (stamp.sourceGeneration > head) {
+    // A stamp AHEAD of the log claims state that never committed — the
+    // projection was stamped against truth that a crash rolled back.
+    return {
+      state: 'incoherent',
+      failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
+    }
+  }
+  if (stamp.sourceGeneration < head) {
+    return { state: 'behind', stampSource: stamp.sourceGeneration, head }
+  }
+  const failures: string[] = []
+  if (stamp.members.mode === 'rollup') {
+    for (const [name, expected] of Object.entries(stamp.members.invariants)) {
+      const observed = actual[name]
+      if (observed === undefined) {
+        failures.push(`rollup invariant '${name}' has no observed value`)
+      } else if (observed !== expected) {
+        failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`)
+      }
+    }
+  } else {
+    for (const member of stamp.members.files) {
+      const observed = actual[member.path]
+      if (observed === undefined) {
+        failures.push(`member '${member.path}' is missing`)
+      } else if (observed !== member.bytes) {
+        failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`)
+      }
+    }
+  }
+  return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' }
+}
diff --git a/src/index.ts b/src/index.ts
index b9135600..bdb5ff73 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -204,6 +204,11 @@ export type {
   FactScanBatch,
   FactScanHandle
 } from './db/factLog.js'
+// The generalized family stamp — which source generation a projection
+// reflects + the surface that verifies it whole; one verifier, both member
+// modes (enumerated byte-exact / rollup invariants).
+export { readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH } from './db/familyStamp.js'
+export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.js'
 // Optional provider capability for generation-aware native indexes
 export { isVersionedIndexProvider } from './plugin.js'
 export type { VersionedIndexProvider } from './plugin.js'
diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts
new file mode 100644
index 00000000..f76c5094
--- /dev/null
+++ b/tests/integration/entity-tree-stamp.test.ts
@@ -0,0 +1,144 @@
+/**
+ * @module tests/integration/entity-tree-stamp
+ * @description The entity tree's FAMILY STAMP: written at flush/close with
+ * `sourceGeneration` (the committed generation the canonical tree reflects)
+ * plus rollup invariants (entity/relationship counts); verified at open by
+ * comparison — coherent on a clean close, LOUD on genuine incoherence
+ * (tampered counters), healed by repairIndex()'s recount + re-stamp. One
+ * verifier reads both member modes.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import {
+  Brainy,
+  readFamilyStamp,
+  verifyFamilyStamp,
+  ENTITY_TREE_STAMP_PATH,
+  type FamilyStamp
+} from '../../src/index.js'
+import { prodLog } from '../../src/utils/logger.js'
+
+describe('entity-tree family stamp', () => {
+  let dir: string
+  let brain: any
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-'))
+    brain = await open()
+  })
+  afterEach(async () => {
+    vi.restoreAllMocks()
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => {
+    for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } })
+    await brain.flush()
+
+    const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
+    expect(stamp).not.toBeNull()
+    expect(stamp.family).toBe('entity-tree')
+    expect(stamp.members.mode).toBe('rollup')
+    const invariants = (stamp.members as any).invariants
+    expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
+    expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
+    expect(stamp.sourceGeneration).toBe(brain.generation())
+    expect(stamp.generation).toBeGreaterThanOrEqual(1)
+  })
+
+  it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => {
+    await brain.add({ data: 'clean', type: 'document', metadata: {} })
+    await brain.close()
+
+    const warn = vi.spyOn(prodLog, 'warn')
+    brain = await open()
+    const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp'))
+    expect(stampWarnings).toEqual([])
+  })
+
+  it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => {
+    for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
+    await brain.close()
+
+    // Simulate counter drift AFTER the stamp was written: inflate the
+    // persisted scalar the way the historical decrement-skip did.
+    brain = await open()
+    ;(brain.storage as any).totalNounCount += 52
+    await (brain.storage as any).persistCounts()
+    await brain.close()
+    // The close boundary re-stamps with the inflated counter — so tamper the
+    // STAMP instead for a deterministic mismatch: stamped counts differ from
+    // the (inflated) live ones at the NEXT open only if the stamp is older.
+    // Rewrite the stamp with the honest counts + current sourceGeneration.
+    const raw = JSON.parse(
+      require('node:zlib')
+        .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
+        .toString('utf-8')
+    ) as FamilyStamp
+    const honest = { ...raw }
+    ;(honest.members as any).invariants.nounCount -= 52
+    fs.writeFileSync(
+      path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`),
+      require('node:zlib').gzipSync(JSON.stringify(honest))
+    )
+
+    const warn = vi.spyOn(prodLog, 'warn')
+    brain = await open()
+    const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
+    expect(incoherent.length).toBeGreaterThanOrEqual(1)
+    expect(String(incoherent[0][0])).toMatch(/nounCount/)
+
+    // The heal: recount from canonical + re-stamp → next open is quiet.
+    await brain.repairIndex()
+    await brain.close()
+    const warn2 = vi.spyOn(prodLog, 'warn')
+    brain = await open()
+    const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
+    expect(stillIncoherent).toEqual([])
+  })
+
+  it('the one verifier handles both member modes', () => {
+    const rollup: FamilyStamp = {
+      family: 'x',
+      generation: 1,
+      committedAt: new Date().toISOString(),
+      sourceGeneration: 5,
+      members: { mode: 'rollup', invariants: { nounCount: 10 } }
+    }
+    expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' })
+    expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent')
+    expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({
+      state: 'behind',
+      stampSource: 5,
+      head: 9
+    })
+    expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head
+    expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
+
+    const enumerated: FamilyStamp = {
+      family: 'y',
+      generation: 1,
+      committedAt: new Date().toISOString(),
+      sourceGeneration: 2,
+      members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] }
+    }
+    expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' })
+    expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent')
+    expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member
+  })
+})

From 4a60b439832c3ce3d4a21568012b12f2e4bdf4e2 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 11:18:27 -0700
Subject: [PATCH 50/82] docs: RELEASES.md entry for 8.4.0 (generation fact log
 + family stamp)

---
 RELEASES.md | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index 69159560..83b04c22 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,41 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream)
+
+A minor release, fully backward-compatible (all additions; no behavior change for existing APIs).
+This is infrastructure: it changes nothing about how you query today, and lays the substrate that
+makes index heals and incremental catch-up sequential-read problems instead of directory walks.
+
+- **Every committed write now also appends a "fact" — an after-image commit record.** Alongside the
+  existing before-image history, each committed generation (single-op and `transact()` alike) appends
+  what each touched entity/relationship *became* — or a body-less tombstone for a removal — to an
+  append-only, checksummed segment log under `_generations/facts/`. Crash-safe by construction: a
+  torn tail is detected and ignored; on open the log is reconciled to committed truth, so an absent
+  generation always means "never committed." `transact()` facts are durable when `transact()`
+  returns; single-op facts ride the same group-commit flush as their history.
+
+- **New: `brain.scanFacts()`** — stream committed facts in commit order, in batches, with heal-grade
+  telemetry (total scope up front; per-batch generation range, byte size, and segment; a summary
+  cross-check at the end; loud abort on any gap — never a silent skip). **`brain.factSegmentPaths()`**
+  hands zero-copy consumers the immutable sealed segment files directly. New exported types:
+  `CommitFact`, `FactOp`, `FactScanBatch`, `FactScanHandle`. Facts accumulate from the first write
+  after upgrading — pre-existing history is not retroactively converted (enumeration remains the
+  fallback for old data).
+
+- **New: the entity-tree family stamp.** At every flush/close, brainy stamps which committed
+  generation the canonical entity files reflect plus the rollup invariants (entity/relationship
+  counts) that verify the tree whole. At open, coherence is a comparison — a genuine divergence is
+  loud and names the failing invariant; `repairIndex()` recounts from canonical and re-stamps. New
+  exports: `readFamilyStamp`, `verifyFamilyStamp`, `ENTITY_TREE_STAMP_PATH`, `FamilyStamp` types.
+
+- **Storage adapters** gain optional binary raw-byte primitives (`appendRawBytes`, `readRawBytes`,
+  `writeRawBytes`, `rawByteSize`) — feature-detected; the filesystem and memory adapters implement
+  them; an adapter without them simply hosts no fact log. The fact-log namespace is registered as a
+  protected family: no sweeper or GC can delete under it.
+
+No API breaks. 24 new tests; the full commit-path regression suite is green.
+
 ## v8.3.3 — 2026-07-15 (rename moves the containment edge — no ghost in the old directory)
 
 One production-reported fix plus a repair path, completing the delete/move hygiene arc (8.3.1 fixed

From 70886da548b826698f835e81f985e1b805e95577 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 11:24:48 -0700
Subject: [PATCH 51/82] chore(release): 8.4.0

---
 CHANGELOG.md      | 7 +++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fcfeadb9..5ebb255e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15)
+
+- docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43)
+- feat: entity-tree family stamp — sourceGeneration + rollup coherence at open (2888ae6)
+- feat: generation fact log — after-image commit records, dual-written at every commit point (38b0041)
+
+
 ### [8.3.3](https://github.com/soulcraftlabs/brainy/compare/v8.3.2...v8.3.3) (2026-07-15)
 
 - docs: RELEASES.md entry for 8.3.3 (rename containment fix + repair) (c3feafd)
diff --git a/package-lock.json b/package-lock.json
index c4e4f9ab..43347132 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.3",
+  "version": "8.4.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.3.3",
+      "version": "8.4.0",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index ee74980b..fbbc8b5b 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.3.3",
+  "version": "8.4.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 352e356fd54d440827f829e06b6dceadb1c644f6 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 12:36:27 -0700
Subject: [PATCH 52/82] feat: provider access to the fact log + shared stamp
 verifier via internals
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three additive surfaces for native index providers, which hold only
`storage` and must never construct their own fact-log reader (the log's
open path is writer-side — it reconciles by truncating/rewriting):

- Fact-scan capability on the storage adapter (the getBinaryBlobPath
  pattern): the host brain wires a closure over its LIVE fact log at
  init; providers call storage.scanFacts()/factLogHeadGeneration()/
  factSegmentPaths() and fall back to the enumeration walk on null.
  Restore/reopen swaps the underlying log transparently.
- The family-stamp trio (readFamilyStamp/writeFamilyStamp/
  verifyFamilyStamp + types) exported from @soulcraft/brainy/internals,
  so 'one verifier' is literally one function shared with the native
  side, never two synchronized copies.
- Rollup invariants widened to number|string: content fingerprints
  (e.g. a per-tree SHA-256) are valid invariant values; strict equality
  either way, and a type mismatch reads as incoherence, never a pass.
---
 src/brainy.ts                                 | 11 +++++
 src/coreTypes.ts                              | 30 ++++++++++++++
 src/db/familyStamp.ts                         | 11 +++--
 src/internals.ts                              | 26 ++++++++++++
 src/storage/baseStorage.ts                    | 40 +++++++++++++++++++
 tests/integration/entity-tree-stamp.test.ts   | 19 +++++++++
 tests/integration/fact-log-dual-write.test.ts | 19 +++++++++
 7 files changed, 153 insertions(+), 3 deletions(-)

diff --git a/src/brainy.ts b/src/brainy.ts
index b18834d8..a188ad87 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -999,6 +999,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         })
       }
 
+      // Fact-scan capability: wire the storage seam through which index
+      // providers (which hold only `storage`) reach the fact log. A closure
+      // over the LIVE log — restore/reopen swaps the instance transparently —
+      // so a provider's heal can switch from the enumeration walk to one
+      // sequential fact scan whenever the log exists.
+      if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') {
+        ;(this.storage as BaseStorage).setFactScanSource(
+          () => this.generationStore?.getFactLog() ?? null
+        )
+      }
+
       // Entity-tree stamp coherence: compare the stamped sourceGeneration +
       // rollup invariants against the log head + live counters. Loud on
       // genuine incoherence (repairIndex heals), silent on absent/coherent,
diff --git a/src/coreTypes.ts b/src/coreTypes.ts
index dc4b3860..26783e0f 100644
--- a/src/coreTypes.ts
+++ b/src/coreTypes.ts
@@ -1159,6 +1159,36 @@ export interface StorageAdapter {
    */
   rawByteSize?(path: string): Promise<number | null>
 
+  /**
+   * @description OPTIONAL fact-scan capability — how an index provider that
+   * holds only `storage` reaches the generation fact log (the host brain
+   * wires it at init; providers must never construct their own fact-log
+   * reader — the log's open path is writer-side). Present ⟺ this store hosts
+   * a fact log AND the host wired the capability. Returns a scan handle over
+   * committed facts (heal-grade telemetry included), or `null` when no fact
+   * log exists — callers fall back to the canonical enumeration walk.
+   */
+  scanFacts?(options?: {
+    fromGeneration?: number
+    toGeneration?: number
+    kinds?: Array<'noun' | 'verb'>
+    batchSize?: number
+  }): import('./db/factLog.js').FactScanHandle | null
+
+  /**
+   * @description OPTIONAL (rides the fact-scan capability): the fact log's
+   * head generation — the replay target for `stamp.sourceGeneration + 1`
+   * catch-ups. `null` when no fact log exists.
+   */
+  factLogHeadGeneration?(): number | null
+
+  /**
+   * @description OPTIONAL (rides the fact-scan capability): the immutable,
+   * sealed fact-segment file paths covering `fromGeneration` — the zero-copy
+   * handoff. The mutable tail is excluded (read it via `scanFacts`).
+   */
+  factSegmentPaths?(options?: { fromGeneration?: number }): string[]
+
   /**
    * Save statistics data
    * @param statistics The statistics data to save
diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts
index 728ad5bb..98342884 100644
--- a/src/db/familyStamp.ts
+++ b/src/db/familyStamp.ts
@@ -41,10 +41,15 @@ export interface EnumeratedMember {
   bytes: number
 }
 
-/** The stamp's verified surface, in one of the two member modes. */
+/**
+ * The stamp's verified surface, in one of the two member modes. Rollup
+ * invariant values may be numbers (counts, byte sizes) or strings (content
+ * fingerprints, e.g. a per-tree SHA-256) — the verifier compares by strict
+ * equality either way, so a type mismatch reads as incoherence, never a pass.
+ */
 export type StampMembers =
   | { mode: 'enumerated'; files: EnumeratedMember[] }
-  | { mode: 'rollup'; invariants: Record<string, number> }
+  | { mode: 'rollup'; invariants: Record<string, number | string> }
 
 /** The generalized family stamp (one shape, one verifier, both engines). */
 export interface FamilyStamp {
@@ -109,7 +114,7 @@ export async function writeFamilyStamp(
 export function verifyFamilyStamp(
   stamp: FamilyStamp | null,
   head: number,
-  actual: Record<string, number>
+  actual: Record<string, number | string>
 ): StampVerdict {
   if (stamp === null) return { state: 'absent' }
   if (stamp.sourceGeneration > head) {
diff --git a/src/internals.ts b/src/internals.ts
index 00a24638..009124fe 100644
--- a/src/internals.ts
+++ b/src/internals.ts
@@ -19,3 +19,29 @@ export type { MemoryInfo, CacheAllocationStrategy } from './utils/memoryDetectio
 // HNSWNounWithMetadata. First-party plugins (Cor) use this to stay in
 // lockstep with the entity shape contract.
 export { resolveEntityField, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
+
+// The generalized family stamp — ONE verifier, both member modes, shared
+// verbatim with native providers so stamp verification is literally one
+// function, never two synchronized copies. Providers write the same shape
+// (their set-swap rewrites stamps, so adoption is migration-free).
+export {
+  readFamilyStamp,
+  writeFamilyStamp,
+  verifyFamilyStamp,
+  FAMILY_STAMPS_PREFIX,
+  ENTITY_TREE_STAMP_PATH
+} from './db/familyStamp.js'
+export type {
+  FamilyStamp,
+  StampMembers,
+  StampVerdict,
+  EnumeratedMember,
+  StampStorage
+} from './db/familyStamp.js'
+
+// Generation fact-log types — the scan surface a provider reaches through the
+// storage capability (`storage.scanFacts` / `storage.factLogHeadGeneration` /
+// `storage.factSegmentPaths`, wired by the host brain at init). Providers
+// never construct a FactLog themselves: open() is writer-side (it reconciles
+// by truncating/rewriting) and there is exactly one writer.
+export type { CommitFact, FactOp, FactScanBatch, FactScanHandle } from './db/factLog.js'
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index db8ffaa6..1983b4bc 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -842,6 +842,46 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     return this.deleteObjectFromPath(path)
   }
 
+  // ==========================================================================
+  // Fact-scan capability — the seam through which an index provider holding
+  // only `storage` reaches the generation fact log. The HOST brain wires the
+  // source at init (a closure over its live fact log, so restore/reopen stays
+  // transparent); providers call storage.scanFacts?.(...) and fall back to the
+  // enumeration walk when it returns null. Providers never construct a
+  // fact-log reader themselves — the log's open path is writer-side.
+  // ==========================================================================
+
+  /** The host-wired fact-scan source (a closure over the live fact log). */
+  private _factScanSource: (() => import('../db/factLog.js').FactLog | null) | null = null
+
+  /** HOST-ONLY: wire (or clear) the fact-scan capability's source. */
+  public setFactScanSource(source: (() => import('../db/factLog.js').FactLog | null) | null): void {
+    this._factScanSource = source
+  }
+
+  /** Open a scan over committed facts, or `null` when no fact log exists. */
+  public scanFacts(options?: {
+    fromGeneration?: number
+    toGeneration?: number
+    kinds?: Array<'noun' | 'verb'>
+    batchSize?: number
+  }): import('../db/factLog.js').FactScanHandle | null {
+    const log = this._factScanSource?.()
+    return log ? log.scanFacts(options) : null
+  }
+
+  /** The fact log's head generation, or `null` when no fact log exists. */
+  public factLogHeadGeneration(): number | null {
+    const log = this._factScanSource?.()
+    return log ? log.headGeneration() : null
+  }
+
+  /** Sealed fact-segment paths (zero-copy handoff); empty when none. */
+  public factSegmentPaths(options?: { fromGeneration?: number }): string[] {
+    const log = this._factScanSource?.()
+    return log ? log.segmentPaths(options) : []
+  }
+
   /**
    * @description Remove the container that held a canonical entity's leg files,
    * called after both legs are deleted so a delete leaves NOTHING behind — no
diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts
index f76c5094..deefc5e6 100644
--- a/tests/integration/entity-tree-stamp.test.ts
+++ b/tests/integration/entity-tree-stamp.test.ts
@@ -140,5 +140,24 @@ describe('entity-tree family stamp', () => {
     expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' })
     expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent')
     expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member
+
+    // Rollup invariants may be STRING fingerprints (e.g. a per-tree SHA-256):
+    // strict equality either way; a type mismatch reads as incoherence.
+    const fingerprinted: FamilyStamp = {
+      family: 'z',
+      generation: 1,
+      committedAt: new Date().toISOString(),
+      sourceGeneration: 3,
+      members: { mode: 'rollup', invariants: { treeDigest: 'abc123', rows: 42 } }
+    }
+    expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: 42 })).toEqual({
+      state: 'coherent'
+    })
+    expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'deadbeef', rows: 42 }).state).toBe(
+      'incoherent'
+    )
+    expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: '42' }).state).toBe(
+      'incoherent' // type mismatch never passes
+    )
   })
 })
diff --git a/tests/integration/fact-log-dual-write.test.ts b/tests/integration/fact-log-dual-write.test.ts
index 9df1b908..5ec66273 100644
--- a/tests/integration/fact-log-dual-write.test.ts
+++ b/tests/integration/fact-log-dual-write.test.ts
@@ -108,6 +108,25 @@ describe('fact log dual-write (memory adapter)', () => {
     }
   })
 
+  it('the storage fact-scan capability serves a provider holding only `storage`', async () => {
+    // An index provider receives `storage` — never the brain — and reaches the
+    // fact log through the host-wired capability (it must never construct its
+    // own fact-log reader: the log's open path is writer-side).
+    const id = await brain.add({ data: 'via storage', type: 'document', metadata: { s: 1 } })
+    await brain.remove(id)
+
+    const storage = brain.storage
+    expect(typeof storage.scanFacts).toBe('function')
+    expect(storage.factLogHeadGeneration()).toBe(brain.scanFacts()!.headGeneration)
+
+    const viaStorage: CommitFact[] = []
+    for await (const b of storage.scanFacts()!.batches()) viaStorage.push(...b.facts)
+    const viaBrain: CommitFact[] = []
+    for await (const b of brain.scanFacts()!.batches()) viaBrain.push(...b.facts)
+    expect(viaStorage.map((f) => f.generation)).toEqual(viaBrain.map((f) => f.generation))
+    expect(storage.factSegmentPaths()).toEqual(brain.factSegmentPaths())
+  })
+
   it('scan telemetry carries the frozen shape end-to-end', async () => {
     for (let i = 0; i < 5; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
 

From e4f37cd32bdcc9b6eedd14b27df5316a2df60b7e Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 12:36:50 -0700
Subject: [PATCH 53/82] docs: RELEASES.md entry for 8.5.0 (provider fact-log
 access + shared verifier)

---
 RELEASES.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/RELEASES.md b/RELEASES.md
index 83b04c22..11f7500e 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,23 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier)
+
+Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass:
+
+- **Index providers can now reach the fact log through the storage adapter** — new optional
+  capability `storage.scanFacts()` / `storage.factLogHeadGeneration()` / `storage.factSegmentPaths()`,
+  wired by the host brain at init as a closure over its live log. Providers hold only `storage` and
+  must never construct their own fact-log reader (the log's open path is writer-side); `null` means
+  "no fact log here — use the enumeration walk."
+- **The family-stamp verifier is shared** via `@soulcraft/brainy/internals`
+  (`readFamilyStamp` / `writeFamilyStamp` / `verifyFamilyStamp` + types) so first-party native
+  providers run literally the same verification function, never a synchronized copy.
+- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a
+  per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass.
+
+No behavior change for applications; all additions.
+
 ## v8.4.0 — 2026-07-15 (the generation fact log — a sequential, self-verifying commit stream)
 
 A minor release, fully backward-compatible (all additions; no behavior change for existing APIs).

From d1ecee10f0bca9fd82433ea5c0e1138854313a21 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 12:44:52 -0700
Subject: [PATCH 54/82] feat: committedGeneration capability + pinned
 durability/stability contracts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- storage.committedGeneration(): the committed watermark exposed on the
  fact-scan capability, so a provider compares its stamp's
  sourceGeneration against the store's truth without parsing the
  private manifest format (the capability source now carries both the
  live fact log and the committed closure).
- Pinned contract tests: fsync-before-ack (transact passes today; the
  single-op path is pinned via it.fails as the documented target —
  group commit must become LATENCY batching, ack waiting on the shared
  fsync, never durability skipping; the pin flips red when that lands
  so there is no cliff to discover) and scan-stability-under-rotation
  (a scan snapshot yields exactly its facts — no gaps, duplicates, or
  bleed-in — while segments rotate beneath it; the reclaim-during-scan
  variant lands with fact compaction, which does not exist yet).
- FactLog accepts a rotateBytes option so tests exercise real rotation.
---
 RELEASES.md                                  |   8 ++
 src/brainy.ts                                |   9 +-
 src/coreTypes.ts                             |   9 ++
 src/db/factLog.ts                            |   7 +-
 src/storage/baseStorage.ts                   |  30 ++++-
 tests/integration/fact-log-contracts.test.ts | 125 +++++++++++++++++++
 6 files changed, 177 insertions(+), 11 deletions(-)
 create mode 100644 tests/integration/fact-log-contracts.test.ts

diff --git a/RELEASES.md b/RELEASES.md
index 11f7500e..457b65d0 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -24,6 +24,14 @@ Small additive follow-up to 8.4.0, from the native accelerator's first consumpti
   providers run literally the same verification function, never a synchronized copy.
 - **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a
   per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass.
+- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider
+  compares its stamp's `sourceGeneration` against the store's truth without parsing the private
+  manifest format.
+- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for
+  `transact()` today; the single-op path is pinned as the documented future target — group commit
+  becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan
+  snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate
+  beneath it).
 
 No behavior change for applications; all additions.
 
diff --git a/src/brainy.ts b/src/brainy.ts
index a188ad87..d5d4bc33 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -1005,9 +1005,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       // so a provider's heal can switch from the enumeration walk to one
       // sequential fact scan whenever the log exists.
       if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') {
-        ;(this.storage as BaseStorage).setFactScanSource(
-          () => this.generationStore?.getFactLog() ?? null
-        )
+        ;(this.storage as BaseStorage).setFactScanSource({
+          factLog: () => this.generationStore?.getFactLog() ?? null,
+          // The committed watermark, exposed as a capability so providers
+          // never parse the store's private manifest format.
+          committedGeneration: () => this.generationStore?.committedGeneration() ?? 0
+        })
       }
 
       // Entity-tree stamp coherence: compare the stamped sourceGeneration +
diff --git a/src/coreTypes.ts b/src/coreTypes.ts
index 26783e0f..e0248d17 100644
--- a/src/coreTypes.ts
+++ b/src/coreTypes.ts
@@ -1182,6 +1182,15 @@ export interface StorageAdapter {
    */
   factLogHeadGeneration?(): number | null
 
+  /**
+   * @description OPTIONAL (rides the fact-scan capability): the COMMITTED
+   * generation watermark — the manifest truth a projection's
+   * `sourceGeneration` compares against. Exposed as a capability so a
+   * provider never parses the store's private manifest format. `null` when
+   * the capability is unwired.
+   */
+  committedGeneration?(): number | null
+
   /**
    * @description OPTIONAL (rides the fact-scan capability): the immutable,
    * sealed fact-segment file paths covering `fromGeneration` — the zero-copy
diff --git a/src/db/factLog.ts b/src/db/factLog.ts
index b43e52d0..4c5e95fd 100644
--- a/src/db/factLog.ts
+++ b/src/db/factLog.ts
@@ -294,6 +294,8 @@ function parseSegment(
  */
 export class FactLog {
   private readonly storage: FactLogStorage
+  /** Rotation threshold (bytes); tests may lower it to exercise rotation. */
+  private readonly rotateBytes: number
   private manifest: FactsManifest = {
     formatVersion: FACTS_FORMAT_VERSION,
     segments: [],
@@ -309,8 +311,9 @@ export class FactLog {
   /** Segment paths appended since the last sync (the fsync batch). */
   private readonly dirtySegments = new Set<string>()
 
-  constructor(storage: FactLogStorage) {
+  constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) {
     this.storage = storage
+    this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES
   }
 
   /** The highest committed generation the log holds (0 = empty). */
@@ -405,7 +408,7 @@ export class FactLog {
     }
     if (this.manifest.tailSegment === null) {
       await this.startTail(fact.generation)
-    } else if (this.tailBytes >= SEGMENT_ROTATE_BYTES) {
+    } else if (this.tailBytes >= this.rotateBytes) {
       await this.rotate(fact.generation)
     }
     const frame = encodeFrame(fact)
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 1983b4bc..73b8e859 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -851,11 +851,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
   // fact-log reader themselves — the log's open path is writer-side.
   // ==========================================================================
 
-  /** The host-wired fact-scan source (a closure over the live fact log). */
-  private _factScanSource: (() => import('../db/factLog.js').FactLog | null) | null = null
+  /** The host-wired fact-scan source (closures over the live generation state). */
+  private _factScanSource: {
+    factLog: () => import('../db/factLog.js').FactLog | null
+    committedGeneration: () => number
+  } | null = null
 
   /** HOST-ONLY: wire (or clear) the fact-scan capability's source. */
-  public setFactScanSource(source: (() => import('../db/factLog.js').FactLog | null) | null): void {
+  public setFactScanSource(
+    source: {
+      factLog: () => import('../db/factLog.js').FactLog | null
+      committedGeneration: () => number
+    } | null
+  ): void {
     this._factScanSource = source
   }
 
@@ -866,19 +874,29 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     kinds?: Array<'noun' | 'verb'>
     batchSize?: number
   }): import('../db/factLog.js').FactScanHandle | null {
-    const log = this._factScanSource?.()
+    const log = this._factScanSource?.factLog()
     return log ? log.scanFacts(options) : null
   }
 
   /** The fact log's head generation, or `null` when no fact log exists. */
   public factLogHeadGeneration(): number | null {
-    const log = this._factScanSource?.()
+    const log = this._factScanSource?.factLog()
     return log ? log.headGeneration() : null
   }
 
+  /**
+   * The COMMITTED generation watermark (the manifest truth) — the value a
+   * projection's `sourceGeneration` compares against. Exposed here so a
+   * provider never parses the store's private manifest format. `null` when
+   * the capability is unwired (no host, or a bare adapter).
+   */
+  public committedGeneration(): number | null {
+    return this._factScanSource ? this._factScanSource.committedGeneration() : null
+  }
+
   /** Sealed fact-segment paths (zero-copy handoff); empty when none. */
   public factSegmentPaths(options?: { fromGeneration?: number }): string[] {
-    const log = this._factScanSource?.()
+    const log = this._factScanSource?.factLog()
     return log ? log.segmentPaths(options) : []
   }
 
diff --git a/tests/integration/fact-log-contracts.test.ts b/tests/integration/fact-log-contracts.test.ts
new file mode 100644
index 00000000..eb579da9
--- /dev/null
+++ b/tests/integration/fact-log-contracts.test.ts
@@ -0,0 +1,125 @@
+/**
+ * @module tests/integration/fact-log-contracts
+ * @description Pinned durability + stability contracts for the fact log.
+ *
+ * (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt
+ *     process end (no flush, no close — reopen from disk).
+ *     - transact(): HOLDS TODAY — the fact is fsync'd before transact returns.
+ *     - single-op: PINNED AS `it.fails` — today's group-commit batches
+ *       DURABILITY (ack precedes the group fsync; a hard kill loses the fact
+ *       AND the generation together, coherently — the documented Model-B
+ *       contract, fine while the tree is authoritative). The destination
+ *       (ack-at-log) requires group commit to become LATENCY batching: the
+ *       ack waits for the shared fsync. When that lands, this pin flips red —
+ *       remove `.fails` and the contract is permanent. No cliff to discover.
+ *
+ * (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment
+ *     rotation yields exactly its snapshot — byte-identical facts, no gaps,
+ *     no duplicates, and no bleed-in of facts appended after the snapshot.
+ *     (The reclaim-during-scan variant lands with fact-log compaction, which
+ *     does not exist yet — segments only rotate today, never reclaim.)
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy, type CommitFact } from '../../src/index.js'
+import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
+import { FactLog, type FactLogStorage } from '../../src/db/factLog.js'
+
+describe('fsync-before-ack contract (fact durability at the ack boundary)', () => {
+  let dir: string
+  let brain: any
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factack-'))
+    brain = await open()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('transact(): the fact is durable the moment the ack returns (kill-after-ack safe)', async () => {
+    const receipt = await brain.transact([
+      { op: 'add', type: 'document', metadata: { durable: 1 }, data: 'ack-at-commit' }
+    ])
+    // Abrupt end: no flush(), no close() — a new instance reads only disk.
+    brain = await open()
+    const facts: CommitFact[] = []
+    for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts)
+    expect(facts.some((f) => f.generation === receipt.generation)).toBe(true)
+  })
+
+  // PINNED (flips red when group commit becomes latency batching — then
+  // remove `.fails` and the ack-at-log contract is permanent on every path).
+  it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => {
+    await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } })
+    const ackedHead = brain.scanFacts()!.headGeneration
+    // Abrupt end immediately after the ack — before any flush window.
+    brain = await open()
+    const facts: CommitFact[] = []
+    for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts)
+    expect(facts.some((f) => f.generation === ackedHead)).toBe(true)
+  })
+})
+
+describe('scan stability under rotation (the snapshot contract)', () => {
+  const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
+  const fact = (generation: number): CommitFact => ({
+    generation,
+    timestamp: 1_700_000_000_000 + generation,
+    ops: [
+      {
+        kind: 'noun',
+        id: UUID(generation),
+        // Padding makes each frame ~1KB so a small rotateBytes forces rotations.
+        record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, vector: null }
+      }
+    ]
+  })
+
+  it('a scan opened before rotations yields its exact snapshot — no gaps, dups, or bleed-in', async () => {
+    const mem: any = new MemoryStorage()
+    await mem.init()
+    const log = new FactLog(mem as FactLogStorage, { rotateBytes: 4096 }) // ~4 facts per segment
+    await log.open(0)
+    for (let g = 1; g <= 10; g++) await log.append(fact(g))
+    await log.sync()
+
+    // Open the snapshot, THEN keep appending — forcing further rotations.
+    const scan = log.scanFacts()
+    expect(scan.headGeneration).toBe(10)
+    for (let g = 11; g <= 25; g++) await log.append(fact(g))
+    await log.sync()
+    expect(log.headGeneration()).toBe(25)
+
+    const seen: number[] = []
+    for await (const batch of scan.batches()) {
+      for (const f of batch.facts) seen.push(f.generation)
+    }
+    // Exactly the snapshot: 1..10 in order, nothing appended-after bleeds in.
+    expect(seen).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+    expect(scan.summary().factsYielded).toBe(10)
+
+    // And a fresh scan sees everything, across all rotated segments.
+    const all: number[] = []
+    for await (const batch of log.scanFacts().batches()) {
+      for (const f of batch.facts) all.push(f.generation)
+    }
+    expect(all).toEqual(Array.from({ length: 25 }, (_, i) => i + 1))
+    expect(log.segmentPaths().length).toBeGreaterThanOrEqual(2) // rotations actually happened
+  })
+})

From 4dc0a928fe011546d909a1ff17616ddeb04e069a Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 12:46:33 -0700
Subject: [PATCH 55/82] test: tolerant timing assertion in the execution-time
 measure test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

setTimeout can fire a few ms early under load (timer coalescing) — a
10ms sleep measured 9ms and failed the >=10 assertion, tripping a
release gate. Sleep 25ms, assert >=20: the test verifies time is
MEASURED, not the OS timer's precision.
---
 tests/transaction/TransactionManager.unit.test.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/transaction/TransactionManager.unit.test.ts b/tests/transaction/TransactionManager.unit.test.ts
index 5ffee866..cc477483 100644
--- a/tests/transaction/TransactionManager.unit.test.ts
+++ b/tests/transaction/TransactionManager.unit.test.ts
@@ -105,14 +105,17 @@ describe('TransactionManager', () => {
       const result = await manager.executeTransactionWithResult(async (tx) => {
         tx.addOperation({
           execute: async () => {
-            await new Promise(resolve => setTimeout(resolve, 10))
+            await new Promise(resolve => setTimeout(resolve, 25))
             return async () => {}
           }
         })
         return 'done'
       })
 
-      expect(result.executionTimeMs).toBeGreaterThanOrEqual(10)
+      // Timer coalescing can fire a setTimeout up to a few ms EARLY under
+      // load, so assert well below the sleep — this tests that time is
+      // MEASURED, not the OS timer's precision.
+      expect(result.executionTimeMs).toBeGreaterThanOrEqual(20)
     })
   })
 

From d60619c83be68a47f74df75f13482020f97963d6 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Wed, 15 Jul 2026 12:49:09 -0700
Subject: [PATCH 56/82] chore(release): 8.5.0

---
 CHANGELOG.md      | 8 ++++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5ebb255e..a7b0044b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15)
+
+- test: tolerant timing assertion in the execution-time measure test (4dc0a92)
+- feat: committedGeneration capability + pinned durability/stability contracts (d1ecee1)
+- docs: RELEASES.md entry for 8.5.0 (provider fact-log access + shared verifier) (e4f37cd)
+- feat: provider access to the fact log + shared stamp verifier via internals (352e356)
+
+
 ### [8.4.0](https://github.com/soulcraftlabs/brainy/compare/v8.3.3...v8.4.0) (2026-07-15)
 
 - docs: RELEASES.md entry for 8.4.0 (generation fact log + family stamp) (4a60b43)
diff --git a/package-lock.json b/package-lock.json
index 43347132..183974d8 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.4.0",
+  "version": "8.5.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.4.0",
+      "version": "8.5.0",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index fbbc8b5b..92b1ba33 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.4.0",
+  "version": "8.5.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 593bb8b0f9f4c3738d442c3842b70359526b815b Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Thu, 16 Jul 2026 10:30:32 -0700
Subject: [PATCH 57/82] docs: external-backups/sparse-storage guide +
 generation fact log concept
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- New public guide (guides/external-backups): the sparse-mmap reality
  for external backup tooling — why a brain directory can show 100+ GB
  apparent size on a small disk, which files are sparse, tar czSf /
  rsync --sparse / cp --sparse=always, live-store caveats, and what
  persist()/restore() already handle natively.
- New public concept (concepts/generation-fact-log): what facts are
  (after-image commit records, body-less tombstones), the crash-safety
  model (checksummed frames, reconcile-to-committed, absent = never
  committed), the scanFacts()/factSegmentPaths() surfaces with the
  telemetry shape, family stamps + open-time coherence, and the
  storage capability seam for plugin authors.
- Cross-link from the snapshots guide; sparse notes added to the
  public persist()/restore() JSDoc (the operator-facing sites).
---
 docs/concepts/generation-fact-log.md          | 117 ++++++++++++++++++
 .../external-backups-and-sparse-storage.md    |  99 +++++++++++++++
 docs/guides/snapshots-and-time-travel.md      |   5 +
 src/brainy.ts                                 |   8 ++
 src/db/db.ts                                  |   7 ++
 5 files changed, 236 insertions(+)
 create mode 100644 docs/concepts/generation-fact-log.md
 create mode 100644 docs/guides/external-backups-and-sparse-storage.md

diff --git a/docs/concepts/generation-fact-log.md b/docs/concepts/generation-fact-log.md
new file mode 100644
index 00000000..f9b3e974
--- /dev/null
+++ b/docs/concepts/generation-fact-log.md
@@ -0,0 +1,117 @@
+---
+title: The Generation Fact Log
+slug: concepts/generation-fact-log
+public: true
+category: concepts
+template: concept
+order: 6
+description: Every committed write also appends a self-verifying "fact" — an after-image commit record — to an append-only log. What facts are, the crash-safety model, the scanFacts() streaming surface, family stamps, and how index providers consume the log for sequential heals.
+next:
+  - concepts/consistency-model
+  - guides/snapshots-and-time-travel
+---
+
+# The Generation Fact Log
+
+Since 8.4.0, every committed generation also appends a **fact** — a compact record of what each
+touched entity or relationship *became* — to an append-only, checksummed log under
+`_generations/facts/`. Where the generational history answers *"what did things look like
+before?"* (before-images, powering `asOf()` and rollback), the fact log answers *"what happened,
+in order?"* — one sequential, self-verifying stream of the store's present being written.
+
+Nothing about querying changes. The fact log exists for three consumers:
+
+1. **Index heals and rebuilds** — one sequential read in commit order replaces a per-entity
+   directory walk over millions of files.
+2. **Incremental catch-up** — a derived index that knows which generation it reflects reads *just
+   the gap*, instead of rebuilding from scratch.
+3. **Replay and audit tooling** — anything that wants the store's committed timeline as a stream.
+
+## What a fact is
+
+One fact per committed generation:
+
+- **`generation`** and **`timestamp`** — which commit, when.
+- **`ops`** — every write in that commit: `{ kind: 'noun' | 'verb', id, record }` where `record`
+  holds the entity's full after-image (both stored legs), or **`null` for a tombstone** — a
+  removal carries no body, by design.
+- **`meta`** — the transaction metadata `transact()` was submitted with, when present.
+- **`blobHashes`** — content-blob references, for exact reclamation accounting.
+
+Facts accumulate **from the first write after upgrading** — pre-existing history is not
+retroactively converted, and consumers fall back to the enumeration walk when no log exists.
+
+## Crash safety, in one paragraph
+
+Facts are appended and fsynced **inside the same durability window as the commit itself**, before
+the commit point — so after a crash, the log can only ever be *ahead* of committed truth, never
+behind it with a hole. On open, the store reconciles the log back to the committed watermark:
+torn tails are detected by per-record checksums and cut; whole records beyond the watermark are
+truncated. The invariant every reader can rely on: **an absent generation was never committed; a
+present fact was.** `transact()` facts are durable the moment `transact()` returns; single-op
+facts share the same group-commit flush as the rest of their generation, so a hard kill loses the
+fact and the generation *together* — never a torn state.
+
+## Reading the log
+
+```typescript
+const scan = brain.scanFacts({ fromGeneration: 1 })
+if (scan) {
+  // Telemetry up front — progress bars get a denominator from second zero.
+  console.log(scan.headGeneration, scan.segmentCount, scan.approxFactCount)
+
+  for await (const batch of scan.batches()) {
+    // Each batch: { facts, firstGeneration, lastGeneration, factCount, byteSize, segmentId }
+    for (const fact of batch.facts) {
+      for (const op of fact.ops) {
+        if (op.record === null) {
+          // a tombstone: op.id was removed in this generation
+        }
+      }
+    }
+  }
+
+  console.log(scan.summary()) // { factsYielded, segmentsRead } — the cross-check
+}
+```
+
+- `scanFacts()` returns `null` when the store hosts no fact log (older store, or a storage adapter
+  without binary append support) — fall back to enumerating entities.
+- Scans run against a **snapshot**: facts appended after the scan opens never bleed in, each fact
+  is yielded exactly once, and a detected gap aborts loudly — never a silent skip.
+- `brain.factSegmentPaths()` returns the immutable, *sealed* segment files for zero-copy consumers
+  (the append-mutable tail is excluded — read it through `scanFacts()`).
+
+## Family stamps: how a projection proves it's current
+
+Anything derived from the store — an index, the entity file tree itself — carries a **family
+stamp**: a small JSON record of *which committed generation the projection reflects*
+(`sourceGeneration`) plus the invariants that verify it whole (exact per-file byte sizes for
+bounded families; rollup invariants like entity counts for unbounded ones). At open, coherence is
+a **comparison**, not a walk:
+
+- stamp equals the committed watermark and invariants hold → serve;
+- stamp is behind → the projection reads just the gap from the fact log;
+- invariants fail → loud, named divergence — `brain.repairIndex()` rebuilds from canonical and
+  re-stamps.
+
+The verifier is exported (`verifyFamilyStamp`) so every projection — TypeScript or native — runs
+literally the same check.
+
+## For plugin authors: the storage capability
+
+Index providers receive the storage adapter, not the brain — so the host wires the log onto it.
+Feature-detect and prefer the stream; fall back to enumeration:
+
+```typescript
+const scan = storage.scanFacts?.({ fromGeneration: stamp.sourceGeneration + 1 })
+if (scan) {
+  // sequential catch-up from the log
+} else {
+  // enumeration walk (older store or adapter)
+}
+const committed = storage.committedGeneration?.() // the watermark stamps compare against
+```
+
+Providers must never construct their own reader over the log's files — the open path belongs to
+the single writer (it reconciles the log at open); the capability is the sanctioned seam.
diff --git a/docs/guides/external-backups-and-sparse-storage.md b/docs/guides/external-backups-and-sparse-storage.md
new file mode 100644
index 00000000..f28dcfd7
--- /dev/null
+++ b/docs/guides/external-backups-and-sparse-storage.md
@@ -0,0 +1,99 @@
+---
+title: External Backups & Sparse Storage
+slug: guides/external-backups
+public: true
+category: guides
+template: guide
+order: 10
+description: How to back up a brain directory with external tools (tar, rsync, cp) without exploding sparse files — why a store can show 100+ GB "apparent" size on a small disk, which files are sparse, and how persist()/restore() handle it for you.
+next:
+  - guides/snapshots-and-time-travel
+  - concepts/storage-adapters
+---
+
+# External Backups & Sparse Storage
+
+The built-in snapshot path — [`db.persist()` and `brain.restore()`](/docs/guides/snapshots-and-time-travel) —
+already handles everything on this page for you. Read this when you back up a brain directory with
+**external tools**: `tar`, `rsync`, `cp`, `scp`, or a filesystem-level backup agent.
+
+## The one-sentence rule
+
+> **Always use the sparse-aware flag**: `tar czSf` (capital `S`), `rsync --sparse`,
+> `cp --sparse=always`. A naive copy can turn a 2 GB store into a 100+ GB one — or fail
+> the disk entirely.
+
+## Why: some files are sparse
+
+When a native accelerator plugin is active, parts of the index live in **memory-mapped files**
+created at a large fixed virtual size — the file's *apparent* size — while the filesystem only
+allocates blocks that were actually written. A brand-new id-mapper file can report tens of
+gigabytes in `ls -l` while occupying a few megabytes on disk.
+
+Check the difference yourself:
+
+```bash
+ls -lh  brain-data/_id_mapper/    # APPARENT size (can be huge)
+du -sh  brain-data/               # ALLOCATED size (the real footprint)
+```
+
+The sparse candidates in a brain directory:
+
+| Path | What it is |
+|---|---|
+| `_id_mapper/` | The native id-mapper's mmap files (large fixed virtual size) |
+| `_blobs/` | Native index files (vector base, segments) — may be mmap-backed |
+
+Everything else (entities, `_system`, `_generations`, `_cas` content blobs) is ordinary dense data.
+
+## Doing it right
+
+**tar** — the `S` flag detects holes and stores only real data:
+
+```bash
+tar czSf brain-backup.tgz /data/brain
+# restore preserves the holes:
+tar xzSf brain-backup.tgz -C /data/
+```
+
+**rsync**:
+
+```bash
+rsync -a --sparse /data/brain/ backup-host:/backups/brain/
+```
+
+**cp**:
+
+```bash
+cp -a --sparse=always /data/brain /backups/brain
+```
+
+**What goes wrong without the flag:** the copy *materializes* every hole as real zero bytes.
+A store whose apparent size exceeds the target disk fails with `ENOSPC` partway through — and a
+copy that *does* fit silently costs the full apparent size in storage and transfer time.
+
+## What the built-in paths do (so you don't have to)
+
+- **`db.persist(path)`** snapshots via **hard links** — instant and space-shared, since every data
+  file is immutable-by-rename. The handful of append-in-place files (the transaction log, the
+  commit fact log's tail segment) and mmap-mutated directories (`_id_mapper/`) are **byte-copied**
+  instead, so a post-snapshot write can never reach through a shared inode into your backup.
+- **`brain.restore(path, { confirm: true })`** is **non-destructive and sparse-aware**: the snapshot
+  is copied into a staging area *before* any live data is touched (all-zero blocks stay holes), and
+  only after the copy fully succeeds does an atomic swap move it into place. A failed copy —
+  including `ENOSPC` — leaves the live store exactly as it was. A crash mid-swap completes forward
+  on the next open.
+
+## Live-store caveats for external tools
+
+1. **Prefer snapshotting a `persist()` output, not the live directory.** `persist()` produces a
+   crash-consistent, immutable snapshot; running `tar` against a live, actively-written directory
+   can capture a torn mid-write state. If you must archive live, stop writes first (or accept that
+   the archive is only as consistent as the moment's flush state).
+2. **Never prune or "clean up" files inside a brain directory.** Index files that look stale or
+   redundant are load-bearing; the store protects its declared index families from in-process
+   deletion, but an external `rm` bypasses that fence. If space is the concern, `du -sh` first —
+   the allocated size is usually far smaller than it looks.
+3. **Verify restores by opening them.** `Brainy.load(path)` opens any snapshot or restored
+   directory read-only — the store verifies its own coherence at open and reports loudly if
+   anything is missing or torn.
diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md
index 6f7b150a..56c49044 100644
--- a/docs/guides/snapshots-and-time-travel.md
+++ b/docs/guides/snapshots-and-time-travel.md
@@ -9,6 +9,7 @@ description: Recipes for the Db API — instant backups with persist(), restore,
 next:
   - concepts/consistency-model
   - guides/optimistic-concurrency
+  - guides/external-backups
 ---
 
 # Snapshots & Time Travel
@@ -41,6 +42,10 @@ bytes. Cross-device targets fall back to per-file byte copies, and
 persisting an in-memory brain serializes it to the same directory layout —
 a real, durable store.
 
+> Archiving a brain directory with **external tools** (`tar`, `rsync`, `cp`)?
+> Some index files are sparse and can explode to their apparent size under a
+> naive copy — see [External Backups & Sparse Storage](/docs/guides/external-backups).
+
 Two things to know:
 
 - `persist()` requires the view to still be the store's **latest**
diff --git a/src/brainy.ts b/src/brainy.ts
index d5d4bc33..383b1195 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -8280,6 +8280,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * replacement. Release them first; a warning is logged when live pins
    * exist.
    *
+   * NON-DESTRUCTIVE STAGING + SPARSE-AWARE: the snapshot is copied into a
+   * staging area BEFORE any live data is touched (all-zero blocks stay
+   * holes, so a sparse mmap store restores at its true allocated size — not
+   * its apparent size). Any copy failure, ENOSPC included, removes only the
+   * staging and throws; the live store is untouched. Only after the copy
+   * succeeds does an atomic per-entry swap move it into place; a crash
+   * mid-swap completes FORWARD on the next open.
+   *
    * @param path - Snapshot directory to restore from.
    * @param options - Must be `{ confirm: true }` — an explicit acknowledgment
    *   that current state is destroyed.
diff --git a/src/db/db.ts b/src/db/db.ts
index 62c3de0e..ac927fc5 100644
--- a/src/db/db.ts
+++ b/src/db/db.ts
@@ -945,6 +945,13 @@ export class Db<T = any> {
    * {@link SpeculativeOverlayError} (commit them with `brain.transact()`
    * first).
    *
+   * SPARSE FILES: a native accelerator's mmap index files can be sparse —
+   * huge apparent size, small allocated size. `persist()` handles them
+   * correctly (hard links share the allocation). But if you then archive the
+   * snapshot with EXTERNAL tools, use the sparse-aware flags (`tar czSf`,
+   * `rsync --sparse`, `cp --sparse=always`) or the copy materializes every
+   * hole — see docs/guides/external-backups-and-sparse-storage.md.
+   *
    * @param path - Absolute directory for the snapshot (created; must be
    *   empty or absent).
    * @throws GenerationConflictError when this view is no longer the latest

From da55be7520eda31c3c84ea307836874062a1296d Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 09:20:09 -0700
Subject: [PATCH 58/82] fix: aggregation state adoption on reopen +
 single-flight backfill + query-cap ratchet removal

- AggregationIndex: defineAggregate before init no longer forces a backfill.
  init reconciles instead of clobbering: the app definition wins, persisted
  state is adopted on hash match, a write landing pre-adoption forces an exact
  rescan, and a successful state load clears the backfill flag. New ready()
  settles every adoption decision before query paths consult backfill state.
- brainy: backfills are single-flight and batched. Concurrent queries share one
  store walk and every pending aggregate fills from that same walk; the old
  behavior let each concurrent query wipe the others' partial state and start
  its own full walk, so a store under steady aggregate traffic never converged.
  queryAggregate also waits for persisted definitions before deciding an
  aggregate does not exist.
- paramValidation: recordQuery is telemetry-only. The duration-based cap
  ratchet (x0.8 per recorded query while lifetime-average exceeded 1s, floored
  at 1000 - below the documented 10000 auto floor, reported under a stale
  basis label) is removed; the cap comes from its construction-time basis or
  explicit overrides alone.
- storage: __aggregation_* and singleton system keys (brainy:entityIdMapper)
  are recognized before the unknown-key warning fires; routing is unchanged.
- docs: find-limits cap-immutability note, aggregation reopen/adoption
  semantics, RELEASES.md 8.5.1 entry.
---
 RELEASES.md                                   |  33 +++
 docs/guides/aggregation.md                    |   8 +-
 docs/guides/find-limits.md                    |   4 +
 src/aggregation/AggregationIndex.ts           | 126 +++++++++-
 src/brainy.ts                                 |  49 +++-
 src/storage/baseStorage.ts                    |   9 +-
 src/utils/paramValidation.ts                  |  33 +--
 .../aggregation-state-persistence.test.ts     | 235 ++++++++++++++++++
 .../unit/aggregation/AggregationIndex.test.ts | 100 ++++++++
 tests/unit/utils/paramValidation.test.ts      |  31 ++-
 10 files changed, 584 insertions(+), 44 deletions(-)
 create mode 100644 tests/integration/aggregation-state-persistence.test.ts

diff --git a/RELEASES.md b/RELEASES.md
index 457b65d0..920f0d79 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,39 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed)
+
+Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box
+while vector search stayed fast, and every `find({ limit: 5000 })` suddenly failing against an
+"auto-configured query limit of 1000"). Three fixes, one cosmetic:
+
+- **Aggregation state is actually adopted on reopen.** The boot pattern `defineAggregate()` →
+  query raced the engine's async state load: the synchronous define always won, flagged a
+  backfill, and the first query then wiped the just-loaded persisted state and re-walked the
+  entire store — every restart, forever. Reopening with an unchanged definition now adopts the
+  persisted state directly (zero scans); a backfill runs only on a real definition change, a
+  missing/failed state load, or a write that landed before adoption (exactness wins). Apps that
+  rely on persisted definitions without re-defining at boot also no longer race a spurious
+  "Aggregate not defined".
+- **Backfills are single-flight and batched.** Concurrent queries on a cold aggregate used to
+  each wipe the others' partial state and start their own full store walk — under steady query
+  arrival the store never converged (the 40–90 s loop). Now all concurrent queries share one
+  walk, and one walk fills every aggregate pending backfill (M aggregates ≠ M scans).
+- **The query-cap "learning" ratchet is removed.** `maxLimit` is a memory-protection bound, but
+  a hidden tuner shrank it 20 % per recorded query while the lifetime-average query time
+  exceeded 1 s — down to a floor of 1000, below the documented 10 000 auto floor, with no
+  practical recovery, and the resulting error blamed "available free memory" (stale basis
+  label). The cap now comes from its construction-time basis (or your explicit
+  `maxQueryLimit` / `reservedQueryMemory`) alone and never changes at runtime; query timing is
+  recorded for diagnostics only.
+- **Cosmetic:** the engine's own persistence keys (`__aggregation_*`, `brainy:entityIdMapper`)
+  no longer log `[Storage] Unknown key format` at boot — they were always routed correctly;
+  they're now recognized before the warning fires.
+
+Operationally: if a host was bitten, upgrading and restarting is the whole fix — no repair
+ritual needed. Setting `maxQueryLimit` explicitly remains the valve that bypasses auto-detection
+entirely.
+
 ## v8.5.0 — 2026-07-15 (provider access to the fact log + the shared stamp verifier)
 
 Small additive follow-up to 8.4.0, from the native accelerator's first consumption pass:
diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md
index e58eb2ce..11d86ec8 100644
--- a/docs/guides/aggregation.md
+++ b/docs/guides/aggregation.md
@@ -11,7 +11,13 @@ No batch jobs. No scheduled recalculations. Aggregates stay current with every w
 **Defining over existing data:** if you define an aggregate on a store that already holds
 matching entities, Brainy backfills it from those entities on the first query (a one-time scan,
 then purely incremental). So `defineAggregate()` behaves the same whether you define it before
-or after the data exists — including when a persisted brain reopens already populated.
+or after the data exists.
+
+**Reopening a persisted brain:** aggregate state persists across restarts. Re-defining the
+same aggregate at boot (the normal declarative pattern) adopts the persisted state directly —
+no rescan. A backfill scan runs only when the definition actually changed, when no persisted
+state exists, or when the state failed to load; and however many aggregates need backfilling,
+they share a single scan.
 
 ## Quick Start
 
diff --git a/docs/guides/find-limits.md b/docs/guides/find-limits.md
index 66418b80..4c7fd252 100644
--- a/docs/guides/find-limits.md
+++ b/docs/guides/find-limits.md
@@ -40,6 +40,10 @@ Brainy picks `maxLimit` from the first of these that's available:
 
 Worked example: a 4 GB Cloud Run container picks priority 3 → `floor(4 GB × 0.25 / 25 KB) = floor(40 960) = 40 000` results. A 900 MB free-memory box on priority 4 gets `floor(900 MB / 25 KB) = ~36 000`.
 
+The cap is fixed at construction and never changes at runtime. Query timing is recorded
+for diagnostics only — a burst of slow queries cannot silently shrink the cap, and the
+auto-detected tiers (3 and 4) never go below a floor of 10 000.
+
 > **Calibration note.** Pre-7.30.2 used 100 KB per result instead of 25 KB, which produced caps that were 4× too tight for typical workloads (an 8 KB / result reality). 7.30.2 recalibrated to match observed entity sizes; existing `limit: 10_000` safety patterns now pass silently on any reasonably-sized box.
 
 ## What happens when you exceed the cap
diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts
index ecd16228..d74d732b 100644
--- a/src/aggregation/AggregationIndex.ts
+++ b/src/aggregation/AggregationIndex.ts
@@ -327,6 +327,22 @@ export class AggregationIndex {
   /** Track aggregates with stale MIN/MAX (need lazy recompute) */
   private staleMinMax = new Map<string, Set<string>>()
 
+  /** Resolves when init() has finished loading persisted definitions/state. */
+  private initPromise: Promise<void> | null = null
+
+  /** True once init() has settled (success or failure). */
+  private initDone = false
+
+  /**
+   * Aggregates registered by the app before init() finished loading persisted
+   * state, awaiting reconciliation: init() adopts the persisted state when the
+   * definition hash matches; anything left unadopted when init settles resolves
+   * to a backfill. Deciding backfill eagerly at define time was the boot-order
+   * bug that wiped valid persisted state on every restart — the synchronous
+   * defineAggregate() always beats the async init().
+   */
+  private pendingAdopt = new Set<string>()
+
   constructor(storage: StorageAdapter, nativeProvider?: AggregationProvider) {
     this.storage = storage
     this.nativeProvider = nativeProvider
@@ -336,19 +352,87 @@ export class AggregationIndex {
 
   /**
    * Initialize: load persisted definitions and state, detect changes, rebuild stale.
+   *
+   * Idempotent — repeated calls return the same promise. Definitions registered
+   * *before* this completes (the normal boot order: `defineAggregate()` is
+   * synchronous and always beats this async load) are reconciled rather than
+   * clobbered: the app's definition wins, and its persisted state is adopted
+   * when the definition hash matches — backfill happens only on a real change.
    */
-  async init(): Promise<void> {
+  init(): Promise<void> {
+    if (!this.initPromise) {
+      this.initPromise = this.loadPersisted().finally(() => {
+        this.resolvePendingAdoptToBackfill()
+        this.initDone = true
+      })
+    }
+    return this.initPromise
+  }
+
+  /**
+   * Await the persisted-state load (if one was started) and settle every
+   * pending adoption decision. After this resolves, `getPendingBackfills()`
+   * is authoritative: a name is listed iff it genuinely needs a rescan.
+   * Query paths must await this before consulting backfill state.
+   */
+  async ready(): Promise<void> {
+    if (this.initPromise) {
+      try {
+        await this.initPromise
+      } catch {
+        // The owner already surfaced the load failure loudly; backfill covers.
+      }
+    }
+    this.resolvePendingAdoptToBackfill()
+  }
+
+  /**
+   * Any definition still awaiting state adoption has no persisted state to
+   * adopt (or init never ran / failed) — it must backfill.
+   */
+  private resolvePendingAdoptToBackfill(): void {
+    for (const name of this.pendingAdopt) this.needsBackfill.add(name)
+    this.pendingAdopt.clear()
+  }
+
+  private async loadPersisted(): Promise<void> {
     // Load persisted definitions
     const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY)
     if (savedDefs && typeof savedDefs === 'object' && savedDefs.definitions) {
       const defs = savedDefs.definitions as Array<AggregateDefinition & { _hash?: string }>
 
       for (const def of defs) {
-        this.definitions.set(def.name, def)
-        const currentHash = hashDefinition(def)
         const savedHash = def._hash || ''
 
-        // Load persisted state
+        if (this.definitions.has(def.name)) {
+          // The app re-registered this aggregate before the load finished.
+          // The app's definition wins — never clobber it with the persisted
+          // copy. Adopt the persisted state when the definition is unchanged
+          // AND no write has landed for it yet (a landed write would be lost
+          // by adoption; the hook flips such names to backfill).
+          const appHash = this.definitionHashes.get(def.name) || ''
+          if (appHash === savedHash && this.pendingAdopt.has(def.name)) {
+            const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
+            if (stateData && stateData.groups) {
+              const groupMap = new Map<string, AggregateGroupState>()
+              for (const group of stateData.groups as AggregateGroupState[]) {
+                groupMap.set(serializeGroupKey(group.groupKey), group)
+              }
+              this.states.set(def.name, groupMap)
+              this.pendingAdopt.delete(def.name)
+              this.needsBackfill.delete(def.name)
+            }
+            // No/invalid persisted state: stays in pendingAdopt and resolves
+            // to backfill when init settles.
+          }
+          continue
+        }
+
+        // Not registered this session — restore definition + state from
+        // persistence.
+        this.definitions.set(def.name, def)
+        const currentHash = hashDefinition(def)
+
         const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
         if (stateData && stateData.groups && savedHash === currentHash) {
           // Definition unchanged — load state
@@ -358,6 +442,7 @@ export class AggregationIndex {
             groupMap.set(serialized, group)
           }
           this.states.set(def.name, groupMap)
+          this.needsBackfill.delete(def.name)
         } else {
           // Definition changed or no saved state — start fresh and backfill from
           // existing entities (the owner drains needsBackfill on first query).
@@ -452,10 +537,19 @@ export class AggregationIndex {
     this.definitions.set(def.name, def)
     this.definitionHashes.set(def.name, newHash)
 
+    // First sight this session, before init() settled: defer the backfill
+    // decision — init() adopts the persisted state on hash match, and anything
+    // left unadopted resolves to backfill. Deciding eagerly here wiped valid
+    // persisted state on every restart.
+    if (!this.states.has(def.name) && !this.initDone) {
+      this.states.set(def.name, new Map())
+      this.pendingAdopt.add(def.name)
+    }
     // Reset state if definition changed or doesn't exist yet, and flag it for
     // backfill so already-stored entities are counted (write-time hooks only see
     // future writes). The owner drains this on the next query via getPendingBackfills().
-    if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
+    else if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
+      this.pendingAdopt.delete(def.name)
       this.states.set(def.name, new Map())
       this.needsBackfill.add(def.name)
     }
@@ -476,6 +570,8 @@ export class AggregationIndex {
     this.definitionHashes.delete(name)
     this.states.delete(name)
     this.staleMinMax.delete(name)
+    this.pendingAdopt.delete(name)
+    this.needsBackfill.delete(name)
 
     // Notify native provider
     if (this.nativeProvider?.removeAggregate) {
@@ -545,6 +641,20 @@ export class AggregationIndex {
 
   // ============= Write-Time Hooks =============
 
+  /**
+   * A write is landing for an aggregate whose persisted-state adoption is still
+   * pending — adopting after this write would lose its contribution. Settle the
+   * decision now: an exact rescan instead of adoption. The window is the few
+   * milliseconds between a boot-time defineAggregate() and init() completing,
+   * so this rarely fires; when it does, correctness wins over the walk.
+   */
+  private resolveAdoptOnWrite(name: string): void {
+    if (this.pendingAdopt.has(name)) {
+      this.pendingAdopt.delete(name)
+      this.needsBackfill.add(name)
+    }
+  }
+
   /**
    * Called when an entity is added. Updates all matching aggregates.
    */
@@ -553,6 +663,7 @@ export class AggregationIndex {
 
     for (const [name, def] of this.definitions) {
       if (!matchesSource(entity, def.source)) continue
+      this.resolveAdoptOnWrite(name)
 
       if (this.nativeProvider) {
         const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'add')
@@ -579,6 +690,10 @@ export class AggregationIndex {
       const oldMatches = matchesSource(oldEntity, def.source)
       const newMatches = matchesSource(newEntity, def.source)
 
+      if (oldMatches || newMatches) {
+        this.resolveAdoptOnWrite(name)
+      }
+
       if (this.nativeProvider && (oldMatches || newMatches)) {
         const results = this.nativeProvider.incrementalUpdate(name, def, newEntity, 'update', oldEntity)
         this.applyNativeResults(name, results)
@@ -605,6 +720,7 @@ export class AggregationIndex {
 
     for (const [name, def] of this.definitions) {
       if (!matchesSource(entity, def.source)) continue
+      this.resolveAdoptOnWrite(name)
 
       if (this.nativeProvider) {
         const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'delete')
diff --git a/src/brainy.ts b/src/brainy.ts
index 383b1195..12fda014 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -598,6 +598,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   private _hub?: IntegrationHub     // Integration Hub for external tools
   private _pendingMigrationRunner?: MigrationRunner  // Deferred migration runner for large datasets
   private _aggregationIndex?: AggregationIndex       // Incremental aggregation engine
+  private _aggregationBackfillFlight: Promise<void> | null = null  // Single-flight backfill walk
   private _materializer?: AggregateMaterializer      // Debounced materialization of aggregate results
   /**
    * Fields registered via `brain.trackField()` — drives optional value validation on
@@ -5783,6 +5784,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   ): Promise<AggregateResult[]> {
     await this.ensureInitialized()
     this.ensureAggregationIndex()
+    // Persisted definitions load asynchronously — wait for them before deciding
+    // the aggregate doesn't exist (an app that relies on persisted definitions
+    // without re-defining at boot would otherwise race a spurious throw here).
+    await this._aggregationIndex!.ready()
     if (!this._aggregationIndex!.hasAggregate(name)) {
       throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`)
     }
@@ -15810,9 +15815,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    */
   private async backfillAggregateIfNeeded(name: string): Promise<void> {
     const index = this._aggregationIndex
-    if (!index || !index.getPendingBackfills().includes(name)) return
+    if (!index) return
 
-    index.beginBackfill(name)
+    // Persisted-state adoption happens inside ready(); after it resolves the
+    // pending-backfill set is authoritative (an unchanged definition with valid
+    // persisted state is NOT listed — no walk at all on a clean reopen).
+    await index.ready()
+
+    // Single-flight: concurrent queries share ONE walk instead of each wiping
+    // the others' partial state and starting their own (the stampede that kept
+    // a busy store from ever converging). The loop covers the rare case where
+    // the in-flight walk snapshotted its batch before `name` became pending —
+    // the next iteration starts a fresh walk that includes it.
+    while (index.getPendingBackfills().includes(name)) {
+      if (!this._aggregationBackfillFlight) {
+        this._aggregationBackfillFlight = this.runAggregationBackfillWalk()
+          .finally(() => {
+            this._aggregationBackfillFlight = null
+          })
+      }
+      await this._aggregationBackfillFlight
+    }
+  }
+
+  /**
+   * One store walk fills EVERY aggregate currently pending backfill — M pending
+   * aggregates cost one enumeration, not M. Only reached when an aggregate
+   * genuinely needs a rescan (new definition over a populated store, changed
+   * definition, or failed state load); a clean reopen adopts persisted state
+   * and never walks.
+   */
+  private async runAggregationBackfillWalk(): Promise<void> {
+    const index = this._aggregationIndex!
+    const names = index.getPendingBackfills()
+    if (names.length === 0) return
+
+    for (const n of names) index.beginBackfill(n)
 
     const PAGE = 500
     let offset = 0
@@ -15822,7 +15860,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
       })
       for (const noun of page.items) {
-        index.backfillEntity(name, noun as unknown as Record<string, unknown>)
+        const record = noun as unknown as Record<string, unknown>
+        for (const n of names) {
+          index.backfillEntity(n, record)
+        }
       }
       if (!page.hasMore || page.items.length === 0) break
       if (page.nextCursor) {
@@ -15832,7 +15873,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       }
     }
 
-    index.finishBackfill(name)
+    for (const n of names) index.finishBackfill(n)
   }
 
   /**
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 73b8e859..7c4c7c45 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -377,7 +377,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
       id.startsWith('statistics_') ||
       id === 'statistics' ||
       id.startsWith('__chunk__') ||      // Metadata index chunks (roaring bitmap data)
-      id.startsWith('__sparse_index__')  // Metadata sparse indices (zone maps + bloom filters)
+      id.startsWith('__sparse_index__') || // Metadata sparse indices (zone maps + bloom filters)
+      id.startsWith('__aggregation_') || // Aggregation engine definitions + state (routing is
+                                         // identical to the unknown-key fallback these keys hit
+                                         // before being listed here — this only kills the
+                                         // per-boot "Unknown key format" warning)
+      isSingletonSystemKey(id)           // Known singletons (e.g. brainy:entityIdMapper) hit the
+                                         // same warn-then-route fallback without this — the
+                                         // routing below already handles them identically
 
     if (isSystemKey) {
       if (isSingletonSystemKey(id)) {
diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts
index 57a9a1c0..ca439524 100644
--- a/src/utils/paramValidation.ts
+++ b/src/utils/paramValidation.ts
@@ -209,8 +209,10 @@ export interface ValidationConfigOptions {
 }
 
 /**
- * Auto-configured limits based on system resources
- * These adapt to available memory and observed performance
+ * Auto-configured limits based on system resources.
+ * Derived from memory (explicit overrides > reserved memory > container limit >
+ * free memory). Query timing is recorded for diagnostics only — it never
+ * changes the cap (see `recordQuery`).
  */
 export class ValidationConfig {
   private static instance: ValidationConfig | null = null
@@ -323,24 +325,23 @@ export class ValidationConfig {
   }
 
   /**
-   * Learn from actual usage to adjust limits
+   * Record query timing for diagnostics. Telemetry ONLY — never mutates the cap.
+   *
+   * `maxLimit` is a MEMORY-protection bound; query duration says nothing about
+   * memory-per-result, so duration must never drive it. An earlier version
+   * "learned" here: while the lifetime-average query time exceeded 1s it shrank
+   * `maxLimit` by 20% per recorded query down to a floor of 1000 — below the
+   * documented `MIN_AUTO_QUERY_LIMIT` (10 000) and with no recovery once slow
+   * samples poisoned the cumulative average. On a production host a burst of
+   * slow aggregate queries silently strangled every consumer's `find()` to
+   * 1000 while the error message blamed "available free memory" — exactly the
+   * silent throttling this module's own contract forbids. The cap now comes
+   * from its construction-time basis (or explicit overrides) alone.
    */
   recordQuery(duration: number, resultCount: number) {
+    void resultCount
     this.queryCount++
     this.avgQueryTime = (this.avgQueryTime * (this.queryCount - 1) + duration) / this.queryCount
-
-    // Only auto-adjust if not using explicit overrides
-    if (this.limitBasis !== 'override') {
-      // If queries are consistently fast with large results, increase limits
-      if (this.avgQueryTime < 100 && resultCount > this.maxLimit * 0.8) {
-        this.maxLimit = Math.min(this.maxLimit * 1.5, 100000)
-      }
-
-      // If queries are slow, reduce limits
-      if (this.avgQueryTime > 1000) {
-        this.maxLimit = Math.max(this.maxLimit * 0.8, 1000)
-      }
-    }
   }
 }
 
diff --git a/tests/integration/aggregation-state-persistence.test.ts b/tests/integration/aggregation-state-persistence.test.ts
new file mode 100644
index 00000000..4ec3d22d
--- /dev/null
+++ b/tests/integration/aggregation-state-persistence.test.ts
@@ -0,0 +1,235 @@
+/**
+ * @module tests/integration/aggregation-state-persistence
+ * @description The boot-order contract for the aggregation engine. Five laws:
+ * (1) STATE ADOPTION — a reopen with an unchanged defineAggregate() adopts the
+ *     persisted state and performs NO store walk. (The pre-fix behavior: the
+ *     synchronous define always beat the async init, flagged a backfill, and
+ *     the first query wiped the just-loaded state and re-walked the whole
+ *     store — every restart, forever.)
+ * (2) SINGLE-FLIGHT + BATCH — concurrent cold queries across multiple pending
+ *     aggregates share exactly ONE store walk; a query never wipes another's
+ *     partial progress and M pending aggregates cost one enumeration, not M.
+ * (3) CHANGED DEFINITION — a real definition change still backfills, exactly.
+ * (4) PERSISTED-ONLY DEFINITIONS — an app that does not re-define at boot can
+ *     query a persisted aggregate without racing a spurious "not defined".
+ * (5) QUIET KEYS — the engine's persistence keys (__aggregation_*) are
+ *     recognized system keys: no "Unknown key format" warning at boot.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+import { NounType } from '../../src/types/graphTypes.js'
+import type { AggregateDefinition } from '../../src/types/brainy.types.js'
+import { prodLog } from '../../src/utils/logger.js'
+
+const SPENDING: AggregateDefinition = {
+  name: 'spending',
+  source: { type: NounType.Event, where: { domain: 'financial' } },
+  groupBy: ['category'],
+  metrics: {
+    total: { op: 'sum', field: 'amount' },
+    count: { op: 'count' }
+  }
+}
+
+/** Same name, different metrics — a REAL definition change (hash differs). */
+const SPENDING_CHANGED: AggregateDefinition = {
+  ...SPENDING,
+  metrics: {
+    total: { op: 'sum', field: 'amount' },
+    count: { op: 'count' },
+    average: { op: 'avg', field: 'amount' }
+  }
+}
+
+describe('aggregation state persistence — boot-order contract', () => {
+  let dir: string
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-agg-persist-'))
+  })
+
+  afterEach(() => {
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  const open = async (): Promise<any> => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true
+    })
+    await b.init()
+    return b
+  }
+
+  /** Count store walks by intercepting the storage adapter's getNouns. */
+  const countWalks = (brain: any): { count: () => number } => {
+    const storage = brain.storage
+    const orig = storage.getNouns.bind(storage)
+    let calls = 0
+    storage.getNouns = async (opts: unknown) => {
+      calls++
+      return orig(opts)
+    }
+    return { count: () => calls }
+  }
+
+  const seed = async (brain: any): Promise<void> => {
+    for (let i = 0; i < 12; i++) {
+      await brain.add({
+        data: `tx ${i}`,
+        type: NounType.Event,
+        metadata: {
+          domain: 'financial',
+          category: i % 2 === 0 ? 'food' : 'transport',
+          amount: 10 + i
+        }
+      })
+    }
+  }
+
+  it('adopts persisted state on reopen with an unchanged definition — zero walks', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    const before = await brain1.queryAggregate('spending')
+    expect(before.length).toBe(2)
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING) // the standard declarative boot pattern
+    await brain2.getNounCount() // settle init paths before counting walks
+    const walks = countWalks(brain2)
+
+    const after = await brain2.queryAggregate('spending')
+
+    expect(walks.count()).toBe(0)
+    const key = (r: any) => r.groupKey.category
+    expect(
+      after.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort()
+    ).toEqual(
+      before.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort()
+    )
+    await brain2.close()
+  })
+
+  it('adopted state keeps accumulating: post-reopen writes land on top of it', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING)
+    // A write BEFORE the first query: if it lands before state adoption the
+    // engine must choose an exact rescan over adoption — either way the
+    // result must include all 13 entities.
+    await brain2.add({
+      data: 'late tx',
+      type: NounType.Event,
+      metadata: { domain: 'financial', category: 'food', amount: 100 }
+    })
+    const rows = await brain2.queryAggregate('spending')
+    const food = rows.find((r: any) => r.groupKey.category === 'food')
+    expect(food.metrics.count).toBe(7) // 6 seeded + 1 late
+    await brain2.close()
+  })
+
+  it('a changed definition still backfills — exactly once, with correct results', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING_CHANGED)
+    await brain2.getNounCount()
+    const walks = countWalks(brain2)
+
+    const rows = await brain2.queryAggregate('spending')
+
+    expect(walks.count()).toBe(1) // 12 entities = one page = one getNouns call
+    const food = rows.find((r: any) => r.groupKey.category === 'food')
+    expect(food.metrics.count).toBe(6)
+    expect(food.metrics.average).toBeCloseTo(food.metrics.total / 6)
+    await brain2.close()
+  })
+
+  it('concurrent cold queries across two aggregates share exactly ONE walk', async () => {
+    const brain = await open()
+    brain.defineAggregate(SPENDING)
+    brain.defineAggregate({
+      ...SPENDING,
+      name: 'by_category_count',
+      metrics: { count: { op: 'count' } }
+    })
+    await seed(brain)
+    await brain.getNounCount()
+    const walks = countWalks(brain)
+
+    const results = await Promise.all([
+      brain.queryAggregate('spending'),
+      brain.queryAggregate('by_category_count'),
+      brain.queryAggregate('spending'),
+      brain.queryAggregate('by_category_count'),
+      brain.queryAggregate('spending'),
+      brain.queryAggregate('by_category_count')
+    ])
+
+    expect(walks.count()).toBe(1) // 12 entities = one page; one walk fills both
+    for (const rows of results) {
+      const total = rows.reduce((s: number, r: any) => s + r.metrics.count, 0)
+      expect(total).toBe(12)
+    }
+
+    // Warm re-query: converged, no further walks.
+    await brain.queryAggregate('spending')
+    expect(walks.count()).toBe(1)
+    await brain.close()
+  })
+
+  it('persisted-only definitions are queryable without re-defining at boot', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    // NO defineAggregate — the app relies on the persisted definition.
+    const walks = countWalks(brain2)
+    const rows = await brain2.queryAggregate('spending')
+    expect(walks.count()).toBe(0) // persisted state adopted here too
+    expect(rows.length).toBe(2)
+    await brain2.close()
+  })
+
+  it('aggregation persistence keys never log "Unknown key format"', async () => {
+    const warnSpy = vi.spyOn(prodLog, 'warn')
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING)
+    await brain2.queryAggregate('spending')
+    await brain2.close()
+
+    const offenders = warnSpy.mock.calls
+      .map(args => String(args[0]))
+      .filter(
+        msg =>
+          msg.includes('Unknown key format') &&
+          (msg.includes('__aggregation_') || msg.includes('brainy:entityIdMapper'))
+      )
+    expect(offenders).toEqual([])
+    warnSpy.mockRestore()
+  })
+})
diff --git a/tests/unit/aggregation/AggregationIndex.test.ts b/tests/unit/aggregation/AggregationIndex.test.ts
index 05f4a62a..a0c10e62 100644
--- a/tests/unit/aggregation/AggregationIndex.test.ts
+++ b/tests/unit/aggregation/AggregationIndex.test.ts
@@ -675,4 +675,104 @@ describe('AggregationIndex', () => {
       await reloaded.close()
     })
   })
+
+  // ============= Boot-order reconciliation =============
+  //
+  // The production boot pattern: defineAggregate() is synchronous and always
+  // beats the async init() that loads persisted state. The old code flagged a
+  // backfill at define time and init never cleared it — so the loaded state
+  // was wiped and the whole store re-walked on EVERY restart.
+
+  describe('boot-order reconciliation (define-before-init)', () => {
+    const DEF: AggregateDefinition = {
+      name: 'boot_agg',
+      source: { type: NounType.Event },
+      groupBy: ['category'],
+      metrics: { count: { op: 'count' } }
+    }
+
+    const entity = (id: string, category: string): Record<string, unknown> => ({
+      id,
+      noun: NounType.Event,
+      metadata: { category },
+      createdAt: Date.now(),
+      updatedAt: Date.now()
+    })
+
+    /** Simulate the previous session: define, contribute, flush. */
+    async function seedAndFlush(store: MemoryStorage): Promise<void> {
+      const first = new AggregationIndex(store)
+      await first.init()
+      first.defineAggregate(DEF)
+      first.onEntityAdded('e1', entity('e1', 'food'))
+      first.onEntityAdded('e2', entity('e2', 'food'))
+      first.onEntityAdded('e3', entity('e3', 'transport'))
+      await first.flush()
+    }
+
+    it('adopts persisted state when define beats init with an unchanged definition', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const second = new AggregationIndex(store)
+      second.defineAggregate(DEF) // synchronous define FIRST — the real boot order
+      await second.init()
+      await second.ready()
+
+      expect(second.getPendingBackfills()).toEqual([])
+      const rows = second.queryAggregate({ name: 'boot_agg' })
+      const food = rows.find(r => r.groupKey.category === 'food')!
+      expect(food.metrics.count).toBe(2)
+    })
+
+    it('a write landing before adoption forces an exact rescan instead', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const second = new AggregationIndex(store)
+      second.defineAggregate(DEF)
+      second.onEntityAdded('e4', entity('e4', 'food')) // lands before init settles
+      await second.init()
+      await second.ready()
+
+      // Adoption would lose e4's contribution — the engine must rescan.
+      expect(second.getPendingBackfills()).toEqual(['boot_agg'])
+    })
+
+    it('init never clobbers a changed app definition registered before it', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const CHANGED: AggregateDefinition = {
+        ...DEF,
+        metrics: { count: { op: 'count' }, total: { op: 'sum', field: 'amount' } }
+      }
+      const second = new AggregationIndex(store)
+      second.defineAggregate(CHANGED)
+      await second.init()
+      await second.ready()
+
+      const def = second.getDefinitions().find(d => d.name === 'boot_agg')!
+      expect(Object.keys(def.metrics).sort()).toEqual(['count', 'total'])
+      expect(second.getPendingBackfills()).toEqual(['boot_agg'])
+    })
+
+    it('init alone restores persisted definitions with adopted state, no backfill', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const second = new AggregationIndex(store)
+      await second.init()
+      await second.ready()
+
+      expect(second.hasAggregate('boot_agg')).toBe(true)
+      expect(second.getPendingBackfills()).toEqual([])
+      const rows = second.queryAggregate({ name: 'boot_agg' })
+      expect(rows.reduce((s, r) => s + (r.metrics.count as number), 0)).toBe(3)
+    })
+  })
 })
diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts
index e7113506..4dc83554 100644
--- a/tests/unit/utils/paramValidation.test.ts
+++ b/tests/unit/utils/paramValidation.test.ts
@@ -270,27 +270,24 @@ describe('Zero-Config Parameter Validation', () => {
       expect(config.availableMemory).toBeGreaterThan(0)
     })
     
-    it('should adapt limits based on query performance', () => {
-      const initialConfig = getValidationConfig()
-      const initialLimit = initialConfig.maxLimit
-      
-      // Simulate fast queries with large results
+    it('never mutates the cap from query timing (telemetry only)', () => {
+      const initialLimit = getValidationConfig().maxLimit
+
+      // Fast queries with large results: no silent growth.
       for (let i = 0; i < 10; i++) {
         recordQueryPerformance(50, initialLimit * 0.9)
       }
-      
-      const updatedConfig = getValidationConfig()
-      // Limit might increase if performance is good
-      expect(updatedConfig.maxLimit).toBeGreaterThanOrEqual(initialLimit)
-      
-      // Simulate slow queries
-      for (let i = 0; i < 10; i++) {
-        recordQueryPerformance(2000, 100)
+      expect(getValidationConfig().maxLimit).toBe(initialLimit)
+
+      // A burst of catastrophically slow queries must not strangle the cap.
+      // The removed "learning" ratchet shrank it 20% per recorded query down
+      // to a floor of 1000 — below the documented MIN_AUTO_QUERY_LIMIT — and
+      // the error message blamed "available free memory" (a production
+      // incident: every find({ limit: 5000 }) failed on an idle 23GB-free box).
+      for (let i = 0; i < 50; i++) {
+        recordQueryPerformance(90_000, 100)
       }
-      
-      const finalConfig = getValidationConfig()
-      // Limit should decrease if performance is poor
-      expect(finalConfig.maxLimit).toBeLessThanOrEqual(updatedConfig.maxLimit)
+      expect(getValidationConfig().maxLimit).toBe(initialLimit)
     })
   })
 })
\ No newline at end of file

From 01a7f3dd017401f92ff62acff645eec9c7b7722a Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 09:29:58 -0700
Subject: [PATCH 59/82] chore(release): 8.5.1

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a7b0044b..95bd9a5d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17)
+
+- fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7)
+- docs: external-backups/sparse-storage guide + generation fact log concept (593bb8b)
+
+
 ### [8.5.0](https://github.com/soulcraftlabs/brainy/compare/v8.4.0...v8.5.0) (2026-07-15)
 
 - test: tolerant timing assertion in the execution-time measure test (4dc0a92)
diff --git a/package-lock.json b/package-lock.json
index 183974d8..1f3f343a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.5.0",
+  "version": "8.5.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.5.0",
+      "version": "8.5.1",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 92b1ba33..a5664887 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.5.0",
+  "version": "8.5.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From a77b064bd7e858e1cab884cf75a2cbe9344b10b7 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 16:00:11 -0700
Subject: [PATCH 60/82] fix: exception-safe aggregation backfill +
 generation-verified adoption + loud open-path guards
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Aggregation backfill walks build into a STAGING map and swap in atomically
  on completion. A mid-walk failure drops the staging map — the previous live
  state keeps serving, the aggregate stays flagged pending, and the storage
  error surfaces to the failing query. Previously the walk wiped live state
  before a scan that could throw, never cleared the pending flag on failure,
  and re-ran a full walk on every subsequent query: a silent wipe/walk/throw
  loop at the caller's retry rate.
- Failed walks are latched: retries within a 30s cooldown rethrow the recorded
  error instantly instead of re-walking, so a tight caller-side retry loop
  costs one loud error per query, never a full store walk per query.
- Persisted aggregation state is stamped with the store's committed generation
  at flush; reopen adoption requires stamp equality. Stale state (unclean
  shutdown) or over-counting state (a log truncation on a copied store pulled
  the watermark back) triggers exactly one loud rescan, never a silent adopt.
- The backfill/adoption path narrates: adoption decisions, walk start/finish
  with counts and duration, and failures all log by default.
- getNouns/getVerbs refuse a supplied-but-undecodable pagination cursor with a
  loud error instead of silently restarting the walk at offset 0 (which
  re-served page 1 forever to any while(hasMore) caller).
- The graph cold-load verb walk aborts loudly on a missing or non-advancing
  cursor with hasMore=true.
- A versioned index provider whose generation is AHEAD of the committed
  watermark (torn copy / crash-recovery truncation) is now named loudly at
  open, alongside the existing behind-direction message.
---
 RELEASES.md                                   |  38 +++++++
 src/aggregation/AggregationIndex.ts           | 101 +++++++++++++++--
 src/brainy.ts                                 | 104 +++++++++++++++---
 src/graph/graphAdjacencyIndex.ts              |  10 ++
 src/storage/baseStorage.ts                    |  28 ++++-
 .../aggregation-state-persistence.test.ts     |  76 +++++++++++++
 .../storage/verb-cursor-pagination.test.ts    |  14 ++-
 7 files changed, 335 insertions(+), 36 deletions(-)

diff --git a/RELEASES.md b/RELEASES.md
index 920f0d79..540fa6fb 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,44 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud)
+
+Hardening patch from a migration incident (a byte-copied store on new hardware; the service
+entered a silent full-CPU loop at boot). Four changes, all in the aggregation engine's
+backfill/adoption path:
+
+- **Backfill walks are exception-safe and non-destructive.** A rescan now builds into a
+  staging map and swaps in atomically on completion; a mid-walk failure drops the staging map,
+  keeps the previous live state serving, and surfaces the storage error to the failing query.
+  Previously the walk wiped live state *before* a scan that could throw, never cleared the
+  pending flag on failure, and re-ran a full walk on every subsequent query — a silent
+  wipe/walk/throw loop at the caller's retry rate.
+- **Failed walks are latched.** After a walk fails, retries within a 30-second cooldown rethrow
+  the recorded error instantly instead of re-walking — a tight caller-side retry loop now costs
+  one loud error per query, never a full store walk per query.
+- **Adoption is generation-verified.** Persisted aggregation state is stamped with the store's
+  committed generation at flush; reopen adoption requires the stamp to equal the current
+  watermark. Stale state (unclean shutdown) or over-counting state (a fact-log truncation on a
+  copied store pulled the watermark back) triggers exactly one loud rescan — never a silent
+  adopt. Pre-8.5.2 state on generation-aware stores rescans once after upgrade, then is stamped.
+- **The path narrates.** Adoption decisions, no-adoptable-state outcomes, walk start/finish
+  (entity count + duration), and walk failures all log by default; a non-advancing storage
+  pagination cursor aborts the walk loudly instead of looping forever.
+
+Plus three guards from a full audit of every loop in the open/init path:
+
+- **Invalid pagination cursors fail loudly.** A supplied-but-undecodable resume token to
+  `getNouns`/`getVerbs` used to silently restart the walk at offset 0 — to a `while(hasMore)`
+  caller that re-serves page 1 forever (an unbounded silent CPU loop). It now throws with a
+  clear message instead.
+- **The graph cold-load verb walk has a stall guard.** `hasMore=true` with a missing or
+  non-advancing cursor aborts loudly instead of re-reading the same page forever.
+- **A derived index AHEAD of the store is named at open.** Brainy already surfaced a provider
+  generation *behind* the committed watermark; the *ahead* direction (the signature of a
+  byte-copy of a live service, or a log truncation during crash recovery) now logs a loud
+  warning explaining what happened and that `brain.repairIndex()` forces a heal — instead of
+  passing unnamed into whatever the derived index does next.
+
 ## v8.5.1 — 2026-07-17 (aggregation state survives restarts + the query-cap ratchet removed)
 
 Patch release from a production incident (aggregate/count paths taking 40–90 s on an idle box
diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts
index d74d732b..7f7ffb27 100644
--- a/src/aggregation/AggregationIndex.ts
+++ b/src/aggregation/AggregationIndex.ts
@@ -29,6 +29,7 @@ import { matchesMetadataFilter } from '../utils/metadataFilter.js'
 import { compareCodePoints } from '../utils/collation.js'
 import { bucketTimestamp } from './timeWindows.js'
 import { NounType } from '../types/graphTypes.js'
+import { prodLog } from '../utils/logger.js'
 
 /** Persistence key for aggregate definitions */
 const DEFINITIONS_KEY = '__aggregation_definitions__'
@@ -343,6 +344,14 @@ export class AggregationIndex {
    */
   private pendingAdopt = new Set<string>()
 
+  /**
+   * In-flight rescan targets. While a name has a staging map, ALL
+   * contributions (the walk's and concurrent write hooks') land there instead
+   * of the live map; the live map keeps serving until {@link finishBackfill}
+   * swaps the staging map in atomically.
+   */
+  private backfillStaging = new Map<string, Map<string, AggregateGroupState>>()
+
   constructor(storage: StorageAdapter, nativeProvider?: AggregationProvider) {
     this.storage = storage
     this.nativeProvider = nativeProvider
@@ -391,10 +400,37 @@ export class AggregationIndex {
    * adopt (or init never ran / failed) — it must backfill.
    */
   private resolvePendingAdoptToBackfill(): void {
+    if (this.pendingAdopt.size > 0) {
+      prodLog.info(
+        `[Aggregation] no adoptable persisted state for: ${Array.from(this.pendingAdopt).join(', ')} — flagged for backfill`
+      )
+    }
     for (const name of this.pendingAdopt) this.needsBackfill.add(name)
     this.pendingAdopt.clear()
   }
 
+  /**
+   * May this persisted state be ADOPTED? When the store exposes its committed
+   * watermark, the state's `sourceGeneration` must EQUAL it: behind means
+   * later writes are missing from the state (unclean shutdown); ahead means
+   * it counts writes that no longer exist (e.g. a fact-log truncation on a
+   * copied store pulled the watermark back). Either way: one exact rescan,
+   * said out loud — never a silent adopt. Stores without the capability (and
+   * pre-stamp state on them) fall back to hash-only adoption.
+   */
+  private stateGenerationAdoptable(name: string, stateData: unknown): boolean {
+    const committed = this.storage.committedGeneration?.() ?? null
+    if (committed === null) return true
+    const raw = (stateData as Record<string, unknown>).sourceGeneration
+    const stamped = typeof raw === 'number' ? raw : null
+    if (stamped === committed) return true
+    prodLog.warn(
+      `[Aggregation] '${name}': persisted state is at generation ${stamped ?? 'unstamped'} ` +
+        `but the store's committed generation is ${committed} — rescanning instead of adopting`
+    )
+    return false
+  }
+
   private async loadPersisted(): Promise<void> {
     // Load persisted definitions
     const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY)
@@ -413,7 +449,11 @@ export class AggregationIndex {
           const appHash = this.definitionHashes.get(def.name) || ''
           if (appHash === savedHash && this.pendingAdopt.has(def.name)) {
             const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
-            if (stateData && stateData.groups) {
+            if (
+              stateData &&
+              stateData.groups &&
+              this.stateGenerationAdoptable(def.name, stateData)
+            ) {
               const groupMap = new Map<string, AggregateGroupState>()
               for (const group of stateData.groups as AggregateGroupState[]) {
                 groupMap.set(serializeGroupKey(group.groupKey), group)
@@ -421,6 +461,9 @@ export class AggregationIndex {
               this.states.set(def.name, groupMap)
               this.pendingAdopt.delete(def.name)
               this.needsBackfill.delete(def.name)
+              prodLog.info(
+                `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — no rescan`
+              )
             }
             // No/invalid persisted state: stays in pendingAdopt and resolves
             // to backfill when init settles.
@@ -434,7 +477,12 @@ export class AggregationIndex {
         const currentHash = hashDefinition(def)
 
         const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
-        if (stateData && stateData.groups && savedHash === currentHash) {
+        if (
+          stateData &&
+          stateData.groups &&
+          savedHash === currentHash &&
+          this.stateGenerationAdoptable(def.name, stateData)
+        ) {
           // Definition unchanged — load state
           const groupMap = new Map<string, AggregateGroupState>()
           for (const group of stateData.groups as AggregateGroupState[]) {
@@ -443,6 +491,9 @@ export class AggregationIndex {
           }
           this.states.set(def.name, groupMap)
           this.needsBackfill.delete(def.name)
+          prodLog.info(
+            `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)`
+          )
         } else {
           // Definition changed or no saved state — start fresh and backfill from
           // existing entities (the owner drains needsBackfill on first query).
@@ -483,14 +534,22 @@ export class AggregationIndex {
     }))
     await this.storage.saveMetadata(DEFINITIONS_KEY, { definitions: defsToSave })
 
-    // Persist dirty states
+    // Persist dirty states, stamped with the committed generation they
+    // reflect. The stamp is what makes reopen-adoption verifiable: state at a
+    // different generation than the store's committed watermark is stale (an
+    // unclean shutdown after later writes) or over-counts (a fact-log
+    // truncation on a copied store pulled the watermark BACK below the
+    // stamp) — either way the answer is one exact rescan, never a silent
+    // adopt. Read the generation after collecting groups so any racing
+    // commit resolves toward rescan, not wrong-adopt.
     for (const name of this.dirty) {
       const stateMap = this.states.get(name)
       if (stateMap) {
         const groups = Array.from(stateMap.values())
+        const sourceGeneration = this.storage.committedGeneration?.() ?? null
         await this.storage.saveMetadata(
           `${STATE_KEY_PREFIX}${name}__`,
-          { groups }
+          sourceGeneration === null ? { groups } : { groups, sourceGeneration }
         )
       }
     }
@@ -609,9 +668,17 @@ export class AggregationIndex {
     return Array.from(this.needsBackfill)
   }
 
-  /** Clear an aggregate's state so a full rescan cannot double-count. */
+  /**
+   * Begin a rescan into a STAGING map. The live state is not touched — it
+   * keeps serving (possibly stale, but flagged pending) until the rescan
+   * completes and swaps in atomically. A mid-walk failure drops the staging
+   * map via {@link abortBackfill} and loses nothing: wiping live state before
+   * a scan that could throw was the destructive-before-durable defect.
+   * Contributions (walk + concurrent write hooks) land in staging while it
+   * exists, so the swapped-in result reflects writes that raced the walk.
+   */
   beginBackfill(name: string): void {
-    this.states.set(name, new Map())
+    this.backfillStaging.set(name, new Map())
     // Reset native provider state for this aggregate too, if present.
     const def = this.definitions.get(name)
     if (def && this.nativeProvider?.removeAggregate && this.nativeProvider?.defineAggregate) {
@@ -620,6 +687,15 @@ export class AggregationIndex {
     }
   }
 
+  /**
+   * Abandon an in-flight rescan after a failure: drop the staging map, keep
+   * the live state serving, leave the aggregate flagged as pending so a later
+   * attempt rescans. The failure itself must be surfaced loudly by the owner.
+   */
+  abortBackfill(name: string): void {
+    this.backfillStaging.delete(name)
+  }
+
   /** Feed one already-stored entity into a single aggregate during backfill. */
   backfillEntity(name: string, entity: Record<string, unknown>): void {
     if (isAggregateEntity(entity)) return
@@ -633,8 +709,13 @@ export class AggregationIndex {
     }
   }
 
-  /** Mark an aggregate's backfill complete; rebuilt state persists on next flush(). */
+  /** Swap the rebuilt staging state in atomically; persists on next flush(). */
   finishBackfill(name: string): void {
+    const staged = this.backfillStaging.get(name)
+    if (staged) {
+      this.states.set(name, staged)
+      this.backfillStaging.delete(name)
+    }
     this.needsBackfill.delete(name)
     this.dirty.add(name)
   }
@@ -873,7 +954,7 @@ export class AggregationIndex {
     def: AggregateDefinition,
     entity: Record<string, unknown>
   ): void {
-    const stateMap = this.states.get(aggName)!
+    const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))!
 
     // Fan out: an unnest dimension makes one entity contribute to several groups.
     for (const groupKey of computeGroupKeys(entity, def.groupBy)) {
@@ -931,7 +1012,7 @@ export class AggregationIndex {
     def: AggregateDefinition,
     entity: Record<string, unknown>
   ): void {
-    const stateMap = this.states.get(aggName)!
+    const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))!
 
     // Fan out: reverse the entity's contribution from every group it joined.
     for (const groupKey of computeGroupKeys(entity, def.groupBy)) {
@@ -987,7 +1068,7 @@ export class AggregationIndex {
    * Apply results from native provider back into the state maps.
    */
   private applyNativeResults(aggName: string, results: AggregateGroupState[]): void {
-    const stateMap = this.states.get(aggName)!
+    const stateMap = (this.backfillStaging.get(aggName) ?? this.states.get(aggName))!
     for (const group of results) {
       const serialized = serializeGroupKey(group.groupKey)
       stateMap.set(serialized, group)
diff --git a/src/brainy.ts b/src/brainy.ts
index 12fda014..bc5cedd5 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -384,6 +384,14 @@ class InsertPreconditionExistsSignal extends Error {
  */
 export type IndexFamily = 'vector' | 'metadata' | 'graph'
 
+/**
+ * How long a failed aggregation-backfill walk suppresses fresh walk attempts.
+ * Within the window, queries rethrow the recorded failure instantly (loud,
+ * cheap); after it, one new attempt is allowed. Bounds the damage of a
+ * caller-side tight retry loop against a deterministically-failing store.
+ */
+const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000
+
 /**
  * The main Brainy class - Clean, Beautiful, Powerful
  * REAL IMPLEMENTATION - No stubs, no mocks
@@ -599,6 +607,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   private _pendingMigrationRunner?: MigrationRunner  // Deferred migration runner for large datasets
   private _aggregationIndex?: AggregationIndex       // Incremental aggregation engine
   private _aggregationBackfillFlight: Promise<void> | null = null  // Single-flight backfill walk
+  // A failed walk latches its error: retries within the cooldown rethrow it
+  // instantly instead of re-walking, so a tight caller-side retry loop costs
+  // one loud error per query, never a full store walk per query.
+  private _aggregationBackfillFailure: { at: number; error: Error } | null = null
   private _materializer?: AggregateMaterializer      // Debounced materialization of aggregate results
   /**
    * Fields registered via `brain.trackField()` — drives optional value validation on
@@ -1182,6 +1194,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
               `(storage committed: ${committed}) — provider replays the gap per ` +
               `the post-commit applier contract`
           )
+        } else if (providerGen > committed) {
+          // The AHEAD direction is incoherence, not a replay gap: the provider's
+          // persisted index claims writes the store no longer has — the signature
+          // of a torn copy or a log truncation that pulled the committed
+          // watermark back (crash recovery, byte-copy of a live store). A replay
+          // can never converge on it and index answers may reference vanished
+          // writes. Name it loudly at open so it is never diagnosed from a
+          // silent journal; the provider's own coherence check / heal walk (or
+          // brain.repairIndex()) is the cure.
+          prodLog.warn(
+            `[Brainy] Versioned index provider is AHEAD of the store: provider ` +
+              `generation ${providerGen} vs committed ${committed}. This store was ` +
+              `likely copied from a live service or truncated during crash recovery. ` +
+              `Derived-index answers may reference rolled-back writes until the ` +
+              `provider heals from canonical (brain.repairIndex() forces it).`
+          )
         }
       }
 
@@ -15828,6 +15856,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // the in-flight walk snapshotted its batch before `name` became pending —
     // the next iteration starts a fresh walk that includes it.
     while (index.getPendingBackfills().includes(name)) {
+      // Failure latch: a deterministically-failing walk must not be re-run at
+      // the caller's retry rate — that is a silent CPU loop wearing a retry
+      // loop's clothes. Within the cooldown, rethrow the recorded failure
+      // immediately; after it, one fresh attempt is allowed.
+      const failure = this._aggregationBackfillFailure
+      if (failure && Date.now() - failure.at < AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS) {
+        throw new Error(
+          `Aggregation backfill for '${name}' is in failure cooldown (retry in ` +
+            `${Math.ceil((AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS - (Date.now() - failure.at)) / 1000)}s). ` +
+            `Last failure: ${failure.error.message}`
+        )
+      }
       if (!this._aggregationBackfillFlight) {
         this._aggregationBackfillFlight = this.runAggregationBackfillWalk()
           .finally(() => {
@@ -15850,30 +15890,60 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     const names = index.getPendingBackfills()
     if (names.length === 0) return
 
+    prodLog.info(`[Aggregation] backfill walk starting for: ${names.join(', ')}`)
+    const startedAt = Date.now()
     for (const n of names) index.beginBackfill(n)
 
-    const PAGE = 500
-    let offset = 0
-    let cursor: string | undefined
-    for (;;) {
-      const page = await this.storage.getNouns({
-        pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
-      })
-      for (const noun of page.items) {
-        const record = noun as unknown as Record<string, unknown>
-        for (const n of names) {
-          index.backfillEntity(n, record)
+    let scanned = 0
+    try {
+      const PAGE = 500
+      let offset = 0
+      let cursor: string | undefined
+      for (;;) {
+        const page = await this.storage.getNouns({
+          pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
+        })
+        for (const noun of page.items) {
+          const record = noun as unknown as Record<string, unknown>
+          for (const n of names) {
+            index.backfillEntity(n, record)
+          }
+        }
+        scanned += page.items.length
+        if (!page.hasMore || page.items.length === 0) break
+        if (page.nextCursor) {
+          if (page.nextCursor === cursor) {
+            // A non-advancing cursor with hasMore=true would loop this walk at
+            // CPU speed forever, silently. That is a storage pagination defect —
+            // fail the waiting queries loudly instead of spinning.
+            throw new Error(
+              `Aggregation backfill aborted: storage pagination returned a non-advancing cursor ` +
+                `after ${scanned} entities with hasMore=true — the storage adapter's getNouns cursor is broken.`
+            )
+          }
+          cursor = page.nextCursor
+        } else {
+          offset += page.items.length
         }
       }
-      if (!page.hasMore || page.items.length === 0) break
-      if (page.nextCursor) {
-        cursor = page.nextCursor
-      } else {
-        offset += page.items.length
-      }
+    } catch (err) {
+      // Non-destructive failure: drop the staging maps (live state keeps
+      // serving), keep the aggregates flagged pending, latch the error so
+      // retries within the cooldown fail fast, and say all of it out loud.
+      for (const n of names) index.abortBackfill(n)
+      this._aggregationBackfillFailure = { at: Date.now(), error: err as Error }
+      prodLog.warn(
+        `[Aggregation] backfill walk FAILED after ${scanned} entities: ${(err as Error).message} — ` +
+          `prior aggregate state preserved; retries suppressed for ${AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS / 1000}s`
+      )
+      throw err
     }
 
     for (const n of names) index.finishBackfill(n)
+    this._aggregationBackfillFailure = null
+    prodLog.info(
+      `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms`
+    )
   }
 
   /**
diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts
index 816856b1..b37391aa 100644
--- a/src/graph/graphAdjacencyIndex.ts
+++ b/src/graph/graphAdjacencyIndex.ts
@@ -282,6 +282,16 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
       }
 
       hasMore = result.hasMore
+      if (hasMore && (!result.nextCursor || result.nextCursor === cursor)) {
+        // A stalled cursor with hasMore=true would re-read the same page
+        // forever — a silent full-CPU loop at cold open. Abort loudly; a
+        // graph read failing beats a process that spins without a log line.
+        throw new Error(
+          `GraphAdjacencyIndex: verb walk stalled after ${count} verbs — storage returned ` +
+            `hasMore=true with ${result.nextCursor ? 'a non-advancing' : 'no'} cursor. ` +
+            `Aborting the cold-load; run brain.repairIndex() if this persists.`
+        )
+      }
       cursor = result.nextCursor
     }
 
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 7c4c7c45..7ad00dab 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -2068,11 +2068,20 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     // Cursor (8.0): resume token carrying the (shard, nounId) of the last returned
     // noun — the noun mirror of getVerbsWithPagination. When present it supersedes
     // `offset` and resumes the shard walk immediately AFTER that position, so a full
-    // walk is O(N) instead of the O(N²) of offset paging. Malformed/foreign tokens
-    // decode to null → offset fallback. (Previously the cursor was ignored, which
-    // was latent — the only multi-page consumer used a single big page — until small
-    // chunk sizes needed page 2 and an offset-0-on-every-call walk never terminated.)
+    // walk is O(N) instead of the O(N²) of offset paging. (Previously the cursor was
+    // ignored, which was latent — the only multi-page consumer used a single big
+    // page — until small chunk sizes needed page 2 and an offset-0-on-every-call
+    // walk never terminated.)
     const cursor = this.decodeNounWalkCursor(options.cursor)
+    if (options.cursor && cursor === null) {
+      // A supplied-but-undecodable resume token must FAIL, not silently restart
+      // at offset 0 — to a while(hasMore) caller the silent fallback re-serves
+      // page 1 forever: an unbounded CPU loop wearing pagination's clothes.
+      throw BrainyError.storage(
+        `getNouns: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` +
+          `Restart it without a cursor.`
+      )
+    }
     const collected: Array<{ noun: HNSWNounWithMetadata; shard: number }> = []
 
     // Peek one past the window so `hasMore` is decidable. Cursor mode collects one
@@ -2366,8 +2375,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     // (shard, verbId) of the last returned verb. When present it SUPERSEDES `offset`
     // and resumes the shard walk immediately AFTER that position, so a full walk is
     // O(N) total instead of the O(N²) of offset paging (which re-scans from shard 0
-    // every page). Malformed / foreign tokens decode to null → offset fallback.
+    // every page).
     const cursor = this.decodeVerbWalkCursor(options.cursor)
+    if (options.cursor && cursor === null) {
+      // A supplied-but-undecodable resume token must FAIL, not silently restart
+      // at offset 0 — to a while(hasMore) caller the silent fallback re-serves
+      // page 1 forever: an unbounded CPU loop wearing pagination's clothes.
+      throw BrainyError.storage(
+        `getVerbs: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` +
+          `Restart it without a cursor.`
+      )
+    }
 
     // Each collected entry remembers its shard so nextCursor can point at the exact
     // (shard, id) resume position.
diff --git a/tests/integration/aggregation-state-persistence.test.ts b/tests/integration/aggregation-state-persistence.test.ts
index 4ec3d22d..db8fc088 100644
--- a/tests/integration/aggregation-state-persistence.test.ts
+++ b/tests/integration/aggregation-state-persistence.test.ts
@@ -209,6 +209,82 @@ describe('aggregation state persistence — boot-order contract', () => {
     await brain2.close()
   })
 
+  it('generation-mismatched persisted state is rescanned once, loudly — never adopted', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    // Simulate the copied-store incident class: a fact-log truncation (or an
+    // unclean shutdown) leaves the committed watermark different from the
+    // generation the flushed state was stamped with.
+    const tamper: any = await open()
+    const key = '__aggregation_state_spending__'
+    const stored = await tamper.storage.getMetadata(key)
+    expect(typeof stored.sourceGeneration).toBe('number') // the stamp is really persisted
+    await tamper.storage.saveMetadata(key, {
+      ...stored,
+      sourceGeneration: stored.sourceGeneration + 5
+    })
+    await tamper.close()
+
+    const warnSpy = vi.spyOn(prodLog, 'warn')
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING)
+    await brain2.getNounCount()
+    const walks = countWalks(brain2)
+
+    const rows = await brain2.queryAggregate('spending')
+
+    expect(walks.count()).toBe(1) // exactly ONE rescan — no silent adopt, no spin
+    const food = rows.find((r: any) => r.groupKey.category === 'food')
+    expect(food.metrics.count).toBe(6) // rescan produced exact results
+    expect(
+      warnSpy.mock.calls.some(args => String(args[0]).includes('rescanning instead of adopting'))
+    ).toBe(true) // and it said so out loud
+    warnSpy.mockRestore()
+    await brain2.close()
+  })
+
+  it('a failing walk is loud, non-destructive, and latched — never a silent retry loop', async () => {
+    // Fresh define + seeded writes: the write hooks have populated LIVE state,
+    // and the first-query rescan is still pending. The incident shape
+    // (wipe-before-scan + no try/catch + per-query re-walk) would have wiped
+    // that live state and silently re-walked on every query.
+    const brain: any = await open()
+    brain.defineAggregate(SPENDING)
+    await seed(brain)
+    expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2)
+
+    const storage = brain.storage
+    const origGetNouns = storage.getNouns.bind(storage)
+    let walkAttempts = 0
+    storage.getNouns = async () => {
+      walkAttempts++
+      throw new Error('injected storage failure')
+    }
+
+    // First query: the walk fails LOUDLY with the storage error.
+    await expect(brain.queryAggregate('spending')).rejects.toThrow('injected storage failure')
+    expect(walkAttempts).toBe(1)
+
+    // Live state was NOT destroyed by the failed walk (staging was dropped).
+    expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2)
+
+    // Second query inside the cooldown: instant loud failure, NO new walk.
+    await expect(brain.queryAggregate('spending')).rejects.toThrow('failure cooldown')
+    expect(walkAttempts).toBe(1)
+
+    // Heal the storage + expire the cooldown: one fresh walk succeeds exactly.
+    storage.getNouns = origGetNouns
+    brain._aggregationBackfillFailure.at = Date.now() - 60_000
+    const rows = await brain.queryAggregate('spending')
+    const food = rows.find((r: any) => r.groupKey.category === 'food')
+    expect(food.metrics.count).toBe(6)
+    await brain.close()
+  })
+
   it('aggregation persistence keys never log "Unknown key format"', async () => {
     const warnSpy = vi.spyOn(prodLog, 'warn')
     const brain1 = await open()
diff --git a/tests/unit/storage/verb-cursor-pagination.test.ts b/tests/unit/storage/verb-cursor-pagination.test.ts
index 985a650f..cff41d02 100644
--- a/tests/unit/storage/verb-cursor-pagination.test.ts
+++ b/tests/unit/storage/verb-cursor-pagination.test.ts
@@ -95,9 +95,15 @@ describe('verb cursor pagination (graph-perf #2)', () => {
     expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen))
   })
 
-  it('a foreign/malformed cursor falls back gracefully (no throw, starts from the beginning)', async () => {
-    const page = await storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
-    expect(page.items.length).toBe(5)
-    expect(page.hasMore).toBe(true)
+  it('a foreign/malformed cursor FAILS LOUDLY — never a silent restart from page 1', async () => {
+    // The old behavior (decode-null → silent offset-0 fallback) re-served page 1
+    // forever to any while(hasMore) walker: an unbounded CPU loop with no log
+    // line. An undecodable resume token now refuses the walk instead.
+    await expect(
+      storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
+    ).rejects.toThrow('invalid pagination cursor')
+    await expect(
+      storage.getNouns({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
+    ).rejects.toThrow('invalid pagination cursor')
   })
 })

From 07144ead866c54baff8d109537cb2d5b8cf62896 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 16:03:37 -0700
Subject: [PATCH 61/82] chore(release): 8.5.2

---
 CHANGELOG.md      | 5 +++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95bd9a5d..75d3cb6f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17)
+
+- fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064)
+
+
 ### [8.5.1](https://github.com/soulcraftlabs/brainy/compare/v8.5.0...v8.5.1) (2026-07-17)
 
 - fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal (da55be7)
diff --git a/package-lock.json b/package-lock.json
index 1f3f343a..6b12754f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.5.1",
+  "version": "8.5.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.5.1",
+      "version": "8.5.2",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index a5664887..f2004a57 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.5.1",
+  "version": "8.5.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 2a03fae0e20091f433018212ac05f68a54d95ec2 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 16:34:45 -0700
Subject: [PATCH 62/82] =?UTF-8?q?feat:=20brain.auditGraph()=20=E2=80=94=20?=
 =?UTF-8?q?read-only=20graph-truth=20audit?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- New public API: walks every canonical relationship record, queries the same
  read path applications use (related() with all visibility tiers included),
  and classifies every discrepancy into its failure family: missingFromReads
  (records the read path omits — stale adjacency), danglingEndpoints
  (endpoint entity gone — the partial-delete scar class), readOnlyVerbIds
  (read-path edges with no canonical record — ghosts). Design-hidden
  internal/system edges are counted separately so intentional hiding is never
  misclassified as loss. Counts exact; example lists capped with an explicit
  truncatedExamples flag; loud narration on incoherence; mutates nothing.
- Classification core lives in src/graph/graphAudit.ts behind injected seams
  (canonical walks + the end-to-end read) so the discrepancy logic is testable
  in isolation; brainy wires the seams to storage walks and related().
- getNounIdsWithPagination now refuses an undecodable resume cursor loudly —
  the third and final pagination walk brought under the 8.5.2 cursor contract.
- Types exported: GraphAuditReport, GraphAuditDiscrepancy. Docs: the
  inspection guide gains the audit -> repair -> audit verification ritual.
---
 RELEASES.md                           |  21 +++
 docs/guides/inspection.md             |  28 ++++
 src/brainy.ts                         |  83 ++++++++++
 src/graph/graphAudit.ts               | 214 ++++++++++++++++++++++++++
 src/index.ts                          |   4 +
 src/storage/baseStorage.ts            |   8 +
 tests/integration/graph-audit.test.ts | 164 ++++++++++++++++++++
 7 files changed, 522 insertions(+)
 create mode 100644 src/graph/graphAudit.ts
 create mode 100644 tests/integration/graph-audit.test.ts

diff --git a/RELEASES.md b/RELEASES.md
index 540fa6fb..5ae0845d 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,27 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument)
+
+A minor release adding one new public API, from the fleet's graph-trust program: a read-only
+audit that **proves whether relationship reads return stored truth** on a given brain.
+
+- **`brain.auditGraph(options?)`** walks every canonical relationship record, queries the same
+  read path applications use (`related()` / VFS `readdir`) with all visibility tiers included,
+  and classifies every discrepancy into its failure family: `missingFromReads` (records the
+  read path omits — a stale adjacency index), `danglingEndpoints` (relationships whose endpoint
+  entity no longer exists — the historical partial-delete scar class), and `readOnlyVerbIds`
+  (read-path edges with no stored record — ghosts). Design-hidden internal/system edges are
+  counted separately so intentional hiding is never misclassified as loss. Counts are exact;
+  example lists cap at `maxExamples` with an explicit `truncatedExamples` flag; the result is
+  narrated loudly on incoherence. Mutates nothing — safe on a live brain.
+- The operational pairing: audit → if incoherent, `repairIndex()` → audit again. A `coherent`
+  report after the repair is the verified all-clear. Run it after any engine upgrade, restore,
+  or migration. Guide: `docs/guides/inspection.md`.
+- Also: `getNounIds` pagination now refuses an undecodable resume cursor loudly (the same
+  contract `getNouns`/`getVerbs` gained in 8.5.2 — the third and final walk brought under it).
+- Types exported: `GraphAuditReport`, `GraphAuditDiscrepancy`.
+
 ## v8.5.2 — 2026-07-17 (aggregation backfill: exception-safe, generation-verified, and loud)
 
 Hardening patch from a migration incident (a byte-copied store on new hardware; the service
diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md
index 015c5118..240e81ae 100644
--- a/docs/guides/inspection.md
+++ b/docs/guides/inspection.md
@@ -166,6 +166,34 @@ brainy inspect diff /data/brain-prod /data/brain-staging
 Sample-based — for a full diff, dump both with `inspect dump` and compare
 the JSONL.
 
+## Auditing graph-read truth
+
+`brain.auditGraph()` (8.6.0+) proves — or disproves — that relationship reads
+return canonical truth on a given brain, without mutating anything. It walks
+every stored relationship record, asks the same read path your application
+uses (`related()`, VFS `readdir`) with every visibility tier included, and
+classifies every discrepancy:
+
+```typescript
+const report = await brain.auditGraph()
+
+report.coherent              // true = related()/readdir can be trusted on this brain
+report.missingFromReadsCount // records the read path omits — stale index
+report.danglingEndpointsCount // relationships whose endpoint entity is gone
+report.readOnlyCount         // read-path edges with NO stored record — ghosts
+report.visibilityHiddenCount // internal/system edges hidden by design (not a fault)
+```
+
+Counts are always exact; the example lists (`missingFromReads`,
+`danglingEndpoints`, `readOnlyVerbIds`) are capped at `maxExamples`
+(default 100) and `truncatedExamples` says so when they are.
+
+Run it after any engine upgrade, restore, or migration. If it reports
+discrepancies, run `brain.repairIndex()` and audit again — a `coherent`
+report after the repair is the verified statement that the heal worked.
+Cost: one relationship-record walk plus one indexed read per distinct
+source entity — safe on a live brain.
+
 ## Repairing a corrupted store
 
 If invariants fail and you suspect index corruption, `inspect repair`
diff --git a/src/brainy.ts b/src/brainy.ts
index bc5cedd5..12a43086 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -46,6 +46,7 @@ import {
   pageRank,
   MinHeap
 } from './graph/analyticsFallback.js'
+import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js'
 import { createPipeline } from './streaming/pipeline.js'
 import { configureLogger, LogLevel, prodLog } from './utils/logger.js'
 import { setGlobalCache } from './utils/unifiedCache.js'
@@ -15489,6 +15490,88 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     )
   }
 
+  /**
+   * Read-only graph-truth audit — proves (or disproves) that relationship
+   * reads return canonical truth on THIS brain, and classifies every
+   * discrepancy into its failure family:
+   *
+   * - `missingFromReads` — canonical verb records the read path omits
+   *   (PRESENT BUT INVISIBLE: adjacency/membership staleness)
+   * - `danglingEndpoints` — verbs whose endpoint entity is gone (SCAR class)
+   * - `readOnlyVerbIds` — read-path edges with no canonical record (GHOSTS)
+   *
+   * Design-hidden edges (internal/system visibility) are counted separately —
+   * the audit reads with every visibility tier included, so intentional
+   * hiding is never misclassified as index loss. Mutates nothing; safe on a
+   * live brain (cost: one canonical walk + one indexed read per source).
+   * Run it after any engine upgrade, restore, or migration; a `coherent`
+   * report is the verified statement that `related()`/`readdir` can be
+   * trusted. If it reports discrepancies, `repairIndex()` is the sanctioned
+   * heal — re-run the audit afterwards to prove the repair.
+   *
+   * @param options.maxExamples - Cap per example list (counts stay exact). Default 100.
+   * @returns The full audit report; also narrated via logs (loud on incoherence).
+   * @since 8.6.0
+   */
+  async auditGraph(options: { maxExamples?: number } = {}): Promise<GraphAuditReport> {
+    await this.ensureInitialized({ needs: ['graph'] })
+
+    const PAGE = 1000
+    return runGraphAudit(
+      {
+        eachNounId: async (consume) => {
+          let cursor: string | undefined
+          let offset = 0
+          for (;;) {
+            const page = await (this.storage as unknown as {
+              getNounIdsWithPagination(o: {
+                limit: number
+                offset?: number
+                cursor?: string
+              }): Promise<{ ids: string[]; hasMore: boolean; nextCursor?: string }>
+            }).getNounIdsWithPagination(
+              cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
+            )
+            for (const id of page.ids) consume(id)
+            if (!page.hasMore || page.ids.length === 0) break
+            if (page.nextCursor) cursor = page.nextCursor
+            else offset += page.ids.length
+          }
+        },
+        eachVerb: async (consume) => {
+          let cursor: string | undefined
+          let offset = 0
+          for (;;) {
+            const page = await this.storage.getVerbs({
+              pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
+            })
+            for (const verb of page.items) {
+              const v = verb as unknown as Record<string, unknown>
+              consume({
+                id: String(v.id),
+                type: String(v.verb ?? 'unknown'),
+                sourceId: String(v.sourceId),
+                targetId: String(v.targetId),
+                visibility: typeof v.visibility === 'string' ? v.visibility : undefined
+              })
+            }
+            if (!page.hasMore || page.items.length === 0) break
+            if (page.nextCursor) cursor = page.nextCursor
+            else offset += page.items.length
+          }
+        },
+        readRelationsFrom: async (sourceId) =>
+          this.related({
+            from: sourceId,
+            includeInternal: true,
+            includeSystem: true,
+            limit: 100000
+          })
+      },
+      options
+    )
+  }
+
   async repairIndex(): Promise<void> {
     await this.ensureInitialized()
 
diff --git a/src/graph/graphAudit.ts b/src/graph/graphAudit.ts
new file mode 100644
index 00000000..d0e44cd9
--- /dev/null
+++ b/src/graph/graphAudit.ts
@@ -0,0 +1,214 @@
+/**
+ * @module graph/graphAudit
+ * @description Read-only graph-truth audit — the graph sibling of `repairIndex()`'s
+ * diagnosis half. Verifies three layers against each other without mutating anything:
+ *
+ *   1. CANONICAL verb records (the storage walk — the source of truth)
+ *   2. the RELATIONSHIP READ PATH (`related()` with all visibility tiers — exactly
+ *      what application reads like a VFS `readdir` consult)
+ *   3. ENTITY ENDPOINTS (does each verb's source/target still exist?)
+ *
+ * and classifies every discrepancy into the three failure families production
+ * incidents have shown:
+ *
+ *   - `missingFromReads` — a canonical verb record the read path does NOT return
+ *     for its source: PRESENT BUT INVISIBLE (adjacency/membership staleness).
+ *   - `danglingEndpoints` — a canonical verb whose endpoint entity is gone:
+ *     the SCAR class (write-path loss / partial delete).
+ *   - `readOnlyVerbIds` — the read path returns an edge with NO canonical
+ *     record: GHOST edges (stale index entries).
+ *
+ * `visibilityHiddenCount` is reported separately: an internal/system edge that is
+ * indexed and present but hidden from DEFAULT reads is working as designed — the
+ * audit reads with all tiers included so design-hiding is never misclassified as
+ * index loss.
+ *
+ * Full counts are always exact; only the example LISTS are capped (`maxExamples`)
+ * — a capped report says so via `truncatedExamples`, never silently.
+ */
+
+import { prodLog } from '../utils/logger.js'
+
+/** One discrepant relationship, identified fully enough to inspect by hand. */
+export interface GraphAuditDiscrepancy {
+  verbId: string
+  from: string
+  to: string
+  type: string
+}
+
+export interface GraphAuditReport {
+  /** True iff every discrepancy count is zero — `related()` returns canonical truth. */
+  coherent: boolean
+  verbsInCanonical: number
+  entitiesInCanonical: number
+  /** Distinct source entities whose read path was actually consulted (coverage honesty). */
+  sourcesChecked: number
+
+  /** PRESENT BUT INVISIBLE: canonical records the read path omits. */
+  missingFromReadsCount: number
+  missingFromReads: GraphAuditDiscrepancy[]
+
+  /** SCAR CLASS: canonical verbs with a missing endpoint entity. */
+  danglingEndpointsCount: number
+  danglingEndpoints: Array<GraphAuditDiscrepancy & { missingEnd: 'from' | 'to' | 'both' }>
+
+  /** GHOST EDGES: read-path verb ids with no canonical record. */
+  readOnlyCount: number
+  readOnlyVerbIds: string[]
+
+  /** Canonical verbs hidden from DEFAULT reads by design (internal/system visibility). */
+  visibilityHiddenCount: number
+
+  /** Example lists above were capped at maxExamples; counts remain exact. */
+  truncatedExamples: boolean
+  durationMs: number
+}
+
+/** A canonical verb record, as the audit needs it. */
+export interface AuditVerbRecord {
+  id: string
+  type: string
+  sourceId: string
+  targetId: string
+  visibility?: string
+}
+
+/** The seams the audit runs over — injected so the walk is testable in isolation. */
+export interface GraphAuditDeps {
+  /** Stream every canonical entity id (id-only; no per-entity reads needed). */
+  eachNounId(consume: (id: string) => void): Promise<void>
+  /** Stream every canonical verb record. */
+  eachVerb(consume: (verb: AuditVerbRecord) => void): Promise<void>
+  /**
+   * The END-TO-END relationship read for one source, ALL visibility tiers
+   * included — must be the same path application reads consult.
+   */
+  readRelationsFrom(sourceId: string): Promise<Array<{ id: string }>>
+}
+
+export interface GraphAuditOptions {
+  /** Cap on entries per example list (counts stay exact). Default 100. */
+  maxExamples?: number
+}
+
+export async function runGraphAudit(
+  deps: GraphAuditDeps,
+  options: GraphAuditOptions = {}
+): Promise<GraphAuditReport> {
+  const maxExamples = options.maxExamples ?? 100
+  const started = Date.now()
+
+  // 1. Canonical entity ids — endpoint existence oracle.
+  const entityIds = new Set<string>()
+  await deps.eachNounId((id) => entityIds.add(id))
+
+  // 2. Canonical verb walk: group by source, check endpoints, note visibility.
+  const canonicalVerbIds = new Set<string>()
+  const bySource = new Map<string, AuditVerbRecord[]>()
+  let verbsInCanonical = 0
+  let visibilityHiddenCount = 0
+  let danglingEndpointsCount = 0
+  const danglingEndpoints: GraphAuditReport['danglingEndpoints'] = []
+
+  await deps.eachVerb((verb) => {
+    verbsInCanonical++
+    canonicalVerbIds.add(verb.id)
+    const list = bySource.get(verb.sourceId)
+    if (list) list.push(verb)
+    else bySource.set(verb.sourceId, [verb])
+
+    if (verb.visibility === 'internal' || verb.visibility === 'system') {
+      visibilityHiddenCount++
+    }
+
+    const fromMissing = !entityIds.has(verb.sourceId)
+    const toMissing = !entityIds.has(verb.targetId)
+    if (fromMissing || toMissing) {
+      danglingEndpointsCount++
+      if (danglingEndpoints.length < maxExamples) {
+        danglingEndpoints.push({
+          verbId: verb.id,
+          from: verb.sourceId,
+          to: verb.targetId,
+          type: verb.type,
+          missingEnd: fromMissing && toMissing ? 'both' : fromMissing ? 'from' : 'to'
+        })
+      }
+    }
+  })
+
+  // 3. Per-source read-path comparison. A verb must be returned by the read
+  //    path of ITS OWN source — the exact consult a readdir/traversal makes.
+  let missingFromReadsCount = 0
+  const missingFromReads: GraphAuditDiscrepancy[] = []
+  let readOnlyCount = 0
+  const readOnlyVerbIds: string[] = []
+  const readOnlySeen = new Set<string>()
+
+  for (const [sourceId, verbs] of bySource) {
+    const readIds = new Set((await deps.readRelationsFrom(sourceId)).map((r) => r.id))
+
+    for (const verb of verbs) {
+      if (!readIds.has(verb.id)) {
+        missingFromReadsCount++
+        if (missingFromReads.length < maxExamples) {
+          missingFromReads.push({
+            verbId: verb.id,
+            from: verb.sourceId,
+            to: verb.targetId,
+            type: verb.type
+          })
+        }
+      }
+    }
+
+    for (const readId of readIds) {
+      if (!canonicalVerbIds.has(readId) && !readOnlySeen.has(readId)) {
+        readOnlySeen.add(readId)
+        readOnlyCount++
+        if (readOnlyVerbIds.length < maxExamples) {
+          readOnlyVerbIds.push(readId)
+        }
+      }
+    }
+  }
+
+  const coherent =
+    missingFromReadsCount === 0 && danglingEndpointsCount === 0 && readOnlyCount === 0
+
+  const report: GraphAuditReport = {
+    coherent,
+    verbsInCanonical,
+    entitiesInCanonical: entityIds.size,
+    sourcesChecked: bySource.size,
+    missingFromReadsCount,
+    missingFromReads,
+    danglingEndpointsCount,
+    danglingEndpoints,
+    readOnlyCount,
+    readOnlyVerbIds,
+    visibilityHiddenCount,
+    truncatedExamples:
+      missingFromReadsCount > missingFromReads.length ||
+      danglingEndpointsCount > danglingEndpoints.length ||
+      readOnlyCount > readOnlyVerbIds.length,
+    durationMs: Date.now() - started
+  }
+
+  if (coherent) {
+    prodLog.info(
+      `[GraphAudit] coherent: ${verbsInCanonical} verbs across ${bySource.size} sources — ` +
+        `the read path returns canonical truth (${report.durationMs}ms)`
+    )
+  } else {
+    prodLog.warn(
+      `[GraphAudit] INCOHERENT: ${missingFromReadsCount} present-but-invisible, ` +
+        `${danglingEndpointsCount} dangling-endpoint, ${readOnlyCount} ghost ` +
+        `(of ${verbsInCanonical} canonical verbs, ${bySource.size} sources, ` +
+        `${visibilityHiddenCount} visibility-hidden by design) — ${report.durationMs}ms`
+    )
+  }
+
+  return report
+}
diff --git a/src/index.ts b/src/index.ts
index bdb5ff73..d1eab40a 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -28,6 +28,10 @@ export type { FileVersion } from './vfs/types.js'
 
 // Export diagnostics result type
 export type { DiagnosticsResult } from './brainy.js'
+export type {
+  GraphAuditReport,
+  GraphAuditDiscrepancy
+} from './graph/graphAudit.js'
 
 // Export Brainy configuration and types
 export type {
diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts
index 7ad00dab..6daf09c0 100644
--- a/src/storage/baseStorage.ts
+++ b/src/storage/baseStorage.ts
@@ -2230,6 +2230,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
     const { limit, offset = 0, filter } = options
     const cursor = this.decodeNounWalkCursor(options.cursor)
+    if (options.cursor && cursor === null) {
+      // Same law as getNouns/getVerbs: an undecodable resume token FAILS
+      // instead of silently restarting the walk at offset 0.
+      throw BrainyError.storage(
+        `getNounIds: invalid pagination cursor '${options.cursor}' — cannot resume this walk. ` +
+          `Restart it without a cursor.`
+      )
+    }
     const collected: Array<{ id: string; shard: number }> = []
     const peekCount = cursor ? limit + 1 : offset + limit + 1
     const startShard = cursor ? cursor.shard : 0
diff --git a/tests/integration/graph-audit.test.ts b/tests/integration/graph-audit.test.ts
new file mode 100644
index 00000000..74fcc96a
--- /dev/null
+++ b/tests/integration/graph-audit.test.ts
@@ -0,0 +1,164 @@
+/**
+ * @module tests/integration/graph-audit
+ * @description brain.auditGraph() — the read-only graph-truth instrument.
+ * Laws under test:
+ * (1) a healthy brain audits COHERENT: every canonical verb is returned by the
+ *     read path of its source, endpoints exist, no ghosts;
+ * (2) design-hidden edges (internal/system visibility) are counted separately
+ *     and never misclassified as index loss;
+ * (3) a verb whose endpoint entity was destroyed at the storage layer (the
+ *     scar class) is flagged as a dangling endpoint, loudly;
+ * (4) the classification core flags present-but-invisible and ghost edges
+ *     exactly (exercised via injected seams — manufacturing a genuinely stale
+ *     adjacency index end-to-end would require corrupting internals the
+ *     public API rightly refuses to corrupt).
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+import { NounType, VerbType } from '../../src/types/graphTypes.js'
+import { runGraphAudit, type AuditVerbRecord } from '../../src/graph/graphAudit.js'
+
+describe('brain.auditGraph() — graph-truth audit', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-graph-audit-'))
+    brain = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true
+    })
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  async function seedGraph(): Promise<{ ids: string[]; verbIds: string[] }> {
+    const ids: string[] = []
+    for (let i = 0; i < 5; i++) {
+      ids.push(
+        await brain.add({
+          data: `entity ${i}`,
+          type: NounType.Concept,
+          metadata: { n: i }
+        })
+      )
+    }
+    const verbIds: string[] = []
+    verbIds.push(await brain.relate({ from: ids[0], to: ids[1], type: VerbType.RelatedTo }))
+    verbIds.push(await brain.relate({ from: ids[0], to: ids[2], type: VerbType.Contains }))
+    verbIds.push(await brain.relate({ from: ids[1], to: ids[3], type: VerbType.DependsOn }))
+    verbIds.push(await brain.relate({ from: ids[3], to: ids[4], type: VerbType.RelatedTo }))
+    return { ids, verbIds }
+  }
+
+  it('audits a healthy brain as coherent, with exact counts', async () => {
+    const { ids } = await seedGraph()
+    const report = await brain.auditGraph()
+
+    expect(report.coherent).toBe(true)
+    expect(report.verbsInCanonical).toBe(4)
+    expect(report.entitiesInCanonical).toBeGreaterThanOrEqual(ids.length) // VFS root etc. may add system nouns
+    expect(report.sourcesChecked).toBe(3) // ids[0], ids[1], ids[3]
+    expect(report.missingFromReadsCount).toBe(0)
+    expect(report.danglingEndpointsCount).toBe(0)
+    expect(report.readOnlyCount).toBe(0)
+    expect(report.truncatedExamples).toBe(false)
+  })
+
+  it('counts design-hidden edges separately and stays coherent', async () => {
+    const { ids } = await seedGraph()
+    await brain.relate({
+      from: ids[2],
+      to: ids[4],
+      type: VerbType.RelatedTo,
+      visibility: 'internal'
+    })
+
+    const report = await brain.auditGraph()
+    expect(report.coherent).toBe(true) // hidden-by-design is NOT a discrepancy
+    expect(report.verbsInCanonical).toBe(5)
+    expect(report.visibilityHiddenCount).toBeGreaterThanOrEqual(1)
+  })
+
+  it('flags a destroyed endpoint as a dangling verb (the scar class)', async () => {
+    const { ids } = await seedGraph()
+    // Destroy ids[4] at the STORAGE layer (bypassing remove(), which would
+    // also delete its verbs) — the historical partial-delete scar shape.
+    await brain.storage.deleteNoun(ids[4])
+
+    const report = await brain.auditGraph()
+    expect(report.coherent).toBe(false)
+    expect(report.danglingEndpointsCount).toBe(1)
+    expect(report.danglingEndpoints[0].to).toBe(ids[4])
+    expect(report.danglingEndpoints[0].missingEnd).toBe('to')
+  })
+})
+
+describe('runGraphAudit classification core (injected seams)', () => {
+  const verb = (id: string, from: string, to: string): AuditVerbRecord => ({
+    id,
+    type: 'relatedTo',
+    sourceId: from,
+    targetId: to
+  })
+
+  const deps = (opts: {
+    nouns: string[]
+    verbs: AuditVerbRecord[]
+    reads: Record<string, string[]> // sourceId -> verb ids the read path returns
+  }) => ({
+    eachNounId: async (consume: (id: string) => void) => {
+      for (const id of opts.nouns) consume(id)
+    },
+    eachVerb: async (consume: (v: AuditVerbRecord) => void) => {
+      for (const v of opts.verbs) consume(v)
+    },
+    readRelationsFrom: async (sourceId: string) =>
+      (opts.reads[sourceId] ?? []).map((id) => ({ id }))
+  })
+
+  it('flags a canonical verb the read path omits — present but invisible', async () => {
+    const report = await runGraphAudit(
+      deps({
+        nouns: ['A', 'B', 'C'],
+        verbs: [verb('v1', 'A', 'B'), verb('v2', 'A', 'C')],
+        reads: { A: ['v1'] } // v2 exists canonically but reads miss it
+      })
+    )
+    expect(report.coherent).toBe(false)
+    expect(report.missingFromReadsCount).toBe(1)
+    expect(report.missingFromReads[0].verbId).toBe('v2')
+  })
+
+  it('flags a read-path edge with no canonical record — a ghost', async () => {
+    const report = await runGraphAudit(
+      deps({
+        nouns: ['A', 'B'],
+        verbs: [verb('v1', 'A', 'B')],
+        reads: { A: ['v1', 'ghost-9'] }
+      })
+    )
+    expect(report.coherent).toBe(false)
+    expect(report.readOnlyCount).toBe(1)
+    expect(report.readOnlyVerbIds).toEqual(['ghost-9'])
+  })
+
+  it('caps example lists but keeps counts exact, and says so', async () => {
+    const verbs = Array.from({ length: 10 }, (_, i) => verb(`v${i}`, 'A', 'B'))
+    const report = await runGraphAudit(
+      deps({ nouns: ['A', 'B'], verbs, reads: { A: [] } }),
+      { maxExamples: 3 }
+    )
+    expect(report.missingFromReadsCount).toBe(10)
+    expect(report.missingFromReads.length).toBe(3)
+    expect(report.truncatedExamples).toBe(true)
+  })
+})

From e0f6e7722f888b6a3711e3868a23074d3df27552 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 16:38:18 -0700
Subject: [PATCH 63/82] chore(release): 8.6.0

---
 CHANGELOG.md      | 5 +++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 75d3cb6f..e89af335 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17)
+
+- feat: brain.auditGraph() — read-only graph-truth audit (2a03fae)
+
+
 ### [8.5.2](https://github.com/soulcraftlabs/brainy/compare/v8.5.1...v8.5.2) (2026-07-17)
 
 - fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards (a77b064)
diff --git a/package-lock.json b/package-lock.json
index 6b12754f..71cba0fe 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.5.2",
+  "version": "8.6.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.5.2",
+      "version": "8.6.0",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index f2004a57..474e41dd 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.5.2",
+  "version": "8.6.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 6ef9fcb7a248453ba1e1a9f1107400368d92a3a5 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 16:49:38 -0700
Subject: [PATCH 64/82] feat: scaled transact budgets + labeled timeout
 diagnostics + envelope docs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- The transact apply budget scales with the batch — max(30s, opCount x 2s) —
  or is exactly the caller's new TransactOptions.timeoutMs. A flat 30s cap
  silently limited honest bulk work to ~15 operations on network-attached
  storage (~2s/op measured in a production import) while looking generous for
  small batches. Internal batch paths (removeMany chunks) get the same
  scaling via the shared transactTimeoutBudget helper.
- TransactionTimeoutError now reports the operation it stopped at as i/N with
  the operation's name, elapsed vs budgeted time, and states the batch rolled
  back atomically and is retryable; context carries the fields
  programmatically.
- The transact operational envelope is documented (optimistic-concurrency
  guide): budget math, chunking with ifAbsent idempotency, when to reach for
  addMany vs transact, and the precompute pattern (embedBatch + per-op
  vector) that keeps inference out of the commit path.
- Embedding claims made honest per an end-to-end probe of the built dist:
  batch and single embedding paths produce bit-identical vectors and a
  vector-supplied add is fully searchable, but batch throughput on the
  default WASM engine measures comparable to sequential (~160ms/text) — the
  5-10x speedup claim in the addMany JSDoc is replaced with the measured
  reality and the actual win (inference outside the budgeted write path).
---
 RELEASES.md                                   | 22 ++++++++++
 docs/guides/optimistic-concurrency.md         | 32 ++++++++++++++
 src/brainy.ts                                 | 41 +++++++++++++-----
 src/db/types.ts                               | 10 +++++
 src/transaction/Transaction.ts                | 24 ++++++++++-
 src/transaction/errors.ts                     | 19 +++++++--
 .../TransactionManager.unit.test.ts           | 42 +++++++++++++++++++
 7 files changed, 175 insertions(+), 15 deletions(-)

diff --git a/RELEASES.md b/RELEASES.md
index 5ae0845d..94ce16ac 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,28 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry)
+
+The bulk-import ergonomics release, from a consumer's measured production incident (a serial
+import on network-attached storage at ~2 s/op met a flat 30 s transaction budget):
+
+- **The transact apply budget now scales with the batch** — `max(30 s, opCount × 2 s)` — or
+  is exactly what you pass as the new `TransactOptions.timeoutMs`. A flat 30 s cap silently
+  limited honest bulk work to ~15 operations on slow disks while looking generous for small
+  batches. Internal batch paths (e.g. `removeMany` chunks) get the same scaling.
+- **`TransactionTimeoutError` is a diagnosis, not just a failure**: it now reports the
+  operation it stopped at as `i/N` with the operation's name, elapsed vs budgeted time, and
+  states the batch rolled back atomically and is retryable. Its `context` carries the same
+  fields programmatically.
+- **The transact envelope is documented** — batch sizing, budget math, chunking with
+  `ifAbsent` idempotency, and the precompute pattern (`embedBatch` + per-op `vector`) that
+  keeps model inference out of the commit path. Guide: `docs/guides/optimistic-concurrency.md`.
+- Note: `brain.embed()` / `brain.embedBatch()` (the precompute APIs) already ship — public,
+  with native-provider passthrough, verified end-to-end (batch and single paths produce
+  bit-identical vectors; a vector-supplied `add` is fully searchable). Honest measurement:
+  on the default WASM engine, batch throughput ≈ sequential (~160 ms/text) — the precompute
+  win is keeping inference out of the budgeted commit path, not raw embedding speed.
+
 ## v8.6.0 — 2026-07-17 (brain.auditGraph — the graph-truth verification instrument)
 
 A minor release adding one new public API, from the fleet's graph-trust program: a read-only
diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md
index a21a8af0..268bc5fa 100644
--- a/docs/guides/optimistic-concurrency.md
+++ b/docs/guides/optimistic-concurrency.md
@@ -180,3 +180,35 @@ Brainy 8.0 has exactly two write-coordination counters, at two granularities:
 They compose: a `transact()` batch can carry per-entity `ifRev` checks *and* a whole-store `ifAtGeneration`; any failed check rejects the entire batch before anything is staged. Generations also power snapshots and time travel (`brain.now()`, `brain.asOf()`, `db.persist()`) — see the [consistency model](../concepts/consistency-model.md) and [Snapshots & Time Travel](./snapshots-and-time-travel.md).
 
 A snapshot or historical view captures each entity *including* its `_rev` at that moment, so reading the past and writing back with `ifRev` against the live state works exactly as you'd hope: the write fails if the entity moved since the state you copied from.
+
+## The transact envelope: batch size, budget, and bulk imports
+
+`transact()` applies its batch atomically under one commit — which means the whole batch
+shares one **apply budget**. Since 8.7.0 the budget scales with the batch:
+`max(30 s, opCount × 2 s)`, or exactly what you pass as `timeoutMs`. A tripped budget rolls
+the entire batch back (nothing partial survives) and throws a retryable
+`TransactionTimeoutError` that names the operation it stopped at, the batch size, and the
+elapsed vs budgeted time — a diagnosis, not just a failure:
+
+```
+Transaction timed out at operation 41/120 ('add') — 246012ms elapsed, budget 240000ms.
+The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.
+```
+
+Practical envelope guidance for bulk work:
+
+1. **Precompute embeddings outside the commit path.** Embedding inside `transact()` spends
+   the budget on model inference. Use `brain.embedBatch(texts)` and pass each vector via
+   the op's `vector` field — the commit then pays only storage costs, and a retried batch
+   never re-pays inference. (The win is *where* the inference happens, not raw embedding
+   throughput: on the default WASM engine, batch and sequential embedding measure
+   comparably, ~160 ms/text; native embedding providers may batch faster.)
+2. **Chunk very large imports** into batches of a few hundred ops with one `transact()`
+   each. You lose whole-import atomicity but keep per-chunk atomicity, bounded memory, and
+   resumability — pair with `ifAbsent` upserts so a retried chunk is idempotent.
+3. **Slow disks change the math, not the contract.** On network-attached storage a single
+   op can cost ~2 s (canonical write + fsync + index maintenance). The scaled default
+   absorbs that; pass an explicit `timeoutMs` only when you know better than the scale.
+4. **`addMany`/`relateMany` are the convenience tier** — they chunk and batch-embed for
+   you, with per-item error reporting instead of batch atomicity. Choose by what you need:
+   atomic-all-or-nothing → `transact()`; resilient bulk load → `addMany`.
diff --git a/src/brainy.ts b/src/brainy.ts
index 12a43086..25c91d0f 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -71,6 +71,7 @@ import type {
 } from './plugin.js'
 import { ConnectionsCodec } from './hnsw/connectionsCodec.js'
 import { TransactionManager } from './transaction/TransactionManager.js'
+import { transactTimeoutBudget } from './transaction/Transaction.js'
 import { RevisionConflictError } from './transaction/RevisionConflictError.js'
 import { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js'
 import {
@@ -1752,7 +1753,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         captureAndCheck({ nouns, verbs } as CommitBeforeImages)
       }
       await this.generationStore.runWithoutGeneration(() =>
-        this.transactionManager.executeTransaction(run)
+        this.transactionManager.executeTransaction(run, {
+          timeout: transactTimeoutBudget(
+            (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0)
+          )
+        })
       )
       const timestamp = Date.now()
       // Bootstrap writes are not generation-stamped; emit without one.
@@ -1764,7 +1769,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       receipt = await this.generationStore.commitSingleOp({
         touched,
         precommit: captureAndCheck,
-        execute: () => this.transactionManager.executeTransaction(run)
+        execute: () =>
+          this.transactionManager.executeTransaction(run, {
+            timeout: transactTimeoutBudget(
+              (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0)
+            )
+          })
       })
     } catch (err) {
       // A failed rollback that left the store inconsistent (a remove/update
@@ -6655,10 +6665,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   /**
    * Add multiple entities in a single batch operation
    *
-   * Uses batch embedding (embedBatch) to pre-compute all vectors in a single
-   * WASM forward pass instead of N individual embed() calls, providing 5-10x
-   * speedup on bulk inserts. Automatically adapts batch size and parallelism
-   * to the storage adapter (e.g., smaller batches for cloud storage).
+   * Uses batch embedding (embedBatch) to pre-compute all vectors before any
+   * storage write, keeping model inference out of the per-item write path.
+   * (On the default WASM engine, batch throughput is comparable to sequential
+   * embed() calls — measured ~160 ms/text either way; a native embedding
+   * provider may batch faster.) Automatically adapts batch size and
+   * parallelism to the storage adapter (e.g., smaller batches for cloud
+   * storage).
    *
    * @param params - Batch add parameters
    * @param params.items - Array of AddParams (same shape as brain.add())
@@ -7797,11 +7810,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         ifAtGeneration: options?.ifAtGeneration,
         precommit: casPrecommit,
         execute: async () => {
-          await this.transactionManager.executeTransaction(async (tx) => {
-            for (const operation of plan.operations) {
-              tx.addOperation(operation)
-            }
-          })
+          await this.transactionManager.executeTransaction(
+            async (tx) => {
+              for (const operation of plan.operations) {
+                tx.addOperation(operation)
+              }
+            },
+            // Budget scales with the batch (or the caller's explicit
+            // timeoutMs): a flat 30s cap silently limited honest bulk work
+            // to ~15 ops on network disks (~2s/op measured in the field).
+            { timeout: transactTimeoutBudget(plan.operations.length, options?.timeoutMs) }
+          )
         }
       }))
     } catch (err) {
diff --git a/src/db/types.ts b/src/db/types.ts
index 2bfd6ac2..d1355dab 100644
--- a/src/db/types.ts
+++ b/src/db/types.ts
@@ -116,6 +116,16 @@ export interface TransactOptions {
    * record is staged.
    */
   ifAtGeneration?: number
+  /**
+   * Budget (ms) for the atomic apply phase. When omitted, the budget SCALES
+   * with the batch: `max(30 000, opCount × 2 000)` — production imports on
+   * network-attached disks measure ~2 s per operation, so a flat 30 s budget
+   * silently capped honest bulk work at ~15 operations. A tripped budget
+   * rolls the whole batch back and throws a retryable
+   * `TransactionTimeoutError` naming the operation it stopped at, the batch
+   * size, and the elapsed/budget times.
+   */
+  timeoutMs?: number
 }
 
 /**
diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts
index 75dccc05..092a2a25 100644
--- a/src/transaction/Transaction.ts
+++ b/src/transaction/Transaction.ts
@@ -37,6 +37,24 @@ const DEFAULT_OPTIONS: Required<TransactionOptions> = {
   maxRollbackRetries: 3
 }
 
+/**
+ * The apply-phase budget for a batch of `opCount` operations.
+ *
+ * An explicit override wins untouched. Otherwise the budget SCALES with the
+ * batch: `max(30 000 ms, opCount × 2 000 ms)`. The per-op term is calibrated
+ * from field data — bulk imports on network-attached disks measure ~2 s per
+ * operation (each op pays canonical writes + fsync + index maintenance) — so
+ * a flat 30 s budget silently capped honest work at ~15 operations while
+ * looking generous for small batches. Scaling keeps small transacts
+ * fast-failing and gives bulk ones a budget proportional to the work they
+ * actually asked for; a trip still rolls back atomically and throws a
+ * retryable, fully-labeled TransactionTimeoutError.
+ */
+export function transactTimeoutBudget(opCount: number, override?: number): number {
+  if (override !== undefined) return override
+  return Math.max(30_000, opCount * 2_000)
+}
+
 /**
  * Transaction class
  */
@@ -114,7 +132,11 @@ export class Transaction implements TransactionContext {
         // into the catch below and rolls back like any other failure — it must
         // never bypass rollback.
         if (Date.now() - this.startTime > this.options.timeout) {
-          throw new TransactionTimeoutError(this.options.timeout, i)
+          throw new TransactionTimeoutError(this.options.timeout, i, {
+            elapsedMs: Date.now() - this.startTime,
+            totalOperations: this.operations.length,
+            operationName: this.operations[i]?.name
+          })
         }
 
         const operation = this.operations[i]
diff --git a/src/transaction/errors.ts b/src/transaction/errors.ts
index c9f6eee1..c270d0ed 100644
--- a/src/transaction/errors.ts
+++ b/src/transaction/errors.ts
@@ -77,11 +77,24 @@ export class InvalidTransactionStateError extends TransactionError {
 export class TransactionTimeoutError extends TransactionError {
   constructor(
     timeoutMs: number,
-    operationIndex: number
+    operationIndex: number,
+    telemetry?: {
+      elapsedMs?: number
+      totalOperations?: number
+      operationName?: string
+    }
   ) {
+    const progress =
+      telemetry?.totalOperations !== undefined
+        ? `${operationIndex}/${telemetry.totalOperations}`
+        : String(operationIndex)
+    const name = telemetry?.operationName ? ` ('${telemetry.operationName}')` : ''
+    const elapsed =
+      telemetry?.elapsedMs !== undefined ? `${telemetry.elapsedMs}ms elapsed, ` : ''
     super(
-      `Transaction timed out after ${timeoutMs}ms at operation ${operationIndex}`,
-      { timeoutMs, operationIndex }
+      `Transaction timed out at operation ${progress}${name} — ${elapsed}budget ${timeoutMs}ms. ` +
+        `The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.`,
+      { timeoutMs, operationIndex, ...telemetry }
     )
     this.name = 'TransactionTimeoutError'
   }
diff --git a/tests/transaction/TransactionManager.unit.test.ts b/tests/transaction/TransactionManager.unit.test.ts
index cc477483..86b1692c 100644
--- a/tests/transaction/TransactionManager.unit.test.ts
+++ b/tests/transaction/TransactionManager.unit.test.ts
@@ -10,6 +10,7 @@
 
 import { describe, it, expect, beforeEach } from 'vitest'
 import { TransactionManager } from '../../src/transaction/TransactionManager.js'
+import { transactTimeoutBudget } from '../../src/transaction/Transaction.js'
 import { TransactionError } from '../../src/transaction/errors.js'
 
 describe('TransactionManager', () => {
@@ -328,4 +329,45 @@ describe('TransactionManager', () => {
       expect(stats1).toEqual(stats2) // Same values
     })
   })
+
+  describe('Timeout budget + telemetry', () => {
+    it('transactTimeoutBudget: explicit override wins; default scales with batch size', () => {
+      expect(transactTimeoutBudget(1)).toBe(30_000) // small batches keep the 30s floor
+      expect(transactTimeoutBudget(15)).toBe(30_000) // the old flat cap's break-even point
+      expect(transactTimeoutBudget(100)).toBe(200_000) // 100 ops × 2s — bulk gets an honest budget
+      expect(transactTimeoutBudget(1000, 5_000)).toBe(5_000) // caller override is untouched
+    })
+
+    it('a tripped budget rolls back and names the operation, progress, and budget', async () => {
+      const rolledBack: string[] = []
+
+      const failing = manager.executeTransaction(
+        async (tx) => {
+          tx.addOperation({
+            name: 'slow-first-op',
+            execute: async () => {
+              await new Promise((r) => setTimeout(r, 30))
+              return async () => {
+                rolledBack.push('slow-first-op')
+              }
+            }
+          })
+          tx.addOperation({
+            name: 'never-reached',
+            execute: async () => undefined
+          })
+        },
+        { timeout: 5 } // the first op's 30ms sleep guarantees the pre-op-2 check trips
+      )
+
+      await expect(failing).rejects.toThrow(TransactionError)
+      const err = await failing.catch((e) => e)
+      expect(err.name).toBe('TransactionTimeoutError')
+      expect(err.message).toContain('operation 1/2') // which op, of how many
+      expect(err.message).toContain("('never-reached')") // its name
+      expect(err.message).toContain('budget 5ms') // the budget that tripped
+      expect(err.message).toContain('rolled back') // the retryability statement
+      expect(rolledBack).toEqual(['slow-first-op']) // the applied op was undone
+    })
+  })
 })

From e450e0eedfd837e693c7ad74166bdd451638a04e Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 16:54:40 -0700
Subject: [PATCH 65/82] chore(release): 8.7.0

---
 CHANGELOG.md      | 5 +++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e89af335..4d7198e5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17)
+
+- feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb)
+
+
 ### [8.6.0](https://github.com/soulcraftlabs/brainy/compare/v8.5.2...v8.6.0) (2026-07-17)
 
 - feat: brain.auditGraph() — read-only graph-truth audit (2a03fae)
diff --git a/package-lock.json b/package-lock.json
index 71cba0fe..aa9a62a3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.6.0",
+  "version": "8.7.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.6.0",
+      "version": "8.7.0",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 474e41dd..9d016ecb 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.6.0",
+  "version": "8.7.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 01a3b46ade9d6b21931ebc398e6b24913af3f870 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 17:11:46 -0700
Subject: [PATCH 66/82] fix: race-proof writer-lock acquisition +
 machine-readable conflict through init

- acquireWriterLock now CLAIMS with an atomic create-exclusive write (O_EXCL)
  inside a bounded retry loop: two processes racing an absent lock can never
  both succeed (the old read-then-tmp-rename flow let the loser keep running
  unlocked, silently). An EEXIST loser re-evaluates and either throws loudly
  with the winner's details or performs a verified stale-takeover (re-read
  before unlink so a lock that changed hands mid-deliberation is never
  clobbered). Exhausted contention fails loudly instead of degrading into a
  lockless open.
- BRAINY_WRITER_LOCKED passes through init() unwrapped: the error documents a
  machine-readable contract (err.code + err.lockInfo with the holder's
  pid/host/heartbeat), but init's blanket error wrapping stripped both,
  leaving consumers a message to regex against.
- Two contract tests added: stale-foreign takeover installs OUR lock via the
  atomic claim; the conflict error carries code + lockInfo at the public
  init() surface.
---
 RELEASES.md                                   |  20 +++
 src/brainy.ts                                 |   7 +
 src/storage/adapters/fileSystemStorage.ts     | 167 +++++++++++++-----
 .../integration/multi-process-safety.test.ts  |  43 +++++
 4 files changed, 188 insertions(+), 49 deletions(-)

diff --git a/RELEASES.md b/RELEASES.md
index 94ce16ac..a07d9326 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,26 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init)
+
+Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a
+second writer on the same brain directory fail loudly):
+
+- **Lock acquisition claims atomically.** The acquire path used read-then-write, leaving a
+  window where two processes racing an *absent* lock could both "succeed" — and the loser
+  kept running unlocked, silently. The claim is now an atomic create-exclusive write
+  (`O_EXCL`): exactly one racer wins; the loser re-evaluates and either fails loudly with
+  the winner's details or performs a verified stale-takeover. Bounded retries; contention
+  beyond them fails loudly rather than degrading into a lockless open.
+- **`BRAINY_WRITER_LOCKED` survives `init()`.** The conflict error documents a
+  machine-readable contract (`err.code`, `err.lockInfo` with the holder's pid/host/
+  heartbeat), but init's error wrapping silently stripped both, leaving consumers a message
+  to regex against. The error now passes through unwrapped.
+
+Measured while verifying (for operators sizing audits): `brain.auditGraph()` at a
+production-consumer scale of ~2,600 relationships / 800 entities costs ~0.1 s warm and
+~0.5 s cold, with exact scar counting across reopen.
+
 ## v8.7.0 — 2026-07-17 (bulk-transact ergonomics: scaled budgets + timeout telemetry)
 
 The bulk-import ergonomics release, from a consumer's measured production incident (a serial
diff --git a/src/brainy.ts b/src/brainy.ts
index 25c91d0f..7aa7e6ca 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -1373,6 +1373,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       if (this._readyReject) {
         this._readyReject(error instanceof Error ? error : new Error(String(error)))
       }
+      // Machine-readable init failures pass through UNWRAPPED — the writer-lock
+      // conflict documents an err.code/err.lockInfo contract ("callers detect
+      // this case via err.code"), and wrapping in a fresh Error silently
+      // stripped both, leaving consumers only a message to regex against.
+      if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') {
+        throw error
+      }
       throw new Error(`Failed to initialize Brainy: ${error}`)
     }
   }
diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index 4f246f67..3c95f9e7 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -1764,64 +1764,117 @@ export class FileSystemStorage extends BaseStorage {
     const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE)
     const os = await import('node:os')
     const hostname = os.hostname()
-    const now = new Date().toISOString()
+    const myPid = typeof process !== 'undefined' && process.pid ? process.pid : 0
 
-    const existing = await this.readWriterLock()
-    if (existing && !options?.force) {
-      // Same-process re-open: a second Brainy instance in this Node process
-      // (e.g. test "simulate server restart" patterns, or a consumer that
-      // explicitly re-instantiates without closing first). This isn't the
-      // dangerous cross-process case the lock exists to prevent — the two
-      // instances share a memory space and can't silently diverge from each
-      // other beyond what their callers already see. Warn and take over the lock.
-      if (existing.pid === (typeof process !== 'undefined' ? process.pid : 0) &&
-          existing.hostname === hostname) {
-        console.warn(
-          `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
-          `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
-        )
-      } else {
-        const stale = await this.isWriterLockStale(existing)
-        if (!stale) {
+    // Bounded acquire loop. The CLAIM itself is an atomic create-exclusive
+    // write (O_EXCL) — two processes racing an ABSENT lock can never both
+    // succeed, which closes the read-then-write window where the loser used
+    // to keep running unlocked, silently. An EEXIST loser loops, re-reads,
+    // and handles whatever it finds honestly (fresh foreign lock → loud
+    // throw; stale/forced → verified takeover).
+    const MAX_ATTEMPTS = 3
+    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+      const now = new Date().toISOString()
+      const existing = await this.readWriterLock()
+
+      if (existing) {
+        // Same-process re-open: a second Brainy instance in this Node process
+        // (e.g. test "simulate server restart" patterns, or a consumer that
+        // explicitly re-instantiates without closing first). This isn't the
+        // dangerous cross-process case the lock exists to prevent — the two
+        // instances share a memory space and can't silently diverge from each
+        // other beyond what their callers already see. Warn and take over.
+        if (existing.pid === myPid && existing.hostname === hostname && !options?.force) {
+          console.warn(
+            `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` +
+            `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.`
+          )
+          const info: WriterLockInfo = {
+            pid: myPid,
+            hostname,
+            startedAt: now,
+            lastHeartbeat: now,
+            version: getBrainyVersion(),
+            rootDir: this.rootDir
+          }
+          await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
+          this.installWriterLock(info)
+          return info
+        }
+
+        const stale = !options?.force && (await this.isWriterLockStale(existing))
+        if (!options?.force && !stale) {
           // Consumer-facing error contract: callers detect this case via
           // err.code and read the holder's details from err.lockInfo.
-          const err = new Error(
-            `Another writer holds this Brainy directory.\n` +
-            `  PID:        ${existing.pid} on host ${existing.hostname}\n` +
-            `  Started:    ${existing.startedAt}\n` +
-            `  Heartbeat:  ${existing.lastHeartbeat}\n` +
-            `  Version:    ${existing.version}\n` +
-            `  Directory:  ${this.rootDir}\n\n` +
-            `For diagnostic queries against this live store, use:\n` +
-            `  const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` +
-            `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.`
-          ) as Error & { code: string; lockInfo: WriterLockInfo }
-          err.code = 'BRAINY_WRITER_LOCKED'
-          err.lockInfo = existing
-          throw err
+          throw this.writerLockedError(existing)
         }
+
         console.warn(
-          `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
-          `(PID ${existing.pid} on ${existing.hostname} appears dead).`
+          options?.force
+            ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
+              `(was held by PID ${existing.pid} on ${existing.hostname}).`
+            : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` +
+              `(PID ${existing.pid} on ${existing.hostname} appears dead).`
         )
+        // Takeover: verify the file still holds the lock we judged (a live
+        // successor may have claimed meanwhile), then remove it and fall
+        // through to the atomic claim below. A racing claimer who beats us to
+        // the create simply wins — our next loop iteration reads their fresh
+        // lock and throws honestly. (Advisory file locking has no
+        // compare-and-delete; staleness requiring a 60s-old heartbeat keeps
+        // the residual verify-to-unlink window practically unreachable.)
+        const recheck = await this.readWriterLock()
+        if (
+          recheck &&
+          (recheck.pid !== existing.pid ||
+            recheck.startedAt !== existing.startedAt ||
+            recheck.lastHeartbeat !== existing.lastHeartbeat)
+        ) {
+          continue // the lock changed hands while we deliberated — re-evaluate
+        }
+        try {
+          await fs.promises.unlink(lockFile)
+        } catch (err: any) {
+          if (err.code !== 'ENOENT') throw err
+        }
       }
-    } else if (existing && options?.force) {
-      console.warn(
-        `[brainy] Force-overwriting writer lock for ${this.rootDir} ` +
-        `(was held by PID ${existing.pid} on ${existing.hostname}).`
-      )
+
+      const info: WriterLockInfo = {
+        pid: myPid,
+        hostname,
+        startedAt: existing && options?.force ? existing.startedAt : now,
+        lastHeartbeat: now,
+        version: getBrainyVersion(),
+        rootDir: this.rootDir
+      }
+
+      // The atomic claim: create-exclusive, so exactly ONE racer wins.
+      try {
+        await fs.promises.writeFile(lockFile, JSON.stringify(info, null, 2), { flag: 'wx' })
+      } catch (err: any) {
+        if (err.code === 'EEXIST') {
+          continue // someone else claimed between our read and create — re-evaluate
+        }
+        throw err
+      }
+
+      this.installWriterLock(info)
+      return info
     }
 
-    const info: WriterLockInfo = {
-      pid: typeof process !== 'undefined' && process.pid ? process.pid : 0,
-      hostname,
-      startedAt: existing && options?.force ? existing.startedAt : now,
-      lastHeartbeat: now,
-      version: getBrainyVersion(),
-      rootDir: this.rootDir
-    }
+    // Attempts exhausted: something is claiming this directory faster than we
+    // can evaluate it. Read whoever holds it now and fail loudly with their
+    // details rather than degrading into a lockless open.
+    const holder = await this.readWriterLock()
+    if (holder) throw this.writerLockedError(holder)
+    throw new Error(
+      `Failed to acquire the writer lock for ${this.rootDir} after ${MAX_ATTEMPTS} attempts — ` +
+      `the lock file is being contended. Retry, or inspect ${lockFile}.`
+    )
+  }
 
-    await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2))
+  /** Record lock ownership + start the unref'd heartbeat. */
+  private installWriterLock(info: WriterLockInfo): void {
     this.writerLockInfo = info
 
     // Heartbeat — rewrite lastHeartbeat every WRITER_HEARTBEAT_MS so other
@@ -1835,8 +1888,24 @@ export class FileSystemStorage extends BaseStorage {
       // Don't keep the event loop alive just for the heartbeat.
       this.writerLockHeartbeat.unref()
     }
+  }
 
-    return info
+  /** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */
+  private writerLockedError(existing: WriterLockInfo): Error {
+    const err = new Error(
+      `Another writer holds this Brainy directory.\n` +
+      `  PID:        ${existing.pid} on host ${existing.hostname}\n` +
+      `  Started:    ${existing.startedAt}\n` +
+      `  Heartbeat:  ${existing.lastHeartbeat}\n` +
+      `  Version:    ${existing.version}\n` +
+      `  Directory:  ${this.rootDir}\n\n` +
+      `For diagnostic queries against this live store, use:\n` +
+      `  const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` +
+      `If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.`
+    ) as Error & { code: string; lockInfo: WriterLockInfo }
+    err.code = 'BRAINY_WRITER_LOCKED'
+    err.lockInfo = existing
+    return err
   }
 
   public override async releaseWriterLock(): Promise<void> {
diff --git a/tests/integration/multi-process-safety.test.ts b/tests/integration/multi-process-safety.test.ts
index 6ac5265d..0eae92b1 100644
--- a/tests/integration/multi-process-safety.test.ts
+++ b/tests/integration/multi-process-safety.test.ts
@@ -110,6 +110,49 @@ describe('Multi-process safety + read-only mode', () => {
       // Don't track `blocked` for afterEach cleanup since init failed.
     })
 
+    it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => {
+      const { mkdirSync, writeFileSync, readFileSync } = await import('node:fs')
+      const { join } = await import('node:path')
+      const os = await import('node:os')
+      mkdirSync(join(dir, 'locks'), { recursive: true })
+      const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString()
+      writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({
+        pid: 999999999, // no such process — provably dead
+        hostname: os.hostname(),
+        startedAt: tenMinutesAgo,
+        lastHeartbeat: tenMinutesAgo,
+        version: '8.0.0',
+        rootDir: dir
+      }))
+
+      writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
+      await writer.init() // stale takeover must succeed
+
+      const lock = JSON.parse(readFileSync(join(dir, 'locks', '_writer.lock'), 'utf-8'))
+      expect(lock.pid).toBe(process.pid) // the atomic wx claim installed OUR lock
+    })
+
+    it('the writer-locked error carries the machine-readable contract (code + lockInfo)', async () => {
+      const { mkdirSync, writeFileSync } = await import('node:fs')
+      const { join } = await import('node:path')
+      const os = await import('node:os')
+      mkdirSync(join(dir, 'locks'), { recursive: true })
+      const otherPid = (process as any).ppid || 1
+      writeFileSync(join(dir, 'locks', '_writer.lock'), JSON.stringify({
+        pid: otherPid,
+        hostname: os.hostname(),
+        startedAt: new Date().toISOString(),
+        lastHeartbeat: new Date().toISOString(),
+        version: '8.7.0',
+        rootDir: dir
+      }))
+
+      const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
+      const err: any = await blocked.init().catch((e) => e)
+      expect(err.code).toBe('BRAINY_WRITER_LOCKED')
+      expect(err.lockInfo?.pid).toBe(otherPid)
+    })
+
     it('allows a second in-process writer with a warning (same PID)', async () => {
       // Two Brainy instances in the same Node process: not the dangerous
       // cross-process case. Should succeed (with a console warning).

From dcd5036fe93edbed5f46abee3abdd2f8af2e7918 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 17:15:13 -0700
Subject: [PATCH 67/82] chore(release): 8.7.1

---
 CHANGELOG.md      | 5 +++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d7198e5..95103bb6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17)
+
+- fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46)
+
+
 ### [8.7.0](https://github.com/soulcraftlabs/brainy/compare/v8.6.0...v8.7.0) (2026-07-17)
 
 - feat: scaled transact budgets + labeled timeout diagnostics + envelope docs (6ef9fcb)
diff --git a/package-lock.json b/package-lock.json
index aa9a62a3..28b5e736 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.7.0",
+  "version": "8.7.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.7.0",
+      "version": "8.7.1",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 9d016ecb..d7c19216 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.7.0",
+  "version": "8.7.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 16a73b8475c359c47fbb498f77ca832ac17de612 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 17:51:54 -0700
Subject: [PATCH 68/82] feat: OS-limit detection for pool-scale deployments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- New src/utils/osLimits.ts: reads RLIMIT_NOFILE (soft/hard, from
  /proc/self/limits) and vm.max_map_count at open — once per process,
  Linux-only, measurement-only — and warns loudly when either sits below the
  pool-scale floors (soft NOFILE < 65536, max_map_count < 262144), with the
  exact raise commands. On stock defaults the failure otherwise arrives as
  EMFILE or a failed mmap deep inside an index open, long after the cause
  stopped being visible. An unreadable limit produces NO warning — no
  measurement, no claim — so non-Linux platforms stay silent.
- Exported for ops doors: checkOsLimits() returns the full OsLimitsReport;
  floors exported as constants. Wired fire-and-forget in performInit after
  storage init; the check can never affect open.
- Unit tests pin the parser (incl. 'unlimited'), the floor thresholds, the
  null-never-warns rule, and the off-Linux silent path.
---
 RELEASES.md                       |  15 ++++
 src/brainy.ts                     |   8 ++
 src/index.ts                      |   6 ++
 src/utils/osLimits.ts             | 141 ++++++++++++++++++++++++++++++
 tests/unit/utils/osLimits.test.ts |  76 ++++++++++++++++
 5 files changed, 246 insertions(+)
 create mode 100644 src/utils/osLimits.ts
 create mode 100644 tests/unit/utils/osLimits.test.ts

diff --git a/RELEASES.md b/RELEASES.md
index a07d9326..7baa89ec 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,21 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments)
+
+Small minor: brains now detect the two OS limits that bite at pool scale and warn **before**
+the incident instead of during it.
+
+- At open (once per process, Linux-only, measurement-only), Brainy reads `RLIMIT_NOFILE`
+  (soft/hard, from `/proc/self/limits`) and `vm.max_map_count`, and warns loudly when either
+  sits below the pool-scale floors (soft NOFILE < 65 536; max_map_count < 262 144) — with the
+  exact raise commands (`ulimit -n` / `LimitNOFILE=` / `sysctl vm.max_map_count`). On stock
+  defaults the failure otherwise arrives as `EMFILE` or a failed mmap deep inside an index
+  open, long after the real cause stopped being visible. An unreadable limit produces **no**
+  warning — no measurement, no claim (non-Linux platforms stay silent).
+- Exported for ops doors: `checkOsLimits()` returns the full `OsLimitsReport`
+  (values + warnings) programmatically, with the floors exported as constants.
+
 ## v8.7.1 — 2026-07-17 (writer-lock acquisition is race-proof + machine-readable through init)
 
 Two hardenings of the multi-process writer lock (the `locks/_writer.lock` lease that makes a
diff --git a/src/brainy.ts b/src/brainy.ts
index 7aa7e6ca..89e1267a 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -49,6 +49,7 @@ import {
 import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js'
 import { createPipeline } from './streaming/pipeline.js'
 import { configureLogger, LogLevel, prodLog } from './utils/logger.js'
+import { warnOnLowOsLimits } from './utils/osLimits.js'
 import { setGlobalCache } from './utils/unifiedCache.js'
 import type { UnifiedCache } from './utils/unifiedCache.js'
 import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js'
@@ -935,6 +936,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       this.storage = await this.setupStorage()
       await this.storage.init()
 
+      // OS-limit detection (once per process, Linux-only, measurement-only):
+      // warn NOW about RLIMIT_NOFILE / vm.max_map_count values that will bite
+      // at pool scale, instead of letting the operator meet them as EMFILE or
+      // a failed mmap deep inside an index open. Fire-and-forget — the check
+      // never affects open.
+      void warnOnLowOsLimits()
+
       // Acquire the writer lock for filesystem (and other locking-capable) backends.
       // Skipped in reader mode and on backends that don't support multi-process locking.
       // Throws if another live writer holds the directory (unless force: true).
diff --git a/src/index.ts b/src/index.ts
index d1eab40a..5c57fe29 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -32,6 +32,12 @@ export type {
   GraphAuditReport,
   GraphAuditDiscrepancy
 } from './graph/graphAudit.js'
+export {
+  checkOsLimits,
+  NOFILE_POOL_FLOOR,
+  MAX_MAP_COUNT_POOL_FLOOR
+} from './utils/osLimits.js'
+export type { OsLimitsReport } from './utils/osLimits.js'
 
 // Export Brainy configuration and types
 export type {
diff --git a/src/utils/osLimits.ts b/src/utils/osLimits.ts
new file mode 100644
index 00000000..57e1ccf1
--- /dev/null
+++ b/src/utils/osLimits.ts
@@ -0,0 +1,141 @@
+/**
+ * @module utils/osLimits
+ * @description Detect-and-warn for OS resource limits that bite at POOL scale.
+ *
+ * A single brain rarely notices them, but a pool of brains — especially with a
+ * native accelerator memory-mapping many index files per brain — consumes file
+ * descriptors and memory mappings multiplicatively. On stock Linux defaults
+ * (RLIMIT_NOFILE soft 1024, vm.max_map_count 65530) the failure arrives as
+ * EMFILE or a failed mmap deep inside an index open, long after the real cause
+ * (the limit) stopped being visible. This module reads the limits at open and
+ * WARNS ONCE per process with the exact raise commands, so the operator learns
+ * the fix before the incident instead of from it.
+ *
+ * Read-only and Linux-only by construction: both sources are `/proc` files.
+ * On platforms where they are absent the check reports nulls and stays silent —
+ * no limit read means no claim made, never a guessed warning.
+ */
+
+import * as fs from 'node:fs'
+import { prodLog } from './logger.js'
+
+/** Soft-NOFILE floor below which pool-scale use is at EMFILE risk. */
+export const NOFILE_POOL_FLOOR = 65536
+
+/** vm.max_map_count floor below which mmap-heavy native indexes are at risk. */
+export const MAX_MAP_COUNT_POOL_FLOOR = 262144
+
+export interface OsLimitsReport {
+  /** RLIMIT_NOFILE soft limit (null when unreadable; Infinity for 'unlimited'). */
+  nofileSoft: number | null
+  /** RLIMIT_NOFILE hard limit (null when unreadable; Infinity for 'unlimited'). */
+  nofileHard: number | null
+  /** vm.max_map_count (null when unreadable). */
+  maxMapCount: number | null
+  /** Human-actionable warnings for limits below the pool floors. Empty = fine. */
+  warnings: string[]
+}
+
+/**
+ * Parse the `Max open files` row of a `/proc/<pid>/limits` document into
+ * soft/hard values. Returns nulls when the row is absent or malformed.
+ */
+export function parseProcLimits(content: string): { soft: number | null; hard: number | null } {
+  const line = content.split('\n').find((l) => l.startsWith('Max open files'))
+  if (!line) return { soft: null, hard: null }
+  const m = line.match(/^Max open files\s+(\S+)\s+(\S+)/)
+  if (!m) return { soft: null, hard: null }
+  const parse = (v: string): number | null => {
+    if (v === 'unlimited') return Infinity
+    const n = Number.parseInt(v, 10)
+    return Number.isNaN(n) ? null : n
+  }
+  return { soft: parse(m[1]), hard: parse(m[2]) }
+}
+
+/**
+ * Assess readable limits against the pool floors. Pure — feed it any values.
+ * A null (unreadable) limit produces NO warning: no measurement, no claim.
+ */
+export function assessOsLimits(limits: {
+  nofileSoft: number | null
+  nofileHard: number | null
+  maxMapCount: number | null
+}): string[] {
+  const warnings: string[] = []
+
+  if (limits.nofileSoft !== null && limits.nofileSoft < NOFILE_POOL_FLOOR) {
+    const hardNote =
+      limits.nofileHard !== null && limits.nofileHard >= NOFILE_POOL_FLOOR
+        ? ` (the hard limit ${limits.nofileHard === Infinity ? 'unlimited' : limits.nofileHard} already allows it — raise the soft limit only)`
+        : ''
+    warnings.push(
+      `RLIMIT_NOFILE soft limit is ${limits.nofileSoft} — below the ${NOFILE_POOL_FLOOR} recommended ` +
+        `for pool-scale use (a pool of brains with a native accelerator opens many index files per brain; ` +
+        `the failure mode is EMFILE deep inside an index open). Raise with \`ulimit -n ${NOFILE_POOL_FLOOR}\` ` +
+        `or LimitNOFILE=${NOFILE_POOL_FLOOR} in the service unit${hardNote}.`
+    )
+  }
+
+  if (limits.maxMapCount !== null && limits.maxMapCount < MAX_MAP_COUNT_POOL_FLOOR) {
+    warnings.push(
+      `vm.max_map_count is ${limits.maxMapCount} — below the ${MAX_MAP_COUNT_POOL_FLOOR} recommended ` +
+        `for mmap-heavy native indexes at pool scale (each mapped index segment consumes map entries; ` +
+        `the failure mode is a failed mmap mid-heal). Raise with ` +
+        `\`sysctl -w vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}\` (persist in /etc/sysctl.d/).`
+    )
+  }
+
+  return warnings
+}
+
+/**
+ * Read the limits from /proc and assess them. `readFile` is injectable for
+ * tests; absent/unreadable sources yield nulls (and therefore no warnings).
+ */
+export async function checkOsLimits(
+  readFile: (path: string) => Promise<string> = async (p) => fs.promises.readFile(p, 'utf-8')
+): Promise<OsLimitsReport> {
+  let nofileSoft: number | null = null
+  let nofileHard: number | null = null
+  let maxMapCount: number | null = null
+
+  try {
+    const parsed = parseProcLimits(await readFile('/proc/self/limits'))
+    nofileSoft = parsed.soft
+    nofileHard = parsed.hard
+  } catch {
+    // Not Linux (or /proc unavailable) — no measurement, no claim.
+  }
+
+  try {
+    const raw = (await readFile('/proc/sys/vm/max_map_count')).trim()
+    const n = Number.parseInt(raw, 10)
+    maxMapCount = Number.isNaN(n) ? null : n
+  } catch {
+    // Not Linux — same rule.
+  }
+
+  const warnings = assessOsLimits({ nofileSoft, nofileHard, maxMapCount })
+  return { nofileSoft, nofileHard, maxMapCount, warnings }
+}
+
+/** Once-per-process latch so a brain pool warns once, not once per brain. */
+let osLimitsWarned = false
+
+/**
+ * Run the check and warn (once per process) about limits below the pool
+ * floors. Called from brain open; safe everywhere (silent off-Linux).
+ */
+export async function warnOnLowOsLimits(): Promise<void> {
+  if (osLimitsWarned) return
+  osLimitsWarned = true
+  try {
+    const report = await checkOsLimits()
+    for (const warning of report.warnings) {
+      prodLog.warn(`[Brainy] OS limit check: ${warning}`)
+    }
+  } catch {
+    // The check must never affect open — measurement-only.
+  }
+}
diff --git a/tests/unit/utils/osLimits.test.ts b/tests/unit/utils/osLimits.test.ts
new file mode 100644
index 00000000..a56d593e
--- /dev/null
+++ b/tests/unit/utils/osLimits.test.ts
@@ -0,0 +1,76 @@
+/**
+ * @module tests/unit/utils/osLimits
+ * @description OS-limit detection for pool-scale use. Laws:
+ * (1) the /proc/self/limits parser reads soft/hard NOFILE exactly, including
+ *     'unlimited'; (2) assessment warns ONLY below the pool floors and NEVER
+ *     on an unreadable (null) limit — no measurement, no claim; (3) the full
+ *     check composes both sources and survives unreadable /proc silently.
+ */
+import { describe, it, expect } from 'vitest'
+import {
+  parseProcLimits,
+  assessOsLimits,
+  checkOsLimits,
+  NOFILE_POOL_FLOOR,
+  MAX_MAP_COUNT_POOL_FLOOR
+} from '../../../src/utils/osLimits.js'
+
+const SAMPLE_LIMITS = [
+  'Limit                     Soft Limit           Hard Limit           Units',
+  'Max cpu time              unlimited            unlimited            seconds',
+  'Max open files            1024                 1048576              files',
+  'Max locked memory         8388608              8388608              bytes'
+].join('\n')
+
+describe('osLimits — detect + warn at pool scale', () => {
+  it('parses soft/hard NOFILE from /proc/self/limits, including unlimited', () => {
+    expect(parseProcLimits(SAMPLE_LIMITS)).toEqual({ soft: 1024, hard: 1048576 })
+    expect(
+      parseProcLimits('Max open files            unlimited            unlimited            files')
+    ).toEqual({ soft: Infinity, hard: Infinity })
+    expect(parseProcLimits('no such row here')).toEqual({ soft: null, hard: null })
+  })
+
+  it('warns below the floors, stays quiet at or above them', () => {
+    const low = assessOsLimits({ nofileSoft: 1024, nofileHard: 1048576, maxMapCount: 65530 })
+    expect(low).toHaveLength(2)
+    expect(low[0]).toContain('RLIMIT_NOFILE soft limit is 1024')
+    expect(low[0]).toContain(`ulimit -n ${NOFILE_POOL_FLOOR}`)
+    expect(low[0]).toContain('raise the soft limit only') // hard already allows it
+    expect(low[1]).toContain('vm.max_map_count is 65530')
+    expect(low[1]).toContain(`vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}`)
+
+    expect(
+      assessOsLimits({
+        nofileSoft: NOFILE_POOL_FLOOR,
+        nofileHard: Infinity,
+        maxMapCount: MAX_MAP_COUNT_POOL_FLOOR
+      })
+    ).toEqual([])
+  })
+
+  it('an unreadable limit makes NO claim — nulls never warn', () => {
+    expect(assessOsLimits({ nofileSoft: null, nofileHard: null, maxMapCount: null })).toEqual([])
+  })
+
+  it('checkOsLimits composes both sources and survives unreadable /proc silently', async () => {
+    const report = await checkOsLimits(async (p) => {
+      if (p === '/proc/self/limits') return SAMPLE_LIMITS
+      if (p === '/proc/sys/vm/max_map_count') return '65530\n'
+      throw new Error('unexpected path')
+    })
+    expect(report.nofileSoft).toBe(1024)
+    expect(report.maxMapCount).toBe(65530)
+    expect(report.warnings).toHaveLength(2)
+
+    const offLinux = await checkOsLimits(async () => {
+      throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' })
+    })
+    expect(offLinux).toEqual({
+      nofileSoft: null,
+      nofileHard: null,
+      maxMapCount: null,
+      warnings: []
+    })
+  })
+})

From 120205b69c444b3bf79e8879819f67a45c2798e5 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Fri, 17 Jul 2026 18:41:27 -0700
Subject: [PATCH 69/82] chore(release): 8.8.0

---
 CHANGELOG.md      | 5 +++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95103bb6..f97046f0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17)
+
+- feat: OS-limit detection for pool-scale deployments (16a73b8)
+
+
 ### [8.7.1](https://github.com/soulcraftlabs/brainy/compare/v8.7.0...v8.7.1) (2026-07-17)
 
 - fix: race-proof writer-lock acquisition + machine-readable conflict through init (01a3b46)
diff --git a/package-lock.json b/package-lock.json
index 28b5e736..198e9116 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.7.1",
+  "version": "8.8.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.7.1",
+      "version": "8.8.0",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index d7c19216..7c3e050f 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.7.1",
+  "version": "8.8.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 4fcef7b8ed187c579b9608f06e488449a3830c93 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sat, 18 Jul 2026 10:40:45 -0700
Subject: [PATCH 70/82] fix: import dedup off-switch honesty + brain-owned
 lifecycle for the background pass
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The post-import background deduplication pass (a merge-DELETE writer,
debounced ~5 minutes after import) had four lifecycle defects:

- enableDeduplication: false did not gate the background schedule — an
  import that explicitly opted out could still have entities auto-removed
  minutes later. The flag now gates both the inline and background passes.
- Each import() constructed its own coordinator-owned deduplicator, so the
  debounce never spanned imports (N imports = N delete timers). The brain
  now owns a single lazy instance (getBackgroundDeduplicator).
- close() never cancelled pending dedup; a delete pass could fire against
  a closed brain. close() now cancels it first.
- The 5-minute timer held the process open (exit-hang class); now unref'd.

Four regression tests pin the contract (background-dedup-lifecycle);
import guides document that false disables both passes.
---
 RELEASES.md                                   | 22 +++++
 docs/guides/import-anything.md                |  5 +-
 docs/guides/import-quick-reference.md         | 13 ++-
 src/brainy.ts                                 | 28 ++++++
 src/import/BackgroundDeduplicator.ts          | 13 ++-
 src/import/ImportCoordinator.ts               | 23 +++--
 .../background-dedup-lifecycle.test.ts        | 85 +++++++++++++++++++
 7 files changed, 179 insertions(+), 10 deletions(-)
 create mode 100644 tests/integration/background-dedup-lifecycle.test.ts

diff --git a/RELEASES.md b/RELEASES.md
index 7baa89ec..4a62123a 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,28 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.8.1 — 2026-07-18 (the import dedup off-switch is now honest + lifecycle-safe)
+
+The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes
+after an import, merging entities judged duplicates by id / name / vector similarity) had
+three lifecycle defects, all fixed:
+
+- **`enableDeduplication: false` now actually disables it.** The background pass was
+  scheduled unconditionally — an import that explicitly opted out could still have
+  entities auto-removed 5 minutes later. The flag now gates BOTH the inline merge and
+  the background pass (regression-pinned).
+- **One deduplicator per brain, owned by the brain.** Each `import()` call constructed its
+  own coordinator + deduplicator, so the "debounced" timer never actually debounced across
+  imports (N imports = N delete timers). The brain now owns a single instance — the
+  debounce genuinely spans imports — and `close()` cancels pending work, so a delete pass
+  can never fire against a closed brain.
+- **The 5-minute timer is unref'd** — a pending pass no longer holds the process open
+  (the exit-hang class; this timer had escaped the earlier sweep).
+
+Retention note for keep-everything deployments: with `enableDeduplication: false` on
+import calls and `retention: 'all'` in config, no engine path removes records
+automatically.
+
 ## v8.8.0 — 2026-07-17 (OS-limit detection for pool-scale deployments)
 
 Small minor: brains now detect the two OS limits that bite at pool scale and warn **before**
diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md
index e0cd94ed..b1bb15ef 100644
--- a/docs/guides/import-anything.md
+++ b/docs/guides/import-anything.md
@@ -300,7 +300,10 @@ await brain.import(data, {
  // Deduplication
  enableDeduplication: true, // Check for duplicate entities (default: true)
  deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85)
- // Note: Auto-disabled for imports >100 entities
+ // Notes: false disables BOTH the inline merge and the background pass that
+ // runs ~5 min after the last import (merged duplicates are deleted).
+ // The inline pass auto-disables for imports >100 entities (O(n²) cost);
+ // the background pass still covers those unless the flag is false.
 
  // Performance
  chunkSize: 100, // Batch size for processing (default: varies by operation)
diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md
index 5a850a86..7837d49e 100644
--- a/docs/guides/import-quick-reference.md
+++ b/docs/guides/import-quick-reference.md
@@ -86,11 +86,22 @@ await brain.import(file, {
 
 ```typescript
 await brain.import(file, {
-  enableDeduplication: true,         // Check for duplicates (default: false)
+  enableDeduplication: true,         // Check for duplicates (default: true)
   deduplicationThreshold: 0.85       // Similarity threshold (default: 0.85)
 })
 ```
 
+Deduplication merges entities judged duplicates — the non-primary records are
+**deleted**. Set `enableDeduplication: false` to disable it entirely: the flag
+gates both the inline merge during import and the background pass that runs
+about 5 minutes after the last import.
+
+```typescript
+await brain.import(file, {
+  enableDeduplication: false         // No merging, inline or background
+})
+```
+
 ### Import Tracking
 
 Track and organize imports by project:
diff --git a/src/brainy.ts b/src/brainy.ts
index 89e1267a..c7b4c89a 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -10128,6 +10128,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     return await coordinator.import(source as Buffer | string | object, options)
   }
 
+  /** Brain-owned background deduplicator (lazy; see getBackgroundDeduplicator). */
+  private _backgroundDedup?: import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator
+
+  /**
+   * The single brain-owned BackgroundDeduplicator, lazily constructed.
+   *
+   * Ownership matters here: the post-import dedup timer must outlive the
+   * per-call ImportCoordinator but never the brain. One instance per brain
+   * restores the intended cross-import debounce (per-coordinator instances
+   * each armed their own timer, so the "debounce" never spanned imports) and
+   * gives close() a handle to cancel pending work — a delete pass must never
+   * fire against a closed brain.
+   * @internal
+   */
+  async getBackgroundDeduplicator(): Promise<
+    import('./import/BackgroundDeduplicator.js').BackgroundDeduplicator
+  > {
+    if (!this._backgroundDedup) {
+      const { BackgroundDeduplicator } = await import('./import/BackgroundDeduplicator.js')
+      this._backgroundDedup = new BackgroundDeduplicator(this)
+    }
+    return this._backgroundDedup
+  }
+
   /**
    * Virtual File System API - Knowledge Operating System
    *
@@ -16070,6 +16094,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
    * This ensures deferred persistence mode data is saved
    */
   async close(): Promise<void> {
+    // Cancel any pending post-import background deduplication FIRST — it is a
+    // writer (merge-deletes), and no delete pass may start mid- or post-close.
+    this._backgroundDedup?.cancelPending()
+
     // Change-feed teardown: no events are delivered for or after close().
     this._changeFeed.close()
 
diff --git a/src/import/BackgroundDeduplicator.ts b/src/import/BackgroundDeduplicator.ts
index 53b28262..073fbd9e 100644
--- a/src/import/BackgroundDeduplicator.ts
+++ b/src/import/BackgroundDeduplicator.ts
@@ -41,6 +41,14 @@ export interface DeduplicationStats {
  * - Import-scoped deduplication (no cross-contamination)
  * - 3-tier strategy (ID → Name → Similarity)
  * - Uses existing indexes (EntityIdMapper, MetadataIndexManager, TypeAware HNSW)
+ *
+ * Lifecycle: ONE instance per brain, owned by Brainy (getBackgroundDeduplicator)
+ * so the debounce genuinely spans imports and brain.close() cancels pending
+ * work via cancelPending() — this pass merge-DELETES duplicate entities, so it
+ * must never fire against a closed brain. The enableDeduplication gate lives
+ * at the scheduling call site (ImportCoordinator); scheduleDedup itself is
+ * unconditional. The timer is unref'd — a pending pass never holds the
+ * process open.
  */
 export class BackgroundDeduplicator {
   private brain: Brainy
@@ -67,12 +75,15 @@ export class BackgroundDeduplicator {
       clearTimeout(this.debounceTimer)
     }
 
-    // Schedule for 5 minutes from now
+    // Schedule for 5 minutes from now. unref'd: a pending dedup pass must
+    // never hold the process open (exit-hang class) — if the process exits
+    // first, the pass simply never runs; imports are already durable.
     this.debounceTimer = setTimeout(() => {
       this.runBatchDedup().catch(error => {
         prodLog.error('[BackgroundDedup] Batch dedup failed:', error)
       })
     }, 5 * 60 * 1000)
+    this.debounceTimer.unref?.()
   }
 
   /**
diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts
index 21d09410..1e1316b7 100644
--- a/src/import/ImportCoordinator.ts
+++ b/src/import/ImportCoordinator.ts
@@ -13,7 +13,6 @@
 import { Brainy } from '../brainy.js'
 import { FormatDetector, SupportedFormat } from './FormatDetector.js'
 import { ImportHistory, type ImportHistoryEntry } from './ImportHistory.js'
-import { BackgroundDeduplicator } from './BackgroundDeduplicator.js'
 import { SmartExcelImporter } from '../importers/SmartExcelImporter.js'
 import { SmartPDFImporter } from '../importers/SmartPDFImporter.js'
 import { SmartCSVImporter } from '../importers/SmartCSVImporter.js'
@@ -112,7 +111,12 @@ export interface ValidImportOptions {
   /** Confidence threshold for entities */
   confidenceThreshold?: number
 
-  /** Enable entity deduplication across imports */
+  /**
+   * Enable entity deduplication (default: true). Gates BOTH passes: the
+   * inline merge during import AND the debounced background pass that runs
+   * ~5 minutes after the last import (which merge-DELETES duplicate entities).
+   * Set false for deployments that must never auto-remove records.
+   */
   enableDeduplication?: boolean
 
   /** Similarity threshold for deduplication (0-1) */
@@ -286,7 +290,6 @@ export class ImportCoordinator {
   private brain: Brainy
   private detector: FormatDetector
   private history: ImportHistory
-  private backgroundDedup: BackgroundDeduplicator
   private excelImporter: SmartExcelImporter
   private pdfImporter: SmartPDFImporter
   private csvImporter: SmartCSVImporter
@@ -300,7 +303,6 @@ export class ImportCoordinator {
     this.brain = brain
     this.detector = new FormatDetector()
     this.history = new ImportHistory(brain)
-    this.backgroundDedup = new BackgroundDeduplicator(brain)
     this.excelImporter = new SmartExcelImporter(brain)
     this.pdfImporter = new SmartPDFImporter(brain)
     this.csvImporter = new SmartCSVImporter(brain)
@@ -1459,9 +1461,16 @@ export class ImportCoordinator {
       }
     }
 
-    // Schedule background deduplication (debounced 5 minutes)
-    if (trackingContext && trackingContext.importId) {
-      this.backgroundDedup.scheduleDedup(trackingContext.importId)
+    // Schedule background deduplication (debounced 5 minutes, brain-owned so
+    // close() can cancel it). Honors the same enableDeduplication gate as the
+    // inline pass — false means NO dedup, inline or background.
+    if (
+      trackingContext &&
+      trackingContext.importId &&
+      options.enableDeduplication !== false
+    ) {
+      const backgroundDedup = await this.brain.getBackgroundDeduplicator()
+      backgroundDedup.scheduleDedup(trackingContext.importId)
     }
 
     return {
diff --git a/tests/integration/background-dedup-lifecycle.test.ts b/tests/integration/background-dedup-lifecycle.test.ts
new file mode 100644
index 00000000..48c7a4a7
--- /dev/null
+++ b/tests/integration/background-dedup-lifecycle.test.ts
@@ -0,0 +1,85 @@
+/**
+ * @module tests/integration/background-dedup-lifecycle
+ * @description The post-import background deduplication pass (a merge-DELETE
+ * writer) obeys the same contract as the inline pass. Laws:
+ * (1) enableDeduplication:false schedules NO background pass — the brain-owned
+ *     deduplicator is never even constructed;
+ * (2) by default the pass IS scheduled, brain-owned, with an unref'd timer
+ *     (a pending pass never holds the process open);
+ * (3) repeated imports debounce into ONE pending batch on ONE instance
+ *     (per-coordinator instances used to arm one timer per import);
+ * (4) close() cancels pending work — no delete pass can fire after close.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../src/brainy.js'
+
+const ROWS = [
+  { name: 'Alice Zephyr', role: 'engineer' },
+  { name: 'Bob Quill', role: 'writer' }
+]
+
+// Keep imports fast and deterministic — dedup scheduling is what's under test.
+const FAST = {
+  enableNeuralExtraction: false,
+  enableRelationshipInference: false,
+  enableConceptExtraction: false
+} as const
+
+// Deterministic stub embedder (hnsw-rebuild.test.ts pattern) — dedup
+// scheduling never inspects vector CONTENT, so skip the WASM model load.
+const stubEmbedding = async (text: string): Promise<number[]> => {
+  const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0)
+  const vector = new Array(384).fill(0).map((_, i) => Math.sin(hash + i))
+  return vector
+}
+
+describe('background dedup lifecycle', () => {
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'memory' as const },
+      embeddingFunction: stubEmbedding
+    })
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('enableDeduplication:false schedules no background pass at all', async () => {
+    await brain.import(ROWS, { ...FAST, enableDeduplication: false })
+    expect((brain as any)._backgroundDedup).toBeUndefined()
+  })
+
+  it('default schedules a brain-owned pass with an unref-ed timer', async () => {
+    await brain.import(ROWS, { ...FAST })
+    const dedup = (brain as any)._backgroundDedup
+    expect(dedup).toBeDefined()
+    expect(dedup.pendingImports.size).toBe(1)
+    const timer = dedup.debounceTimer
+    expect(timer).toBeDefined()
+    // Node timers expose hasRef(); an unref'd timer must not hold the process.
+    expect(typeof timer.hasRef).toBe('function')
+    expect(timer.hasRef()).toBe(false)
+  })
+
+  it('imports debounce into one pending batch on one brain-owned instance', async () => {
+    await brain.import(ROWS, { ...FAST })
+    const first = (brain as any)._backgroundDedup
+    await brain.import([{ name: 'Cara Vex', role: 'analyst' }], { ...FAST })
+    expect((brain as any)._backgroundDedup).toBe(first)
+    expect(first.pendingImports.size).toBe(2)
+  })
+
+  it('close() cancels pending background dedup', async () => {
+    await brain.import(ROWS, { ...FAST })
+    const dedup = (brain as any)._backgroundDedup
+    expect(dedup.debounceTimer).toBeDefined()
+    await brain.close()
+    expect(dedup.debounceTimer).toBeUndefined()
+    expect(dedup.pendingImports.size).toBe(0)
+  })
+})

From 6207e48b518bd80bdbb0113099a69a7a619e0957 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sat, 18 Jul 2026 10:51:37 -0700
Subject: [PATCH 71/82] fix: O(1) adaptive retention accounting + historyStats
 fleet audit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Under default adaptive retention, every flush() recomputed total history
bytes by walking EVERY committed generation's delta — O(all generations)
with disk re-reads past the 4096-entry delta-cache bound. On a production
brain with 70,000+ accumulated generations this turned every write into a
full-tail scan (60-100s writes, escalating with history growth), even
though the free-RAM budget never tripped and nothing was ever reclaimed
(SELF-GENERATIONS-GROWTH).

historyBytes() now maintains a running total: seeded by one walk on first
use, then updated incrementally at both commit paths (+bytes) and the
compaction reclaim loop (−bytes), dropped on reopenAfterRestore. The
adaptive retention check on every flush is O(1). Invariant regression-
pinned: running total ≡ fresh walk through transact commits, single-op
group commits, and compaction.

New brain.historyStats() (exported HistoryStats): read-only generation
count / bytes / generation+timestamp range / horizon / retention mode /
effective budget — the one-call per-brain fleet audit for retention
exposure.
---
 RELEASES.md                           | 26 ++++++++-
 src/brainy.ts                         | 28 ++++++++++
 src/db/generationStore.ts             | 72 +++++++++++++++++++++++--
 src/db/types.ts                       | 30 +++++++++++
 src/index.ts                          |  1 +
 tests/unit/db/generationStore.test.ts | 76 +++++++++++++++++++++++++++
 6 files changed, 228 insertions(+), 5 deletions(-)

diff --git a/RELEASES.md b/RELEASES.md
index 4a62123a..fd9d64ec 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,7 +10,31 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
-## v8.8.1 — 2026-07-18 (the import dedup off-switch is now honest + lifecycle-safe)
+## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest)
+
+### The flush-storm fix (production incident, reported by a long-running deployment)
+
+Under the default adaptive retention, **every `flush()` re-walked the entire committed
+generation history** to compute total history bytes for the budget check — O(all
+generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with
+70,000+ accumulated generations that turned every write into a full-tail scan (60-100s
+writes), even though the budget (free-RAM-based) never tripped and nothing was ever
+reclaimed. Fixed:
+
+- `historyBytes()` now maintains a **running total**: seeded by one walk on first use,
+  then updated incrementally at every commit and reclaim — the adaptive retention check
+  on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through
+  both commit paths and compaction).
+- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count,
+  total on-disk bytes, generation/timestamp range, compaction horizon, retention mode,
+  and the effective adaptive budget — the one-call fleet-audit for sizing retention
+  exposure per brain.
+- Interim guidance for keep-everything deployments already affected: `retention: 'all'`
+  skips the adaptive accounting entirely (and is the correct policy if you never want
+  history reclaimed). The accumulated files are harmless at rest; this release removes
+  the per-write cost of their existence.
+
+### The import dedup off-switch (lifecycle honesty)
 
 The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes
 after an import, merging entities judged duplicates by id / name / vector similarity) had
diff --git a/src/brainy.ts b/src/brainy.ts
index c7b4c89a..008733b9 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -192,6 +192,7 @@ import { MemoryStorage } from './storage/adapters/memoryStorage.js'
 import type {
   CompactHistoryOptions,
   CompactHistoryResult,
+  HistoryStats,
   TransactOptions,
   TransactReceipt,
   TxLogEntry,
@@ -8211,6 +8212,33 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     return this.generationStore.compact(options)
   }
 
+  /**
+   * @description Read-only generational-history footprint for fleet audits:
+   * generation count, total on-disk bytes, generation/timestamp range, the
+   * compaction horizon, and the retention policy in force. Touches no data and
+   * changes nothing. First call pays one walk over committed deltas to seed
+   * the running byte total (subsequent calls — and every adaptive retention
+   * check — are then O(1)).
+   *
+   * A pool operator's exposure check is one call per brain:
+   * @example
+   * const stats = await brain.historyStats()
+   * console.log(`${stats.generations} generations, ${stats.bytes} bytes, mode=${stats.retentionMode}`)
+   */
+  async historyStats(): Promise<HistoryStats> {
+    await this.ensureInitialized()
+    const stats = await this.generationStore.historyStats()
+    const policy = this.resolveRetentionPolicy()
+    return {
+      ...stats,
+      retentionMode: policy.mode,
+      effectiveBudgetBytes:
+        policy.mode === 'adaptive'
+          ? this.adaptiveHistoryBudgetBytes(policy.budgetBytes)
+          : null
+    }
+  }
+
   /**
    * @description Drive the adaptive retention byte budget at runtime — the
    * settable input a machine-level coordinator (e.g. cor's `ResourceManager`,
diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts
index 0d813f4d..dd70f7aa 100644
--- a/src/db/generationStore.ts
+++ b/src/db/generationStore.ts
@@ -257,6 +257,15 @@ export class GenerationStore {
    */
   private deltaCacheMax = 4096
 
+  /**
+   * Running total of on-disk history bytes across committed generations —
+   * `null` until {@link historyBytes} pays its one seeding walk. Maintained
+   * incrementally at commit/reclaim so the adaptive retention check on every
+   * flush() is O(1), never a tail re-walk. Never updated by cache re-reads
+   * ({@link setDelta} inserts are cache population, not new history).
+   */
+  private historyBytesTotal: number | null = null
+
   /**
    * Model-B per-write group-commit — the in-memory PENDING tier.
    *
@@ -483,6 +492,40 @@ export class GenerationStore {
     return this.horizonGen
   }
 
+  /**
+   * @description Read-only history footprint for fleet audits: how much
+   * generational history this store holds on disk. `bytes` pays (and seeds)
+   * the one-time {@link historyBytes} walk on first call — subsequent calls
+   * are O(1). The oldest/newest timestamps come from those generations'
+   * deltas (cache-bounded reads).
+   * @returns Counts, bytes, generation range, and the compaction horizon.
+   */
+  async historyStats(): Promise<{
+    generations: number
+    bytes: number
+    oldestGeneration: number | null
+    newestGeneration: number | null
+    oldestTimestamp: number | null
+    newestTimestamp: number | null
+    horizon: number
+  }> {
+    let oldest: number | null = null
+    let newest: number | null = null
+    for (const gen of this.committedGensAsc()) {
+      if (oldest === null) oldest = gen
+      newest = gen
+    }
+    return {
+      generations: this.committedCount(),
+      bytes: await this.historyBytes(),
+      oldestGeneration: oldest,
+      newestGeneration: newest,
+      oldestTimestamp: oldest !== null ? (await this.getDelta(oldest)).timestamp : null,
+      newestTimestamp: newest !== null ? (await this.getDelta(newest)).timestamp : null,
+      horizon: this.horizonGen
+    }
+  }
+
   /**
    * @description Read one generation's persisted before-image records — the
    * compaction fallback for generations written before deltas carried
@@ -849,6 +892,9 @@ export class GenerationStore {
           timestamp,
           bytes: delta.bytes ?? 0
         })
+        if (this.historyBytesTotal !== null) {
+          this.historyBytesTotal += delta.bytes ?? 0
+        }
         this.extendChains(gen, nouns, verbs)
         const logEntry: TxLogEntry = { generation: gen, timestamp, ...(args.meta && { meta: args.meta }) }
         await this.storage.appendTxLogLine(JSON.stringify(logEntry))
@@ -1302,6 +1348,9 @@ export class GenerationStore {
           timestamp: buf.timestamp,
           bytes: genBytes.get(gen) ?? 0
         })
+        if (this.historyBytesTotal !== null) {
+          this.historyBytesTotal += genBytes.get(gen) ?? 0
+        }
         this.pendingBuffer.delete(gen)
       }
       this.pendingGens = []
@@ -2122,17 +2171,26 @@ export class GenerationStore {
   /**
    * @description Total serialized bytes of the ON-DISK generational history —
    * the sum of every committed generation's recorded `bytes`. Backs the
-   * `maxBytes` and adaptive retention caps. Reads each committed generation's
-   * delta (cached; a re-read only for cache-evicted ones) — O(committed
-   * generations), bounded by retention itself and invoked only at compaction
-   * time. Pending (un-flushed) generations are excluded (they are not on disk).
+   * `maxBytes` and adaptive retention caps. O(1) after the first call: the
+   * total is computed by ONE walk over committed deltas, then maintained
+   * incrementally at every commit (+bytes) and reclaim (−bytes) and dropped on
+   * a wholesale state replacement (restore). Without the running total, the
+   * adaptive auto-compaction on every flush() re-walked the ENTIRE history —
+   * O(committed generations) file reads per flush past the delta-cache bound —
+   * which is how a 70k-generation production brain turned every write into a
+   * full-tail scan (SELF-GENERATIONS-GROWTH). Pending (un-flushed) generations
+   * are excluded (they are not on disk).
    * @returns The total on-disk history byte count.
    */
   async historyBytes(): Promise<number> {
+    if (this.historyBytesTotal !== null) {
+      return this.historyBytesTotal
+    }
     let total = 0
     for (const gen of this.committedGensAsc()) {
       total += (await this.getDelta(gen)).bytes
     }
+    this.historyBytesTotal = total
     return total
   }
 
@@ -2206,6 +2264,9 @@ export class GenerationStore {
 
         await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${gen}`)
         this.deltaCache.delete(gen)
+        if (this.historyBytesTotal !== null) {
+          this.historyBytesTotal -= delta.bytes
+        }
 
         // AFTER the record-set is gone (over-count-only crash ordering):
         // release its history references and reclaim any blob left with zero
@@ -2266,6 +2327,9 @@ export class GenerationStore {
   async reopenAfterRestore(floorGeneration: number): Promise<void> {
     await this.withMutex(async () => {
       this.deltaCache.clear()
+      // The running history-byte total describes the REPLACED store — drop it;
+      // the next historyBytes() re-seeds with one walk over the new state.
+      this.historyBytesTotal = null
       // A wholesale state replacement invalidates any buffered single-op
       // history — discard the pending tier (its live writes are gone with the
       // replaced store).
diff --git a/src/db/types.ts b/src/db/types.ts
index d1355dab..521396b8 100644
--- a/src/db/types.ts
+++ b/src/db/types.ts
@@ -193,6 +193,36 @@ export interface CompactHistoryResult {
   horizon: number
 }
 
+/**
+ * @description Result of `brain.historyStats()` — the read-only generational
+ * history footprint, for fleet audits and ops doors. A pool operator runs this
+ * per brain to size retention exposure (how much MVCC history each brain
+ * carries and under which policy) without touching any data.
+ */
+export interface HistoryStats {
+  /** Committed generation record-sets currently on disk. */
+  generations: number
+  /** Total on-disk history bytes across those record-sets. */
+  bytes: number
+  /** Oldest committed generation still on disk (null when history is empty). */
+  oldestGeneration: number | null
+  /** Newest committed generation (null when history is empty). */
+  newestGeneration: number | null
+  /** Commit timestamp (ms) of the oldest on-disk generation. */
+  oldestTimestamp: number | null
+  /** Commit timestamp (ms) of the newest on-disk generation. */
+  newestTimestamp: number | null
+  /** Compaction horizon — generations below it were reclaimed. */
+  horizon: number
+  /** The effective retention mode this brain runs under. */
+  retentionMode: 'all' | 'adaptive' | 'explicit'
+  /**
+   * The adaptive byte budget in force (coordinator-driven or the local
+   * free-memory probe); null under 'all' or explicit caps.
+   */
+  effectiveBudgetBytes: number | null
+}
+
 // ============================================================================
 // Db surfaces
 // ============================================================================
diff --git a/src/index.ts b/src/index.ts
index 5c57fe29..ee01d885 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -201,6 +201,7 @@ export type {
   TxLogEntry,
   CompactHistoryOptions,
   CompactHistoryResult,
+  HistoryStats,
   ChangedIds,
   DiffResult,
   HistoryVersion,
diff --git a/tests/unit/db/generationStore.test.ts b/tests/unit/db/generationStore.test.ts
index 184d3974..611a97d7 100644
--- a/tests/unit/db/generationStore.test.ts
+++ b/tests/unit/db/generationStore.test.ts
@@ -489,4 +489,80 @@ describe('db/GenerationStore', () => {
       store.release(2)
     })
   })
+
+  // ==========================================================================
+  describe('history-bytes running total (the O(1) retention check)', () => {
+    /** A fresh walk with the cache dropped — ground truth for the invariant. */
+    async function groundTruthBytes(): Promise<number> {
+      ;(store as any).historyBytesTotal = null
+      return store.historyBytes()
+    }
+
+    it('is seeded once, then maintained through commits WITHOUT re-walks', async () => {
+      await commitWrite(ID_A, 1)
+      await commitWrite(ID_A, 2)
+      const seeded = await store.historyBytes()
+      expect(seeded).toBe(await groundTruthBytes())
+
+      // From here every read must come from the running total, not a walk:
+      // getDelta re-reads are the walk's cost — commits must not trigger any.
+      const getDeltaSpy = vi.spyOn(store as any, 'getDelta')
+      await commitWrite(ID_B, 1)
+      const afterCommit = await store.historyBytes()
+      expect(getDeltaSpy).not.toHaveBeenCalled()
+      getDeltaSpy.mockRestore()
+      expect(afterCommit).toBe(await groundTruthBytes())
+    })
+
+    it('stays exact through single-op group commits and compaction', async () => {
+      await commitWrite(ID_A, 1)
+      await store.historyBytes() // seed
+      // Single-op path: buffered generations flushed as one group commit.
+      await store.commitSingleOp({
+        touched: { nouns: [ID_B] },
+        execute: async () => {
+          await storage.saveNounMetadata(ID_B, metadataFixture(1))
+        }
+      })
+      await store.flushPendingSingleOps()
+      expect(await store.historyBytes()).toBe(await groundTruthBytes())
+
+      await store.historyBytes() // re-seed after ground-truth reset
+      await store.compact({ maxGenerations: 1 })
+      expect(await store.historyBytes()).toBe(await groundTruthBytes())
+    })
+
+    it('historyStats reports counts, bytes, range, and horizon read-only', async () => {
+      await commitWrite(ID_A, 1)
+      await commitWrite(ID_B, 1)
+      const stats = await store.historyStats()
+      expect(stats.generations).toBe(2)
+      expect(stats.bytes).toBe(await store.historyBytes())
+      expect(stats.oldestGeneration).toBe(1)
+      expect(stats.newestGeneration).toBe(2)
+      expect(stats.oldestTimestamp).toBeLessThanOrEqual(stats.newestTimestamp!)
+      expect(stats.horizon).toBe(0)
+      // Read-only: nothing was reclaimed by asking.
+      expect(store.committedGeneration()).toBe(2)
+
+      await store.compact({ maxGenerations: 1 })
+      const after = await store.historyStats()
+      expect(after.generations).toBe(1)
+      expect(after.oldestGeneration).toBe(2)
+      expect(after.horizon).toBe(1)
+    })
+
+    it('empty history reports null range and zero bytes', async () => {
+      const stats = await store.historyStats()
+      expect(stats).toMatchObject({
+        generations: 0,
+        bytes: 0,
+        oldestGeneration: null,
+        newestGeneration: null,
+        oldestTimestamp: null,
+        newestTimestamp: null,
+        horizon: 0
+      })
+    })
+  })
 })

From a544225872d11f61439585305653d590b0970fd9 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sat, 18 Jul 2026 10:57:30 -0700
Subject: [PATCH 72/82] chore(release): 8.8.1

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f97046f0..a4a97ba3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18)
+
+- fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48)
+- fix: import dedup off-switch honesty + brain-owned lifecycle for the background pass (4fcef7b)
+
+
 ### [8.8.0](https://github.com/soulcraftlabs/brainy/compare/v8.7.1...v8.8.0) (2026-07-17)
 
 - feat: OS-limit detection for pool-scale deployments (16a73b8)
diff --git a/package-lock.json b/package-lock.json
index 198e9116..3f157871 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.8.0",
+  "version": "8.8.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.8.0",
+      "version": "8.8.1",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index 7c3e050f..fb467c2e 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.8.0",
+  "version": "8.8.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 42037d0cd0e85d91555842e1e6badb4a1ecf51db Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sat, 18 Jul 2026 14:02:23 -0700
Subject: [PATCH 73/82] chore: push public docs to the soulcraft.com ingest
 door on release
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

scripts/push-docs.js collects docs/**/*.md with public:true frontmatter
and POSTs them (batches of 10, idempotent per slug) to the docs ingest
door after npm publish — release.sh step 12. Absent secret = loud skip
(publish already happened; the serving side can interim-sync); a failed
push exits non-zero so the docs site never silently trails npm. The
combined /docs landing index is deliberately NOT pushed per-repo — it
spans both engine corpora and is authored on the serving side.
---
 scripts/push-docs.js | 116 +++++++++++++++++++++++++++++++++++++++++++
 scripts/release.sh   |  12 +++++
 2 files changed, 128 insertions(+)
 create mode 100644 scripts/push-docs.js

diff --git a/scripts/push-docs.js b/scripts/push-docs.js
new file mode 100644
index 00000000..699d332b
--- /dev/null
+++ b/scripts/push-docs.js
@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+/**
+ * @module scripts/push-docs
+ * @description Push this repo's PUBLIC docs to the soulcraft.com docs ingest
+ * door after an npm publish (VENUE-DOCS-RELEASE-PUSH — retires the old
+ * build-time docs sync).
+ *
+ * Contract (mirrors the reference implementation on the serving side):
+ *   POST {base}/api/docs/ingest
+ *   headers: x-service-secret: $DOCS_INGEST_SECRET, Content-Type: application/json
+ *   body: { docs: [{ slug, title, markdown, nav: { order, section } }] }
+ *   batches of 10, idempotent per slug.
+ *
+ * A doc is public iff its frontmatter has `public: true` AND a `slug`. The
+ * frontmatter is stripped; `category` → nav.section, `order` → nav.order.
+ *
+ * Deliberately NOT pushed: the combined /docs landing index. It spans BOTH
+ * engine corpora (this repo's and the native accelerator's), so a per-repo
+ * push would clobber the union — the index is authored on the serving side.
+ *
+ * Env: DOCS_INGEST_SECRET (required), DOCS_INGEST_BASE (default
+ * https://soulcraft.com). Exits 0 with a LOUD warning when the secret is
+ * absent (the npm publish has already happened; the serving side runs its
+ * interim sync on request) and exits 1 when a push actually fails — the docs
+ * site would silently trail npm otherwise, and that must be visible.
+ */
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+
+const BASE = (process.env.DOCS_INGEST_BASE || 'https://soulcraft.com').replace(/\/+$/, '')
+const SECRET = process.env.DOCS_INGEST_SECRET
+const DOCS_DIR = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'docs')
+const BATCH = 10
+
+if (!SECRET) {
+  console.warn(
+    '⚠️  DOCS PUSH SKIPPED: DOCS_INGEST_SECRET is not set.\n' +
+      '    soulcraft.com/docs now TRAILS this npm release until docs are pushed.\n' +
+      '    Either export DOCS_INGEST_SECRET and re-run `node scripts/push-docs.js`,\n' +
+      '    or ping venue on VENUE-DOCS-RELEASE-PUSH for the interim sync.'
+  )
+  process.exit(0)
+}
+
+/** Minimal frontmatter split — returns [meta, body] or [null, raw]. */
+function parseFrontmatter(raw) {
+  const m = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/)
+  if (!m) return [null, raw]
+  const meta = {}
+  for (const line of m[1].split('\n')) {
+    const kv = line.match(/^(\w[\w-]*):\s*(.*)$/)
+    if (kv) meta[kv[1]] = kv[2].trim().replace(/^["']|["']$/g, '')
+  }
+  return [meta, m[2]]
+}
+
+const docs = []
+;(function walk(dir) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name)
+    if (entry.isDirectory()) walk(full)
+    else if (entry.name.endsWith('.md')) {
+      const [meta, body] = parseFrontmatter(fs.readFileSync(full, 'utf-8'))
+      if (!meta || meta.public !== 'true' || !meta.slug) continue
+      docs.push({
+        slug: meta.slug,
+        title: meta.title || meta.slug,
+        markdown: body.trim(),
+        nav: {
+          order: Number.parseInt(meta.order || '99', 10) || 99,
+          section: meta.category || 'guides'
+        }
+      })
+    }
+  }
+})(DOCS_DIR)
+
+if (docs.length === 0) {
+  console.error('❌ DOCS PUSH FAILED: zero public docs collected — refusing to push an empty corpus.')
+  process.exit(1)
+}
+docs.sort((a, b) => a.slug.localeCompare(b.slug))
+console.log(`Pushing ${docs.length} public docs to ${BASE}/api/docs/ingest …`)
+
+let failed = false
+for (let i = 0; i < docs.length; i += BATCH) {
+  const batch = docs.slice(i, i + BATCH)
+  try {
+    const res = await fetch(`${BASE}/api/docs/ingest`, {
+      method: 'POST',
+      headers: {
+        'x-service-secret': SECRET,
+        'Content-Type': 'application/json',
+        'User-Agent': 'brainy-docs-push/1.0'
+      },
+      body: JSON.stringify({ docs: batch }),
+      signal: AbortSignal.timeout(120_000)
+    })
+    if (!res.ok) {
+      throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`)
+    }
+    console.log(`  batch ${i / BATCH + 1}: ${batch.map((d) => d.slug).join(', ')} → ok`)
+  } catch (err) {
+    failed = true
+    console.error(`  batch ${i / BATCH + 1} FAILED: ${err instanceof Error ? err.message : err}`)
+  }
+}
+
+if (failed) {
+  console.error(
+    '❌ DOCS PUSH INCOMPLETE — soulcraft.com/docs may trail npm. ' +
+      'Re-run `node scripts/push-docs.js` or ping venue on VENUE-DOCS-RELEASE-PUSH.'
+  )
+  process.exit(1)
+}
+console.log('✅ Docs pushed.')
diff --git a/scripts/release.sh b/scripts/release.sh
index 0e6a9c43..7d860564 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -196,6 +196,18 @@ else
 fi
 echo -e "${GREEN}✅ GitHub release created${NC}\n"
 
+# Step 12: Push public docs to the soulcraft.com docs ingest door
+# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when
+# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
+# that already happened) when a push errors, so the docs site never
+# silently trails npm.
+echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
+if node scripts/push-docs.js; then
+  echo -e "${GREEN}✅ Docs push step done${NC}\n"
+else
+  echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
+fi
+
 echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
 echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
 echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"

From 945d92d29e64ec84bee370b22692637ffed31e4c Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 10:54:36 -0700
Subject: [PATCH 74/82] fix: one field-resolution law across aggregation hooks,
 source.where, removeMany, and find() spellings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Four fixes from a consumer conformance report, one root disease — two
field-resolution regimes where there must be one:

- The delete/update aggregation hooks fed the engine a partial entity
  view (type/service/data/metadata only), so a reserved-field groupBy
  (subtype, visibility, ...) resolved to a nonexistent group on the way
  down: counts drifted upward forever after deletes, and updates moving
  an entity between reserved-field groups double-counted. The hooks now
  pass the full-fidelity view via entityForAggFromRawRecord (every
  reserved field top-level, mirroring the add path); the update sites
  pass the full get() view instead of a hand-rolled subset.
- Aggregation source.where resolved fields only against the custom
  metadata bag, so where on a reserved field silently matched nothing.
  The matcher now resolves each filtered field through
  resolveEntityField — the same single source of truth groupBy uses.
- removeMany() with no usable selector (bare array passed positionally,
  empty params, ids: []) resolved successfully having deleted nothing.
  All three now throw; the two legacy tests that pinned the silent
  no-op as 'graceful' now pin the refusal.
- find() where keys accept both spellings: a metadata.-prefixed key
  falls back to its flattened spelling when the prefixed one is not
  indexed (metadata is flattened at index time). A literal nested custom
  key named metadata still wins when indexed as spelled.

Five regression pins in aggregate-reserved-fields.test.ts (4 of 5 vary
red on the unfixed code).
---
 RELEASES.md                                   |  26 +++
 src/aggregation/AggregationIndex.ts           |  14 +-
 src/brainy.ts                                 | 104 +++++++----
 src/utils/metadataIndex.ts                    |  21 ++-
 .../aggregate-reserved-fields.test.ts         | 169 ++++++++++++++++++
 .../metadata-index-cleanup.unit.test.ts       |   8 +-
 tests/unit/brainy/batch-operations.test.ts    |   6 +-
 7 files changed, 303 insertions(+), 45 deletions(-)
 create mode 100644 tests/integration/aggregate-reserved-fields.test.ts

diff --git a/RELEASES.md b/RELEASES.md
index fd9d64ec..d1374d5a 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -10,6 +10,32 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 
 ---
 
+## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting)
+
+Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution
+regimes where there must be one:
+
+- **Aggregates grouped by a RESERVED field (`subtype`, `visibility`, …) now decrement on
+  delete.** The delete/update hooks fed the aggregation engine a partial entity view (type,
+  service, data, metadata only), so a reserved-field `groupBy` resolved to a nonexistent group
+  on the way DOWN — counts drifted upward forever after any delete, and updates that moved an
+  entity between reserved-field groups double-counted it. The hooks now pass the full-fidelity
+  entity view (every reserved field top-level, the same shape the add path uses). If your
+  deployment derives stats from reserved-field aggregates, re-define those aggregates once
+  after upgrading (a changed definition triggers one rescan) or run them fresh — the drifted
+  persisted counts do not self-heal retroactively.
+- **Aggregation `source.where` on reserved fields now filters** instead of silently matching
+  nothing: the matcher resolves fields through the same resolver `groupBy` uses (top-level
+  standard fields + custom metadata), so `where: { subtype: 'note' }` means what it says.
+- **`removeMany()` refuses empty/invalid selectors loudly.** A bare array passed positionally
+  (`removeMany([id])` instead of `removeMany({ ids: [id] })`), an empty params object, or
+  `ids: []` used to resolve successfully having deleted nothing. All three now throw.
+- **`find()` accepts both where-key spellings.** Metadata is flattened at index time
+  (`metadata.entry.title` indexes as `entry.title`); a `metadata.`-prefixed where key now
+  falls back to its flattened spelling when the prefixed one isn't indexed — the
+  "unindexed field(s), returning []" confusion for storage-shaped spellings is gone. (A
+  literal nested custom key named `metadata` still wins when indexed as spelled.)
+
 ## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest)
 
 ### The flush-storm fix (production incident, reported by a long-running deployment)
diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts
index 7f7ffb27..f9382218 100644
--- a/src/aggregation/AggregationIndex.ts
+++ b/src/aggregation/AggregationIndex.ts
@@ -88,10 +88,18 @@ function matchesSource(entity: Record<string, unknown>, source: AggregateDefinit
     if (entity.service !== source.service) return false
   }
 
-  // Metadata where filter — match against the entity's metadata sub-object
+  // Where filter — resolve each filtered field through resolveEntityField,
+  // the SAME single source of truth groupBy uses (top-level standard fields
+  // + custom metadata). Matching only the metadata sub-object made
+  // where:{subtype}/{visibility}/… a silent no-op: reserved fields never
+  // live in the custom bag, so those filters could never match anything.
   if (source.where && Object.keys(source.where).length > 0) {
-    const metadata = (entity.metadata ?? entity) as Record<string, unknown>
-    if (!matchesMetadataFilter(metadata, source.where)) return false
+    const e = entity as unknown as HNSWNounWithMetadata
+    const resolved: Record<string, unknown> = {}
+    for (const key of Object.keys(source.where)) {
+      resolved[key] = resolveEntityField(e, key)
+    }
+    if (!matchesMetadataFilter(resolved, source.where)) return false
   }
 
   return true
diff --git a/src/brainy.ts b/src/brainy.ts
index 008733b9..21eab679 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -1883,6 +1883,29 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
   }
 
+  /**
+   * @description Build the AGGREGATION view of an entity from a stored flat
+   * metadata record — EVERY reserved field mapped to its top-level entity
+   * name (stored `noun` → `type`), custom metadata in `metadata`. This must
+   * mirror the add-path `entityForIndexing` shape exactly: the aggregation
+   * engine resolves groupBy/where fields via `resolveEntityField`
+   * (top-level standard fields + custom metadata), so a view that drops a
+   * reserved field makes every aggregate grouped by that field decrement a
+   * group that does not exist — counts then drift upward forever after
+   * deletes (SELF-AGGREGATE-DELETE-DRIFT). Do not hand-roll subsets of this.
+   * @param record - The stored flat metadata record (before-image or pre-delete read).
+   * @returns The full-fidelity entity view for aggregation hooks.
+   */
+  private entityForAggFromRawRecord(record: Record<string, unknown>): Record<string, unknown> {
+    const { reserved, custom } = splitNounMetadataRecord(record)
+    const { noun, ...rest } = reserved
+    return {
+      type: noun,
+      ...rest,
+      metadata: custom
+    }
+  }
+
   /**
    * @description Add an entity (noun) to the brain. Embeds `data` into a vector and
    * indexes the entity across all three intelligences — vector similarity, graph
@@ -3125,15 +3148,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           ]
         : undefined)
 
-      // Aggregation hook (outside transaction — derived data)
+      // Aggregation hook (outside transaction — derived data). `existing` is
+      // the full get() view — every reserved field top-level — and must be
+      // passed whole: a subset view makes the old-side decrement miss any
+      // reserved-field group (update would then double-count it).
       if (this._aggregationIndex) {
-        const oldEntityForAgg = {
-          type: existing.type,
-          service: existing.service,
-          data: existing.data,
-          metadata: existing.metadata
-        }
-        this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg)
+        this._aggregationIndex.onEntityUpdated(
+          params.id,
+          entityForIndexing,
+          existing as unknown as Record<string, unknown>
+        )
       }
   }
 
@@ -3245,19 +3269,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           ]
         : undefined)
 
-      // Aggregation hook (outside transaction — derived data)
+      // Aggregation hook (outside transaction — derived data). The view must
+      // carry EVERY reserved field top-level (not a subset): a groupBy on
+      // subtype/visibility/etc. otherwise decrements a nonexistent group and
+      // the real count never comes down.
       if (this._aggregationIndex && metadata) {
-        // Reconstruct entity-like object from stored metadata via the
-        // canonical reserved/custom split (the hand-rolled destructure here
-        // missed subtype/_rev, leaking them into the aggregation view).
-        const { reserved, custom } = splitNounMetadataRecord(metadata)
-        const entityForAgg = {
-          type: reserved.noun,
-          service: reserved.service,
-          data: reserved.data,
-          metadata: custom
-        }
-        this._aggregationIndex.onEntityDeleted(id, entityForAgg)
+        this._aggregationIndex.onEntityDeleted(
+          id,
+          this.entityForAggFromRawRecord(metadata as Record<string, unknown>)
+        )
       }
   }
 
@@ -6885,6 +6905,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     this.assertWritable('removeMany')
     await this.ensureInitialized()
 
+    // Loud selector validation: a call with no usable selector used to
+    // resolve successfully having deleted NOTHING (total: 0) — the classic
+    // silent no-op being a bare array passed positionally
+    // (removeMany([id]) instead of removeMany({ ids: [id] })). The caller
+    // believes the delete happened; every count derived afterwards is "wrong"
+    // while the engine was never even asked. Refuse instead.
+    if (Array.isArray(params)) {
+      throw new Error(
+        `removeMany() takes a params object, not a bare array — use removeMany({ ids: [...] })`
+      )
+    }
+    if (!params || (!params.ids && !params.type && !params.where)) {
+      throw new Error(
+        `removeMany() requires a selector: { ids } and/or { type, where }. ` +
+          `An empty selector would silently delete nothing — refusing.`
+      )
+    }
+    if (params.ids && params.ids.length === 0) {
+      throw new Error(
+        `removeMany() received ids: [] — an empty id list deletes nothing. ` +
+          `Pass the ids to delete, or omit ids and select by { type, where }.`
+      )
+    }
+
     // Determine what to delete
     let idsToDelete: string[] = []
 
@@ -9388,12 +9432,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     )
     plan.touchedNouns.push(params.id)
 
-    const oldEntityForAgg = {
-      type: existing.type,
-      service: existing.service,
-      data: existing.data,
-      metadata: existing.metadata
-    }
+    // The full planGetEntity view, passed whole — a subset view makes the
+    // old-side decrement miss reserved-field groups (double-count on update).
+    const oldEntityForAgg = existing as unknown as Record<string, unknown>
     plan.postCommit.push(() => {
       if (this._aggregationIndex) {
         this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg)
@@ -9514,14 +9555,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
 
     if (metadata) {
-      // Canonical reserved/custom split — mirror of remove()'s aggregation hook.
-      const { reserved, custom } = splitNounMetadataRecord(metadata)
-      const entityForAgg = {
-        type: reserved.noun,
-        service: reserved.service,
-        data: reserved.data,
-        metadata: custom
-      }
+      // Mirror of remove()'s aggregation hook — the FULL reserved view, so
+      // reserved-field groupBy decrements find their group.
+      const entityForAgg = this.entityForAggFromRawRecord(metadata as Record<string, unknown>)
       plan.postCommit.push(() => {
         if (this._aggregationIndex) {
           this._aggregationIndex.onEntityDeleted(id, entityForAgg)
diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts
index fd325319..c772bce4 100644
--- a/src/utils/metadataIndex.ts
+++ b/src/utils/metadataIndex.ts
@@ -1867,9 +1867,26 @@ export class MetadataIndexManager implements MetadataIndexProvider {
     // not once per AND-clause inside it.
     const unindexedFields: string[] = []
 
-    for (const [field, condition] of Object.entries(filter)) {
+    for (const [rawField, condition] of Object.entries(filter)) {
       // Skip logical operators
-      if (field === 'allOf' || field === 'anyOf' || field === 'not') continue
+      if (rawField === 'allOf' || rawField === 'anyOf' || rawField === 'not') continue
+
+      // Metadata is FLATTENED at index time (metadata.entry.title indexes as
+      // entry.title), so a `metadata.`-prefixed where key is almost always
+      // the caller spelling the STORAGE shape rather than the index shape.
+      // Accept both spellings: when the key as spelled is unindexed but its
+      // stripped spelling is, query the stripped one. A literal nested
+      // custom key named `metadata` still wins when indexed as spelled
+      // (checked first), so that rare shape keeps working.
+      let field = rawField
+      if (
+        rawField.startsWith('metadata.') &&
+        this.columnStore &&
+        !this.columnStore.hasField(rawField) &&
+        this.columnStore.hasField(rawField.slice('metadata.'.length))
+      ) {
+        field = rawField.slice('metadata.'.length)
+      }
 
       let fieldResults: string[] = []
 
diff --git a/tests/integration/aggregate-reserved-fields.test.ts b/tests/integration/aggregate-reserved-fields.test.ts
new file mode 100644
index 00000000..e81692d6
--- /dev/null
+++ b/tests/integration/aggregate-reserved-fields.test.ts
@@ -0,0 +1,169 @@
+/**
+ * @module tests/integration/aggregate-reserved-fields
+ * @description One field-resolution law across the whole aggregation + query
+ * surface (SELF-AGGREGATE-DELETE-DRIFT). Laws:
+ * (1) aggregates grouped by a RESERVED field (subtype) decrement on delete —
+ *     the delete-side entity view carries every reserved field, so the
+ *     decrement finds its group (counts must never drift from ground truth);
+ * (2) same for update: moving an entity between reserved-field groups
+ *     decrements the old group and increments the new one (no double-count);
+ * (3) aggregation source.where on a reserved field (subtype) FILTERS instead
+ *     of silently matching nothing;
+ * (4) removeMany refuses empty/invalid selectors loudly (bare array, empty
+ *     object, ids: []) instead of resolving as a silent no-op;
+ * (5) find() accepts both where spellings: flattened (entry.title) and
+ *     storage-shaped (metadata.entry.title) resolve to the same rows.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../src/brainy.js'
+import { NounType } from '../../src/types/graphTypes.js'
+
+const stubEmbedding = async (text: string): Promise<number[]> => {
+  const hash = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0)
+  return new Array(384).fill(0).map((_, i) => Math.sin(hash + i))
+}
+
+describe('aggregation + query field-resolution law', () => {
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'memory' as const },
+      embeddingFunction: stubEmbedding
+    })
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('reserved-field groupBy decrements on delete (the drift bug)', async () => {
+    brain.defineAggregate({
+      name: 'by_subtype',
+      source: { type: NounType.Document },
+      groupBy: ['subtype'],
+      metrics: { count: { op: 'count' } }
+    })
+
+    const ids: string[] = []
+    for (let i = 0; i < 5; i++) {
+      ids.push(
+        await brain.add({
+          data: `doc-${i}`,
+          type: NounType.Document,
+          subtype: 'note',
+          metadata: { team: 'alpha' }
+        })
+      )
+    }
+    let groups = await brain.queryAggregate('by_subtype')
+    expect(groups).toHaveLength(1)
+    expect(groups[0].groupKey).toEqual({ subtype: 'note' })
+    expect(groups[0].metrics.count).toBe(5)
+
+    await brain.remove(ids[0])
+    await brain.flush()
+
+    groups = await brain.queryAggregate('by_subtype')
+    expect(groups[0].metrics.count).toBe(4)
+    const live = await brain.find({ type: NounType.Document, limit: 100 })
+    expect(groups[0].metrics.count).toBe(live.length)
+  })
+
+  it('reserved-field groupBy moves between groups on update (no double-count)', async () => {
+    brain.defineAggregate({
+      name: 'by_subtype',
+      source: { type: NounType.Document },
+      groupBy: ['subtype'],
+      metrics: { count: { op: 'count' } }
+    })
+    const id = await brain.add({
+      data: 'doc-move',
+      type: NounType.Document,
+      subtype: 'draft'
+    })
+    await brain.update({ id, subtype: 'published' })
+
+    const groups = await brain.queryAggregate('by_subtype')
+    const byKey = Object.fromEntries(
+      groups.map((g) => [String(g.groupKey.subtype), g.metrics.count])
+    )
+    expect(byKey['published']).toBe(1)
+    // The old group must be gone or zero — never still counting the entity.
+    expect(byKey['draft'] ?? 0).toBe(0)
+  })
+
+  it('source.where on a reserved field filters instead of matching nothing', async () => {
+    brain.defineAggregate({
+      name: 'notes_only',
+      source: { type: NounType.Document, where: { subtype: 'note' } },
+      groupBy: ['team'],
+      metrics: { count: { op: 'count' } }
+    })
+    await brain.add({
+      data: 'n1',
+      type: NounType.Document,
+      subtype: 'note',
+      metadata: { team: 'alpha' }
+    })
+    await brain.add({
+      data: 'd1',
+      type: NounType.Document,
+      subtype: 'draft',
+      metadata: { team: 'alpha' }
+    })
+
+    const groups = await brain.queryAggregate('notes_only')
+    expect(groups).toHaveLength(1)
+    expect(groups[0].metrics.count).toBe(1) // the note, never the draft
+  })
+
+  it('removeMany refuses empty/invalid selectors loudly', async () => {
+    const id = await brain.add({ data: 'keep-me', type: NounType.Document })
+
+    // Bare array passed positionally — the classic silent no-op.
+    await expect(
+      brain.removeMany([id] as unknown as Parameters<typeof brain.removeMany>[0])
+    ).rejects.toThrow(/bare array/)
+    // Empty selector object.
+    await expect(
+      brain.removeMany({} as Parameters<typeof brain.removeMany>[0])
+    ).rejects.toThrow(/requires a selector/)
+    // Explicit empty id list.
+    await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/)
+
+    // Nothing was deleted by any of the refused calls.
+    expect(await brain.get(id)).toBeTruthy()
+  })
+
+  it('find() accepts both flattened and metadata.-prefixed where spellings', async () => {
+    await brain.add({
+      data: 'nested-doc',
+      type: NounType.Document,
+      metadata: { entry: { title: 'T1' }, classifier: { contextHints: { vfsPath: '/n/a.md' } } }
+    })
+    await brain.flush()
+
+    const flat = await brain.find({
+      type: NounType.Document,
+      where: { 'entry.title': 'T1' },
+      limit: 10
+    })
+    const prefixed = await brain.find({
+      type: NounType.Document,
+      where: { 'metadata.entry.title': 'T1' },
+      limit: 10
+    })
+    const deepPrefixed = await brain.find({
+      type: NounType.Document,
+      where: { 'metadata.classifier.contextHints.vfsPath': '/n/a.md' },
+      limit: 10
+    })
+    expect(flat).toHaveLength(1)
+    expect(prefixed).toHaveLength(1)
+    expect(prefixed[0].id).toBe(flat[0].id)
+    expect(deepPrefixed).toHaveLength(1)
+  })
+})
diff --git a/tests/regression/metadata-index-cleanup.unit.test.ts b/tests/regression/metadata-index-cleanup.unit.test.ts
index 266b9a4d..0984d727 100644
--- a/tests/regression/metadata-index-cleanup.unit.test.ts
+++ b/tests/regression/metadata-index-cleanup.unit.test.ts
@@ -205,10 +205,10 @@ describe('Metadata index cleanup after remove / removeMany', () => {
       }
     })
 
-    it('handles empty ids array gracefully', async () => {
-      const result = await brain.removeMany({ ids: [] })
-      expect(result.successful).toHaveLength(0)
-      expect(result.failed).toHaveLength(0)
+    it('refuses an empty ids array loudly (a silent no-op is not "graceful")', async () => {
+      // 8.8.2: an empty selector used to resolve successfully having deleted
+      // NOTHING — the caller believed the delete happened. Now it throws.
+      await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/)
     })
 
     it('handles large batch (> 1 chunk) without leaving stale index entries', async () => {
diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts
index b2cc7f09..58b25744 100644
--- a/tests/unit/brainy/batch-operations.test.ts
+++ b/tests/unit/brainy/batch-operations.test.ts
@@ -533,8 +533,10 @@ describe('Brainy Batch Operations', () => {
       expect(result.successful).toHaveLength(0)
       
       await brain.updateMany({ items: [] })
-      await brain.removeMany({ ids: [] })
-      // Should not throw
+      // removeMany is the exception (8.8.2): an empty id list is a refused
+      // selector, not an empty batch — deleting "nothing" silently was the
+      // bug class (a positional/bare-array call looked identical).
+      await expect(brain.removeMany({ ids: [] })).rejects.toThrow(/ids: \[\]/)
     })
     
     it('should validate batch size limits', async () => {

From a16567d626198765fd26be77a471c0f911a6510b Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 11:18:18 -0700
Subject: [PATCH 75/82] chore(release): 8.8.2

---
 CHANGELOG.md      | 6 ++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a4a97ba3..6b4f1a1c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19)
+
+- fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d)
+- chore: push public docs to the soulcraft.com ingest door on release (42037d0)
+
+
 ### [8.8.1](https://github.com/soulcraftlabs/brainy/compare/v8.8.0...v8.8.1) (2026-07-18)
 
 - fix: O(1) adaptive retention accounting + historyStats fleet audit (6207e48)
diff --git a/package-lock.json b/package-lock.json
index 3f157871..c1d62fc9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.8.1",
+  "version": "8.8.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.8.1",
+      "version": "8.8.2",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index fb467c2e..b43633ac 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.8.1",
+  "version": "8.8.2",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From 300d9f2a16944fe49cbece344df48bc97c4bbfed Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 12:04:39 -0700
Subject: [PATCH 76/82] =?UTF-8?q?feat:=20flush()=20never=20compacts=20?=
 =?UTF-8?q?=E2=80=94=20history=20maintenance=20moves=20to=20close()=20with?=
 =?UTF-8?q?=20bounded=20passes?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

flush() is durability work: it must cost what the current window's
deltas cost, never what the history backlog costs. Under adaptive
retention the byte budget derives from free memory, so bulk-load
pressure shrank the budget exactly at peak write volume and flush paid
actual reclaim inline — a production deployment measured single writes
blocked 25-191s behind reclaim-on-flush.

- flush() no longer calls autoCompactHistory(); close() is THE
  auto-compaction site (already ran there; now alone).
- Every auto pass is time-bounded (CLOSE_COMPACTION_BUDGET_MS = 5s):
  reclamation is oldest-first, so an early stop is a consistent prefix
  and the next pass resumes. Explicit compactHistory() gains an
  optional timeBudgetMs for caller-chosen maintenance windows.
- Documented trade stated where operators read: a long-lived writer
  that never closes accumulates history until its next explicit
  compactHistory() — predictable writes, explicit maintenance.

Pins: flush-never-reclaims + close-reclaims-durably (db-mvcc), bounded
pass stops-then-resumes as a consistent prefix (generationStore unit).
---
 docs/guides/snapshots-and-time-travel.md | 13 +++++--
 src/brainy.ts                            | 48 +++++++++++++++++-------
 src/db/generationStore.ts                |  6 +++
 src/db/types.ts                          |  9 +++++
 src/types/brainy.types.ts                |  9 +++--
 tests/integration/db-mvcc.test.ts        | 32 ++++++++++------
 tests/unit/db/generationStore.test.ts    | 14 +++++++
 7 files changed, 100 insertions(+), 31 deletions(-)

diff --git a/docs/guides/snapshots-and-time-travel.md b/docs/guides/snapshots-and-time-travel.md
index 56c49044..490aecab 100644
--- a/docs/guides/snapshots-and-time-travel.md
+++ b/docs/guides/snapshots-and-time-travel.md
@@ -344,8 +344,12 @@ For per-entity write coordination (rather than whole-store history), the
 ## Keeping history bounded
 
 Under Model-B every write is a generation, so history can grow quickly —
-Brainy auto-compacts on every `flush()`/`close()` under the **`retention`**
-knob (configured on the constructor):
+Brainy auto-compacts at `close()` (time-bounded per pass) under the
+**`retention`** knob (configured on the constructor). Since 8.9.0, `flush()`
+never compacts: flushing is durability work and costs only what the current
+window's writes cost, regardless of history backlog. A long-lived writer that
+never closes keeps its history until its next explicit `compactHistory()` —
+schedule one in your maintenance window if you run bounded retention:
 
 ```typescript
 // Zero-config: ADAPTIVE — keep as much history as free disk/RAM allows,
@@ -359,10 +363,13 @@ new Brainy({ retention: 'all' })
 new Brainy({ retention: { maxGenerations: 1000, maxAge: 7 * 86_400_000, maxBytes: 512 * 1024 ** 2 } })
 ```
 
-Reclaim manually at any time (the same caps):
+Reclaim manually at any time (the same caps, plus an optional per-pass time
+budget for maintenance windows — an early stop is a consistent prefix and the
+next pass resumes):
 
 ```typescript
 await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 24 * 60 * 60 * 1000 })
+await brain.compactHistory({ maxBytes: 512 * 1024 ** 2, timeBudgetMs: 10_000 })
 ```
 
 Compaction never breaks a pinned read — record-sets are reclaimed only when
diff --git a/src/brainy.ts b/src/brainy.ts
index 21eab679..8ba991dd 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -396,6 +396,15 @@ export type IndexFamily = 'vector' | 'metadata' | 'graph'
  */
 const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000
 
+/**
+ * Time budget for the auto-compaction pass at close() (8.9.0). Bounds how long
+ * a clean shutdown spends reclaiming history backlog — an early stop is a
+ * consistent prefix and the next close/explicit pass resumes. Explicit
+ * `compactHistory()` calls are unbounded unless the caller passes their own
+ * `timeBudgetMs` (maintenance windows choose their own budgets).
+ */
+const CLOSE_COMPACTION_BUDGET_MS = 5_000
+
 /**
  * The main Brainy class - Clean, Beautiful, Powerful
  * REAL IMPLEMENTATION - No stubs, no mocks
@@ -8362,19 +8371,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
 
   /**
    * @description Run history compaction under the resolved `retention` policy
-   * when `autoCompact` is on (the default). Invoked from `flush()` and
-   * `close()` so generational record-sets cannot accumulate unbounded across a
-   * long-lived writer's lifetime.
+   * when `autoCompact` is on (the default). Invoked from `close()` ONLY
+   * (8.9.0) — flush() is durability work and never pays maintenance costs; a
+   * production deployment measured reclaim-on-flush blocking single writes
+   * for 25-191s under memory pressure. Long-lived writers that never close
+   * accumulate history until their next explicit `compactHistory()` — the
+   * documented trade: predictable writes, explicit maintenance.
    *
    * - `'all'` → returns without reclaiming (index compaction for speed still
    *   runs elsewhere; history is decoupled and kept).
    * - `'adaptive'` → reclaim oldest-unpinned history down to the byte budget.
    * - `'explicit'` → apply the supplied `maxGenerations`/`maxAge`/`maxBytes` caps.
    *
+   * Every auto pass is TIME-BOUNDED ({@link CLOSE_COMPACTION_BUDGET_MS}) so a
+   * large backlog can never stall a clean shutdown; the next pass resumes.
    * Read-only instances and an explicit `autoCompact: false` skip silently.
    * Pinned generations are never reclaimed ({@link GenerationStore.compact}).
    * Failures are logged and swallowed — compaction is housekeeping and must
-   * never fail a flush or a clean shutdown.
+   * never fail a clean shutdown.
    */
   private async autoCompactHistory(): Promise<void> {
     // Nothing to compact on a read-only instance or before init wired up the
@@ -8390,12 +8404,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       if (policy.mode === 'adaptive') {
         const budget = this.adaptiveHistoryBudgetBytes(policy.budgetBytes)
         if (budget === Infinity) return // no pressure signal → keep everything this pass
-        await this.generationStore.compact({ maxBytes: budget })
+        await this.generationStore.compact({
+          maxBytes: budget,
+          timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS
+        })
       } else {
         await this.generationStore.compact({
           maxGenerations: policy.maxGenerations,
           maxAge: policy.maxAge,
-          maxBytes: policy.maxBytes
+          maxBytes: policy.maxBytes,
+          timeBudgetMs: CLOSE_COMPACTION_BUDGET_MS
         })
       }
     } catch (error) {
@@ -10540,12 +10558,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       this.generationStore.persistCounterNow()
     ])
 
-    // 6. Auto-compact generational history per config.retention (default on).
-    //    Runs after the flush so durable state is in place; respects live
-    //    Db pins and an explicit autoCompact: false.
-    await this.autoCompactHistory()
+    // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY
+    // work — it must cost what this window's deltas cost, never what the
+    // history backlog costs. Auto-compaction (a MAINTENANCE concern) runs at
+    // close() and via explicit compactHistory(); a production deployment
+    // measured reclaim-on-flush stalling single writes for 25-191s under
+    // memory pressure, which is exactly the class this separation ends.
 
-    // 7. Stamp the entity tree: which source generation the canonical tree
+    // 6. Stamp the entity tree: which source generation the canonical tree
     //    reflects + the rollup invariants that verify it whole (the counters
     //    persisted in step 1). Written at flush boundaries — the tree tracks
     //    every commit by construction, so the stamp is a durable checkpoint,
@@ -16173,8 +16193,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
 
     // Phase 0b: Auto-compact generational history per config.retention (default
-    // on) BEFORE the generation store closes below. Respects live Db pins and
-    // an explicit autoCompact: false; no-op on read-only instances.
+    // on) BEFORE the generation store closes below. This is THE auto-compaction
+    // site (8.9.0 — flush() never compacts): time-bounded per pass, respects
+    // live Db pins and an explicit autoCompact: false; no-op on read-only
+    // instances.
     await this.autoCompactHistory()
 
     // Phase 1: Flush ALL components in parallel to persist buffered data
diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts
index dd70f7aa..4e3738d9 100644
--- a/src/db/generationStore.ts
+++ b/src/db/generationStore.ts
@@ -2220,6 +2220,11 @@ export class GenerationStore {
       const maxAge = options?.maxAge
       const maxBytes = options?.maxBytes
       const ageCutoff = maxAge !== undefined ? Date.now() - maxAge : undefined
+      // Bounded maintenance pass (8.9.0): stop reclaiming once the budget is
+      // spent. Safe mid-loop — reclamation is oldest-first, so an early stop
+      // leaves a consistent contiguous prefix and the next pass resumes.
+      const deadline =
+        options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined
       const noCaps =
         maxGenerations === undefined && maxAge === undefined && maxBytes === undefined
 
@@ -2233,6 +2238,7 @@ export class GenerationStore {
       for (const gen of [...this.committedGensAsc()]) {
         // Pins are always exempt: never reclaim a generation a live pin needs.
         if (gen > minPinned) break // committedGensAsc ascending → nothing newer is eligible either
+        if (deadline !== undefined && Date.now() >= deadline) break // budget spent — resume next pass
         const delta = await this.getDelta(gen)
         if (!noCaps) {
           const violatesCount = maxGenerations !== undefined && remainingCount > maxGenerations
diff --git a/src/db/types.ts b/src/db/types.ts
index 521396b8..4c8a4957 100644
--- a/src/db/types.ts
+++ b/src/db/types.ts
@@ -176,6 +176,15 @@ export interface CompactHistoryOptions {
    * of each surviving generation's serialized record set (`GenerationDelta.bytes`).
    */
   maxBytes?: number
+  /**
+   * Stop reclaiming after this many milliseconds even if caps are still
+   * exceeded (8.9.0). Compaction is maintenance — a bounded pass keeps
+   * `close()` (and any explicit maintenance window) from stalling on a large
+   * backlog; the next pass resumes where this one stopped (reclamation is
+   * oldest-first, so an early stop is always a consistent prefix). Unset =
+   * run to completion.
+   */
+  timeBudgetMs?: number
 }
 
 /**
diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts
index c2344375..1ec4c4c3 100644
--- a/src/types/brainy.types.ts
+++ b/src/types/brainy.types.ts
@@ -1737,7 +1737,10 @@ export interface BrainyConfig {
    * Under Model-B EVERY write (`transact()` AND single-op `add`/`update`/
    * `remove`/`relate`) produces an immutable generation record-set serving
    * historical reads (`asOf()`, pinned `Db` values). Without compaction those
-   * accumulate, so Brainy **auto-compacts on every `flush()` and `close()`**.
+   * accumulate, so Brainy **auto-compacts at `close()`** (time-bounded per
+   * pass; 8.9.0 removed compaction from `flush()` — flush is durability work
+   * and never pays maintenance costs). A long-lived writer that never closes
+   * accumulates history until its next explicit `compactHistory()` call.
    * Live `Db` pins are ALWAYS exempt from reclamation, in every mode.
    *
    * Modes:
@@ -1754,7 +1757,7 @@ export interface BrainyConfig {
    *   the oldest unpinned generations while ANY supplied cap is exceeded
    *   (predictable ops). `maxAge` in ms; `maxBytes` total history bytes.
    *
-   * `autoCompact: false` disables the automatic flush/close compaction (manage
+   * `autoCompact: false` disables the automatic close() compaction (manage
    * manually via `brain.compactHistory()`). `budgetBytes` is the settable
    * adaptive byte budget a coordinator drives (also via `brain.setRetentionBudget()`).
    * Long-term archives belong in `db.persist(path)` snapshots, which compaction
@@ -1772,7 +1775,7 @@ export interface BrainyConfig {
         maxBytes?: number
         /** Adaptive byte budget for this brain, driven by a coordinator (e.g. cor). */
         budgetBytes?: number
-        /** Run compaction automatically on flush()/close() (default: true). */
+        /** Run compaction automatically at close() (default: true; 8.9.0 — flush() never compacts). */
         autoCompact?: boolean
       }
 
diff --git a/tests/integration/db-mvcc.test.ts b/tests/integration/db-mvcc.test.ts
index 10a158ae..959d0053 100644
--- a/tests/integration/db-mvcc.test.ts
+++ b/tests/integration/db-mvcc.test.ts
@@ -1280,24 +1280,32 @@ describe('8.0 Db API — generational MVCC', () => {
     await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError)
   })
 
-  it('Model-B retention — setRetentionBudget drives adaptive reclaim on flush; live data intact', async () => {
-    // Default brain → ADAPTIVE retention. A coordinator (e.g. cor's ResourceManager)
-    // pushes a byte budget via setRetentionBudget(); auto-compaction on flush() reclaims
-    // oldest history down toward it. Each update's before-image carries the full prior
-    // 384-dim vector (~KBs), so ~13 generations far exceed a few-KB budget.
-    const { brain } = await openFsBrain()
+  it('Model-B retention — flush() NEVER compacts (8.9.0); adaptive reclaim runs at close()', async () => {
+    // Default brain → ADAPTIVE retention with a driven byte budget far below
+    // the accumulated history (~13 generations of full-vector before-images).
+    // The 8.9.0 law: flush() is durability-only — it must not reclaim even
+    // when the budget is exceeded (reclaim-on-flush blocked production writes
+    // for 25-191s). Maintenance runs at close(), time-bounded.
+    const { brain, dir } = await openFsBrain()
     const a = uid('ret-budget')
     await brain.add({ id: a, type: NounType.Document, data: 'v0', vector: vec(1), metadata: { v: 0 } })
     for (let v = 1; v <= 12; v++) await brain.update({ id: a, metadata: { v } })
 
     brain.setRetentionBudget(6000) // ~6 KB — well below the accumulated history
-    await brain.flush() // group-commit + adaptive auto-compaction under the budget
+    await brain.flush()
 
-    // History was reclaimed (the horizon advanced past the oldest generations)…
-    expect(generationStoreOf(brain).horizon()).toBeGreaterThan(0)
-    await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError)
-    // …but the budget reclaims HISTORY only — the live record is untouched.
-    expect((await brain.get(a))?.metadata?.v).toBe(12)
+    // flush() paid durability only: nothing reclaimed, all history readable.
+    expect(generationStoreOf(brain).horizon()).toBe(0)
+    const probe = await brain.asOf(1) // readable proves nothing was reclaimed…
+    await probe.release() // …and MUST be released: a held pin would (correctly)
+    // protect every newer generation through the close() compaction below.
+    await brain.close() // ← THE auto-compaction site now
+
+    // close() reclaimed under the budget; live record intact; horizon durable.
+    const { brain: reopened } = await openFsBrain(dir)
+    expect(generationStoreOf(reopened).horizon()).toBeGreaterThan(0)
+    await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError)
+    expect((await reopened.get(a))?.metadata?.v).toBe(12)
   })
 
   // ==========================================================================
diff --git a/tests/unit/db/generationStore.test.ts b/tests/unit/db/generationStore.test.ts
index 611a97d7..5b667415 100644
--- a/tests/unit/db/generationStore.test.ts
+++ b/tests/unit/db/generationStore.test.ts
@@ -488,6 +488,20 @@ describe('db/GenerationStore', () => {
       expect(result.removedGenerations).toBe(2)
       store.release(2)
     })
+
+    it('timeBudgetMs bounds a pass; the next pass resumes the same prefix', async () => {
+      await manyGens(4)
+      // A spent budget (0ms) stops before reclaiming anything — an early stop
+      // is a consistent prefix, never a partial generation.
+      const bounded = await store.compact({ timeBudgetMs: 0 })
+      expect(bounded.removedGenerations).toBe(0)
+      expect(bounded.horizon).toBe(0)
+      // The next (unbounded) pass picks up exactly where the bounded one
+      // stopped and completes the same work.
+      const resumed = await store.compact()
+      expect(resumed.removedGenerations).toBe(4)
+      expect(resumed.horizon).toBe(4)
+    })
   })
 
   // ==========================================================================

From 70e4bc8a794aaa53dd28f78e3f28e9ec4cdb0644 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 12:52:24 -0700
Subject: [PATCH 77/82] =?UTF-8?q?fix:=20release=20drains=20in-flight=20wri?=
 =?UTF-8?q?ter-lock=20heartbeat=20=E2=80=94=20no=20phantom=20lock=20after?=
 =?UTF-8?q?=20unlink?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

clearInterval() stops future heartbeat ticks but not one already in
flight: a straggler tick past its ownership guards could land its
atomic lock rewrite AFTER releaseWriterLock()'s unlink, re-creating
the lock file as a phantom that blocks the next writer until the
stale TTL expires (~60s) — the pool-eviction reopen case. Found as an
ENOENT heartbeat warning during benchmark teardown; the quiet variant
is the harmful one.

releaseWriterLock() now awaits the in-flight tick (tracked per tick,
self-clearing) before reading/unlinking, so a straggler's write always
lands BEFORE the unlink and gets removed with everything else. The
heartbeat's ENOENT is also now benign-by-contract (lock or directory
removed under us — the next acquire recreates it); other errors stay
loud.

Pin: straggler-past-guards simulation — lock file absent after close,
directory immediately claimable (fails on the undrained code).
---
 src/storage/adapters/fileSystemStorage.ts     | 31 +++++++++++++-
 .../integration/multi-process-safety.test.ts  | 40 +++++++++++++++++++
 2 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts
index 3c95f9e7..5eb4785a 100644
--- a/src/storage/adapters/fileSystemStorage.ts
+++ b/src/storage/adapters/fileSystemStorage.ts
@@ -96,6 +96,14 @@ export class FileSystemStorage extends BaseStorage {
   private static readonly WRITER_STALE_THRESHOLD_MS = 60_000
   private writerLockHeartbeat?: NodeJS.Timeout
   private writerLockInfo?: WriterLockInfo
+  /**
+   * The currently-executing heartbeat refresh, if any. `releaseWriterLock()`
+   * awaits it before unlinking: clearInterval() stops FUTURE ticks but not a
+   * tick already in flight, and a straggler landing after the unlink would
+   * RE-CREATE the lock file — a phantom lock blocking the next writer until
+   * the stale TTL expires (the pool-eviction reopen case).
+   */
+  private writerHeartbeatInFlight?: Promise<void>
 
   // Flush-request RPC state. The writer polls `locks/_flush_requests/` for
   // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after
@@ -1880,8 +1888,18 @@ export class FileSystemStorage extends BaseStorage {
     // Heartbeat — rewrite lastHeartbeat every WRITER_HEARTBEAT_MS so other
     // processes can tell a live writer from one that crashed without releasing.
     this.writerLockHeartbeat = setInterval(() => {
-      this.refreshWriterLockHeartbeat().catch((err) => {
-        console.warn('[brainy] Failed to refresh writer lock heartbeat:', err)
+      const tick = this.refreshWriterLockHeartbeat().catch((err) => {
+        // ENOENT = the lock (or its directory) vanished mid-refresh — the
+        // store was released or removed under us; the next acquire recreates
+        // it. Benign by construction; anything else stays loud.
+        if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
+          console.warn('[brainy] Failed to refresh writer lock heartbeat:', err)
+        }
+      })
+      this.writerHeartbeatInFlight = tick.finally(() => {
+        if (this.writerHeartbeatInFlight === tick) {
+          this.writerHeartbeatInFlight = undefined
+        }
       })
     }, FileSystemStorage.WRITER_HEARTBEAT_MS)
     if (typeof this.writerLockHeartbeat.unref === 'function') {
@@ -1913,6 +1931,15 @@ export class FileSystemStorage extends BaseStorage {
       clearInterval(this.writerLockHeartbeat)
       this.writerLockHeartbeat = undefined
     }
+    // Drain an in-flight heartbeat tick BEFORE unlinking: clearInterval stops
+    // future ticks only, and a straggler write landing after the unlink would
+    // re-create the lock as a phantom (blocking the next writer until the
+    // stale TTL). After the drain, any refresh is either fully landed (we
+    // unlink its output below) or not started (it sees writerLockInfo
+    // undefined and returns).
+    if (this.writerHeartbeatInFlight) {
+      await this.writerHeartbeatInFlight
+    }
     if (!this.writerLockInfo) {
       return
     }
diff --git a/tests/integration/multi-process-safety.test.ts b/tests/integration/multi-process-safety.test.ts
index 0eae92b1..592d7969 100644
--- a/tests/integration/multi-process-safety.test.ts
+++ b/tests/integration/multi-process-safety.test.ts
@@ -153,6 +153,46 @@ describe('Multi-process safety + read-only mode', () => {
       expect(err.lockInfo?.pid).toBe(otherPid)
     })
 
+    it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => {
+      // The race (8.9.0): clearInterval stops FUTURE heartbeat ticks, but a
+      // tick already in flight could land its lock rewrite AFTER release's
+      // unlink — re-creating the lock as a phantom that blocks the next
+      // writer until the stale TTL. Simulate the in-flight tick explicitly
+      // and prove release waits for it.
+      writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
+      await writer.init()
+      const storage: any = (writer as any).storage
+
+      // An in-flight refresh that is ALREADY PAST its ownership guards
+      // (captured the lock info before release ran) and lands its atomic
+      // rewrite slowly — the exact straggler shape; absent the drain it
+      // writes after the unlink.
+      const { join: joinPath } = await import('node:path')
+      const capturedInfo = { ...storage.writerLockInfo }
+      const lockPath = joinPath(dir, 'locks', '_writer.lock')
+      const slowTick = (async () => {
+        await new Promise((r) => setTimeout(r, 100))
+        await storage.writeFileAtomic(
+          lockPath,
+          JSON.stringify({ ...capturedInfo, lastHeartbeat: new Date().toISOString() })
+        )
+      })()
+      storage.writerHeartbeatInFlight = slowTick.catch(() => {})
+
+      await writer.close() // → releaseWriterLock must drain slowTick first
+      await slowTick.catch(() => {}) // both paths fully settled either way
+      writer = null
+
+      const { existsSync } = await import('node:fs')
+      const { join } = await import('node:path')
+      expect(existsSync(join(dir, 'locks', '_writer.lock'))).toBe(false)
+
+      // And the directory is immediately claimable — no stale-TTL wait.
+      const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
+      await expect(next.init()).resolves.toBeUndefined()
+      await next.close()
+    })
+
     it('allows a second in-process writer with a warning (same PID)', async () => {
       // Two Brainy instances in the same Node process: not the dangerous
       // cross-process case. Should succeed (with a console warning).

From 5cabd784f4e422328c01ab757b928dfb4fc3e194 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 13:35:04 -0700
Subject: [PATCH 78/82] docs: measured performance envelopes v1 (per-op p50/p95
 at 1k and 10k, pure-JS floor)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

First edition of the per-release performance-envelope contract: every
number measured against the built dist on stated hardware, never
projected. Sub-0.1ms get/related (adjacency O(degree), scale-flat),
1-9ms indexed metadata finds, ~178ms semantic (query embedding
dominates), ~167ms durability-priced single-op writes flat across
scale, 8-45ms steady-state flush independent of history backlog
(the 8.9.0 change). Two weak spots stated honestly: addMany commits
per-item today (batched chunk commits belong to the unified-commit
roadmap), and pure-JS warm open grows with corpus (4.9s at 10k) —
the native accelerator's reason to exist. Refresh rule: any release
touching a measured path re-measures in the same release.
---
 RELEASES.md                   | 45 +++++++++++++++++++
 docs/performance-envelopes.md | 83 +++++++++++++++++++++++++++++++++++
 2 files changed, 128 insertions(+)
 create mode 100644 docs/performance-envelopes.md

diff --git a/RELEASES.md b/RELEASES.md
index d1374d5a..7799c6f6 100644
--- a/RELEASES.md
+++ b/RELEASES.md
@@ -8,8 +8,53 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so
 - Debugging data, query, or storage behaviour
 - A new Brainy feature is available that you want to adopt
 
+## Removed APIs — 7.x → 8.x (the complete ledger)
+
+Every public API removed at the 8.0 major, with its sanctioned replacement. If your code
+still calls a left-column name on 8.x it throws (or the config key is rejected) — the
+replacement is always a one-line change. (Standing contract from 8.9.0 forward: removals
+happen only at majors, after ≥1 minor of loud runtime deprecation naming the replacement.)
+
+| Removed (7.x) | Replacement (8.x) |
+|---|---|
+| `brain.search(query, k)` | `find({ query })` — semantic; `find({ query, searchMode })` for hybrid |
+| `brain.getRelations({...})` | `related(id, opts)` for adjacency; `find({ connected: {...} })` for scoped traversal |
+| `brain.neural()` clustering | `find({ vector })` + aggregation `GROUP BY` |
+| `Db.search()` | `db.find({ vector })` |
+| Pre-8.0 storage path aliases (`directory`, `basePath`, …) | one `storage.path` key (old aliases throw) |
+| Reserved keys inside `metadata` bags (silently remapped in 7.x) | top-level params (`subtype`, `visibility`, `confidence`, `weight`, …) — reserved-in-bag throws |
+| 7.x COW branches layout (`branches/main/`) | generational MVCC (`asOf()`, `now()`, `db.persist(path)`) — on-disk migration is automatic at first 8.x open |
+
+The fork/snapshot family (`brain.snapshot()`, `createSnapshot()`, `restoreSnapshot()`)
+is sometimes cited as a 7.x removal — those methods never existed on 7.x; the 8.0 Db API
+(`asOf`/`persist`/`restore({confirm})`) is their first real implementation.
+
 ---
 
+## v8.9.0 — 2026-07-19 (flush is durability-only: history maintenance moves to close())
+
+The write path stops paying maintenance costs — the last structural piece of the
+flush-storm class (a production deployment measured single writes blocked 25–191s behind
+history reclaim running inline on flush under memory pressure):
+
+- **`flush()` never compacts history.** It persists the current window's deltas and
+  nothing else — its cost no longer depends on history backlog or retention mode, in any
+  configuration. **`close()` is the auto-compaction site** (time-bounded per pass, ~5s;
+  an early stop is a consistent prefix and the next pass resumes).
+- **`compactHistory()` gains `timeBudgetMs`** — bound your own maintenance windows; the
+  same resumable-prefix guarantee applies.
+- **The documented trade**: a long-lived writer that never closes accumulates history
+  until its next explicit `compactHistory()`. Predictable writes, explicit maintenance.
+  If you run bounded retention on an always-on service, schedule a periodic
+  `compactHistory({ ...caps, timeBudgetMs })` in your maintenance window.
+- **New public doc: `docs/performance-envelopes.md`** — measured per-op envelopes
+  (p50/p95 at stated scales, hardware, and backend, with the measuring script cited).
+  Refresh rule going forward: any release touching a measured path re-runs that op's
+  benchmark and updates the envelope in the same release.
+- **New in this file: the Removed APIs 7.x→8.x table** (top of this document) — every
+  removal with its sanctioned replacement, one place, per the engine-currency contract.
+  Standing from here: removals only at majors, after ≥1 minor of loud runtime deprecation.
+
 ## v8.8.2 — 2026-07-19 (one field-resolution law: reserved-field aggregates stop drifting)
 
 Four fixes from a consumer conformance audit, all rooted in the same disease — two field-resolution
diff --git a/docs/performance-envelopes.md b/docs/performance-envelopes.md
new file mode 100644
index 00000000..d29677e3
--- /dev/null
+++ b/docs/performance-envelopes.md
@@ -0,0 +1,83 @@
+---
+title: Performance Envelopes
+slug: guides/performance-envelopes
+public: true
+category: guides
+template: guide
+order: 40
+description: Measured per-operation latency envelopes at stated scales — what to expect, on what hardware, and exactly how each number was produced.
+next:
+  - guides/find-limits
+---
+
+# Performance Envelopes
+
+Every number on this page is **measured, never projected** — produced by the script
+cited at the bottom, against the built package (the artifact you install), on the stated
+hardware. Each entry says what was measured, at what scale, on which storage backend.
+When a release touches a measured path, that operation is re-measured and this page
+updates in the same release.
+
+Two scopes to keep straight:
+
+- **These envelopes are the pure-JS engine** (no native accelerator registered) on
+  filesystem storage. This is the floor every deployment gets from `npm install` alone.
+- **Accelerated deployments** (the optional native provider) publish their own numbers —
+  this page never claims them.
+
+## Read operations
+
+Reads are where the architecture pays off: after the write path has done its indexing
+work, queries answer from purpose-built indexes without scanning.
+
+| Operation | 1,000 entities | 10,000 entities | Notes |
+|---|---|---|---|
+| `get(id)` (warm) | p50 < 0.1ms | p50 < 0.1ms | served from cache/metadata index |
+| `find` (metadata: indexed equality + range, limit 100) | p50 1.0ms · p95 1.8ms | p50 7.0ms · p95 8.9ms | column-store bitmap paths |
+| `related(id)` (per-node adjacency) | p50 < 0.1ms · p95 0.2ms | p50 < 0.1ms | LSM adjacency index — O(degree), scale-independent |
+| `find` (semantic: embed + HNSW, 1k docs) | p50 178ms · p95 393ms | — | dominated by WASM query embedding (measured on a machine under concurrent load — treat the p95 as an upper bound); the vector search itself is single-digit ms |
+
+## Write operations
+
+Under Model-B **every write is its own durable generation** — a single-op `add` pays
+serialization, before-image staging, and fsync before it acks. That durability is priced
+into the write path visibly, by design:
+
+| Operation | 1,000 entities | 10,000 entities | Notes |
+|---|---|---|---|
+| `add` (single-op) | p50 167ms · p95 171ms | p50 165ms · p95 172ms | full durable generation per write — flat across scale |
+| `addMany` (bulk) | ~163ms/entity | ~187ms/entity | **currently per-item commits** — see the honest note below |
+| `relateMany` | ~0.8ms/edge | ~0.9ms/edge | edges batch efficiently today |
+| `flush` (steady-state, 1 pending write) | p50 8ms · p95 10ms | p50 45ms · p95 52ms | durability-only since 8.9.0 — cost no longer depends on history backlog or retention mode |
+
+**The honest note on bulk writes:** `addMany` today commits each item as its own
+generation (the same durability as single-op `add`, serialized by the single-writer
+lock), so bulk-load cost is N × single-op cost. Batched chunk commits (one generation
+and one fsync window per chunk, as `removeMany` already does) are designed into the
+unified-commit work on the current roadmap. Until that ships, size bulk imports
+accordingly — 10k entities is minutes, not seconds, on filesystem storage.
+
+## Open / close
+
+| Operation | 1,000 entities | 10,000 entities | Notes |
+|---|---|---|---|
+| `open` (empty store) | ~560ms | ~190ms | includes embedder initialization |
+| `open` (warm, populated, clean shutdown) | 763ms | 4.9s | pure-JS vector index load dominates and grows with entity count; the native accelerator exists precisely to remove this |
+| `close` | bounded | bounded | auto-compaction pass is time-bounded (~5s max) since 8.9.0 |
+
+A store that was NOT cleanly closed pays index rebuilds on top of the warm-open
+number (tens of seconds at 10k) — clean shutdown is worth engineering for.
+
+## How these were produced
+
+- **Hardware**: Intel Core i9-14900HX (32 threads), 62GB RAM, NVMe, Linux, Node v22.
+- **Backend**: `storage: { type: 'filesystem' }`, pure JS (no native providers).
+- **Embeddings**: deterministic stub for non-semantic ops (isolates engine cost);
+  the real WASM embedder for the semantic row (that's what you'll run).
+- **Method**: p50/p95 over 50–200 samples per op against the built `dist/`;
+  the measuring script ships in the repo history and re-runs per release.
+
+Numbers on different hardware will differ; the *shape* (sub-2ms indexed reads,
+~160ms embedding-bound semantic queries, durability-priced writes) is the envelope
+you should hold your deployment against. If your measurements diverge from these
+shapes by an order of magnitude, something is wrong — file it.

From d08679fc843d31add8adc4d23f3b6e4190847423 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 14:02:25 -0700
Subject: [PATCH 79/82] chore(release): 8.9.0

---
 CHANGELOG.md      | 7 +++++++
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6b4f1a1c..fd0de54e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [8.9.0](https://github.com/soulcraftlabs/brainy/compare/v8.8.2...v8.9.0) (2026-07-19)
+
+- docs: measured performance envelopes v1 (per-op p50/p95 at 1k and 10k, pure-JS floor) (5cabd78)
+- fix: release drains in-flight writer-lock heartbeat — no phantom lock after unlink (70e4bc8)
+- feat: flush() never compacts — history maintenance moves to close() with bounded passes (300d9f2)
+
+
 ### [8.8.2](https://github.com/soulcraftlabs/brainy/compare/v8.8.1...v8.8.2) (2026-07-19)
 
 - fix: one field-resolution law across aggregation hooks, source.where, removeMany, and find() spellings (945d92d)
diff --git a/package-lock.json b/package-lock.json
index c1d62fc9..fb9262e9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.8.2",
+  "version": "8.9.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@soulcraft/brainy",
-      "version": "8.8.2",
+      "version": "8.9.0",
       "license": "MIT",
       "dependencies": {
         "@msgpack/msgpack": "^3.1.2",
diff --git a/package.json b/package.json
index b43633ac..7366ce98 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "8.8.2",
+  "version": "8.9.0",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",

From f8e6da2b6603e52e12ea35983c4e7a122921d790 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 14:54:36 -0700
Subject: [PATCH 80/82] =?UTF-8?q?feat:=20scanFacts=20liveness=20contract?=
 =?UTF-8?q?=20=E2=80=94=20first=20batch=20or=20loud=20failure=20within=20a?=
 =?UTF-8?q?=20documented=20bound?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Stage-2 D1 contract item (co-frozen): a fact scan may be slow, never
silent. batches() now races its FIRST pull against
SCANFACTS_FIRST_BATCH_MS (10s, exported; test-overridable) — a wedged
or unreadably slow store produces a loud abort naming the contract
instead of a consumer hanging indistinguishably from progress (the
production shape: a heal against a generations-backlogged brain
wedged silently on the first segment read).

Only the first pull is raced: the bound is time-to-first-batch (proof
the producer is alive), not per-batch pacing, and it runs only while
a pull is pending — consumer think-time between pulls never counts
against the producer (pinned).

Three pins: wedged-store loud failure within the bound, healthy scan
untouched end-to-end, slow-consumer immunity.
---
 src/db/factLog.ts              | 52 ++++++++++++++++++++++++++++++++--
 src/index.ts                   |  1 +
 tests/unit/db/fact-log.test.ts | 48 +++++++++++++++++++++++++++++++
 3 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/src/db/factLog.ts b/src/db/factLog.ts
index 4c5e95fd..94e79700 100644
--- a/src/db/factLog.ts
+++ b/src/db/factLog.ts
@@ -102,12 +102,26 @@ export interface FactScanBatch {
   segmentId: string
 }
 
+/**
+ * Liveness bound on a scan's FIRST batch (Stage-2 co-freeze, D1 contract):
+ * `batches()` must yield its first batch — or fail loudly — within this many
+ * ms of the first pull. A backlogged or damaged store may be SLOW, but it may
+ * never be SILENT: a consumer awaiting the first batch is otherwise
+ * indistinguishable from a wedge (the exact failure shape a production heal
+ * hit against a generations-backlogged brain).
+ */
+export const SCANFACTS_FIRST_BATCH_MS = 10_000
+
 /** The telemetry a scan OPEN returns (frozen shape). */
 export interface FactScanHandle {
   headGeneration: number
   segmentCount: number
   approxFactCount: number
-  /** Ordered batches; a detected gap aborts LOUDLY, never a silent skip. */
+  /**
+   * Ordered batches; a detected gap aborts LOUDLY, never a silent skip.
+   * Liveness contract: the FIRST batch resolves or rejects within
+   * {@link SCANFACTS_FIRST_BATCH_MS} of the first pull — never a silent hang.
+   */
   batches: () => AsyncGenerator<FactScanBatch>
   /** Close telemetry — the invariant cross-check, valid after iteration ends. */
   summary: () => { factsYielded: number; segmentsRead: number }
@@ -440,6 +454,8 @@ export class FactLog {
     toGeneration?: number
     kinds?: Array<'noun' | 'verb'>
     batchSize?: number
+    /** Test override for the first-batch liveness bound (default {@link SCANFACTS_FIRST_BATCH_MS}). */
+    firstBatchTimeoutMs?: number
   }): FactScanHandle {
     const from = options?.fromGeneration ?? 1
     const to = options?.toGeneration ?? this.head
@@ -514,11 +530,43 @@ export class FactLog {
       }
     }
 
+    // Liveness wrapper: the FIRST pull races the contract deadline. Only the
+    // first — the bound is time-to-first-batch (proof the producer is alive),
+    // not per-batch pacing; and it runs only while a pull is actually pending,
+    // so consumer think-time between pulls never counts against the producer.
+    const firstBatchTimeoutMs = options?.firstBatchTimeoutMs ?? SCANFACTS_FIRST_BATCH_MS
+    async function* batchesWithLiveness(this: void): AsyncGenerator<FactScanBatch> {
+      const inner = batches()
+      let timer: NodeJS.Timeout | undefined
+      try {
+        const deadline = new Promise<never>((_, reject) => {
+          timer = setTimeout(
+            () =>
+              reject(
+                new Error(
+                  `fact log: scanFacts produced no first batch within ${firstBatchTimeoutMs}ms ` +
+                    `(liveness contract) — the store is wedged or unreadably slow; aborting scan LOUDLY ` +
+                    `instead of hanging the consumer.`
+                )
+              ),
+            firstBatchTimeoutMs
+          )
+          timer.unref?.()
+        })
+        const first = await Promise.race([inner.next(), deadline])
+        if (first.done) return
+        yield first.value
+      } finally {
+        clearTimeout(timer)
+      }
+      yield* inner
+    }
+
     return {
       headGeneration: this.head,
       segmentCount: segments.length + (tailSnapshot.length > 0 ? 1 : 0),
       approxFactCount,
-      batches,
+      batches: batchesWithLiveness,
       summary: () => ({ factsYielded, segmentsRead })
     }
   }
diff --git a/src/index.ts b/src/index.ts
index ee01d885..b978b9fd 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -213,6 +213,7 @@ export type {
   CommitFact,
   FactOp,
   FactScanBatch,
+  SCANFACTS_FIRST_BATCH_MS,
   FactScanHandle
 } from './db/factLog.js'
 // The generalized family stamp — which source generation a projection
diff --git a/tests/unit/db/fact-log.test.ts b/tests/unit/db/fact-log.test.ts
index abce2dc9..f1c226cc 100644
--- a/tests/unit/db/fact-log.test.ts
+++ b/tests/unit/db/fact-log.test.ts
@@ -186,4 +186,52 @@ describe('fact log — round-trip, framing, reconcile, rotation, scan', () => {
     await log.sync()
     expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed
   })
+
+  describe('scanFacts liveness contract (Stage-2 D1)', () => {
+    it('a wedged store fails LOUDLY within the first-batch bound — never a silent hang', async () => {
+      // Force a sealed segment (tiny rotateBytes) so the scan must READ from
+      // storage, then wedge that read: the exact production shape (a
+      // backlogged brain whose segment read never returned).
+      const mem: any = new MemoryStorage()
+      await mem.init()
+      const wedgeable = new FactLog(mem, { rotateBytes: 1 })
+      await wedgeable.open(0)
+      await wedgeable.append(fact(1))
+      await wedgeable.append(fact(2)) // second append rotates → seg 1 sealed
+      await wedgeable.sync()
+
+      const realRead = mem.readRawBytes.bind(mem)
+      mem.readRawBytes = (p: string) =>
+        p.includes('facts/seg-') ? new Promise(() => {}) : realRead(p) // hangs forever
+
+      const scan = wedgeable.scanFacts({ firstBatchTimeoutMs: 200 })
+      const started = Date.now()
+      await expect(scan.batches().next()).rejects.toThrow(/no first batch within 200ms/)
+      expect(Date.now() - started).toBeLessThan(5_000) // bound held, not a hang
+    })
+
+    it('a healthy scan is unaffected — first batch well inside the bound, all facts delivered', async () => {
+      for (let g = 1; g <= 5; g++) await log.append(fact(g))
+      await log.sync()
+      const scan = log.scanFacts({ batchSize: 2 })
+      const all: CommitFact[] = []
+      for await (const b of scan.batches()) all.push(...b.facts)
+      expect(all.map((f) => f.generation)).toEqual([1, 2, 3, 4, 5])
+      expect(scan.summary().factsYielded).toBe(5)
+    })
+
+    it('consumer think-time between pulls never counts against the producer', async () => {
+      for (let g = 1; g <= 4; g++) await log.append(fact(g))
+      await log.sync()
+      // Bound tighter than the consumer's pause: only the FIRST pull is
+      // raced, so a slow consumer after batch 1 must not trip the deadline.
+      const gen = log.scanFacts({ batchSize: 2, firstBatchTimeoutMs: 150 }).batches()
+      const first = await gen.next()
+      expect(first.done).toBe(false)
+      await new Promise((r) => setTimeout(r, 400)) // dawdle past the bound
+      const second = await gen.next()
+      expect(second.done).toBe(false)
+      expect((await gen.next()).done).toBe(true)
+    })
+  })
 })

From d8acb3776b2e64db79332cb70fb3b0d7588cef99 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 15:14:27 -0700
Subject: [PATCH 81/82] =?UTF-8?q?feat:=20generation-segment=20store=20?=
 =?UTF-8?q?=E2=80=94=20the=20D1+D3=20packed-tier=20file=20format?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

First stage of the co-frozen D1+D3+repacking unit: the format core,
self-contained under _generations/segments/.

- seg-<firstGen,20pad>.bgs: append-once packs of consecutive
  generations (magic BGS1; frame = u32 len + u32 crc32c + msgpack
  [generation, timestamp, delta, records, flags]; flags reserves
  compressed-payload evolution without a format break). Sealed
  segments are immutable — fold refuses overlap with sealed ranges.
- seg-<firstGen>.idx: DERIVED sidecar (per-generation frame offsets +
  per-id generation postings + checksums); lost/corrupt sidecars
  rebuild from their segment loudly; a damaged segment (frame CRC
  mismatch) fails loudly, never serves wrong bytes.
- manifest.json: the one discovery path — open() reads it and never
  lists the packed backlog (the scan-wedge class's cure); refuses a
  newer manifest version rather than serving partial history.
- D3 semantics: dropSegmentsBelow reclaims WHOLE segments at
  boundaries only and bumps compactedBelow durably; archival-profile
  enforcement stays with the caller per the co-freeze.
- D8 rider: digestThroughPacked(g) — deterministic crc32c chain over
  sealed-segment checksums (+ frame-level prefix mid-segment),
  O(segments), reopen-stable.

Also fixes a cross-adapter contract bug the suite caught: memory
storage's deleteObjectFromPath ignored the raw-bytes store, so
deleteRawObject on a raw-bytes path (fact-log or segment files)
silently no-op'd — deletes now match filesystem unlink semantics.

Six pins. Wiring into GenerationStore (two-tier reads, the repacker,
cold-open manifest path) lands with the rest of the unit before its
release; cortex's fact-record/stamp shapes reconcile the sidecar
keying when they post.
---
 src/db/generationSegments.ts              | 459 ++++++++++++++++++++++
 src/storage/adapters/memoryStorage.ts     |   5 +
 tests/unit/db/generation-segments.test.ts | 150 +++++++
 3 files changed, 614 insertions(+)
 create mode 100644 src/db/generationSegments.ts
 create mode 100644 tests/unit/db/generation-segments.test.ts

diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts
new file mode 100644
index 00000000..0c14b60c
--- /dev/null
+++ b/src/db/generationSegments.ts
@@ -0,0 +1,459 @@
+/**
+ * @module db/generationSegments
+ * @description The generation-segment store — Stage-2 D1+D3+repacking's file
+ * format (co-frozen 2026-07-19; design: the d1-d3-repacking spec).
+ *
+ * Packs CONSECUTIVE cold generations' record-sets (before-images + delta)
+ * into append-once segment files with derived sidecar indexes, so history
+ * scales in SEGMENTS (tens) instead of FILES-PER-GENERATION (hundreds of
+ * thousands), and cold-open reads ONE manifest instead of listing the
+ * backlog. Layout under `_generations/segments/`:
+ *
+ * - `seg-<firstGen, zero-padded 20>.bgs` — magic "BGS1", then one frame per
+ *   generation: `u32 payloadLen | u32 crc32c | msgpack payload`. Payload is
+ *   POSITIONAL: `[generation, timestamp, delta, records[], flags]` with
+ *   records `[kindByte, id, record]`. `flags` reserves encoding evolution
+ *   (bit 0 = compressed payload — v1 always 0; a future writer upgrade,
+ *   never a format break). Sealed segments are IMMUTABLE — the fact log's
+ *   own law, generalized.
+ * - `seg-<firstGen>.idx` — DERIVED sidecar (msgpack): per-generation frame
+ *   offsets (point reads = one ranged read, never a listing) + per-id
+ *   generation postings (per-id chain rebuilds read only what they need).
+ *   Corrupt/missing → rebuilt from its segment in one sequential read,
+ *   loudly.
+ * - `manifest.json` — the segment catalogue + `compactedBelow` (D3's
+ *   horizon marker). Cold-open reads THIS; the packed backlog is never
+ *   listed.
+ *
+ * D3 semantics carried here: bounded-retention reclaim drops WHOLE segments
+ * at boundaries (O(1) per segment, no rewrite); under the archival profile
+ * (`retention: 'all'`) nothing here is ever dropped — folding is the only
+ * transform (re-representation, never deletion).
+ */
+
+import { encode as msgpackEncode, decode as msgpackDecode } from '@msgpack/msgpack'
+import { crc32c } from '../utils/crc32c.js'
+import type { FactLogStorage } from './factLog.js'
+import { prodLog } from '../utils/logger.js'
+
+/** Directory for segment files + manifest, under the generations prefix. */
+export const SEGMENTS_PREFIX = '_generations/segments'
+
+/** Target sealed-segment size (co-freeze proposal; tunable on evidence). */
+export const SEGMENT_TARGET_BYTES = 64 * 1024 * 1024
+
+const MAGIC = new TextEncoder().encode('BGS1')
+const FRAME_PREFIX_BYTES = 8 // u32 payloadLen + u32 crc32c
+const MANIFEST_PATH = `${SEGMENTS_PREFIX}/manifest.json`
+
+/** One generation's fold input — exactly what the live tier holds for it. */
+export interface FoldGeneration {
+  generation: number
+  timestamp: number
+  /** The tx.json delta object, carried verbatim. */
+  delta: unknown
+  /** The before-image record-set (empty for record-less generations). */
+  records: Array<{ kind: 'noun' | 'verb'; id: string; record: unknown }>
+}
+
+/** Manifest entry for one sealed segment. */
+export interface SegmentMeta {
+  file: string
+  firstGeneration: number
+  lastGeneration: number
+  frames: number
+  bytes: number
+  /** crc32c of the full segment byte stream — the digest chain's link. */
+  checksum: number
+}
+
+interface SegmentManifest {
+  version: 1
+  compactedBelow: number
+  segments: SegmentMeta[]
+}
+
+interface SidecarIndex {
+  version: 1
+  /** [generation, frameOffset, frameLen] ascending by generation. */
+  generations: Array<[number, number, number]>
+  /** `${kindByte}:${id}` → ascending generations holding a record for it. */
+  ids: Record<string, number[]>
+}
+
+const segmentFileName = (firstGeneration: number): string =>
+  `seg-${String(firstGeneration).padStart(20, '0')}.bgs`
+const sidecarFileName = (firstGeneration: number): string =>
+  `seg-${String(firstGeneration).padStart(20, '0')}.idx`
+
+/**
+ * The generation-segment store. Owns the packed tier ONLY — the live
+ * per-generation tier and the routing between tiers belong to
+ * `GenerationStore`. All mutating entry points here are called under the
+ * generation store's commit mutex.
+ */
+export class GenerationSegmentStore {
+  private readonly storage: FactLogStorage
+  private manifest: SegmentManifest = { version: 1, compactedBelow: 0, segments: [] }
+  /** Sidecar cache — segments are immutable, so entries never invalidate. */
+  private readonly sidecars = new Map<string, SidecarIndex>()
+
+  constructor(storage: FactLogStorage) {
+    this.storage = storage
+  }
+
+  /** Load the manifest (ONE read — never a directory listing). */
+  async open(): Promise<void> {
+    const raw = (await this.storage.readRawObject(MANIFEST_PATH)) as SegmentManifest | null
+    if (raw) {
+      if (raw.version !== 1) {
+        throw new Error(
+          `[GenerationSegments] manifest version ${String(raw.version)} is newer than this ` +
+            `engine understands — refusing to serve partial history. Upgrade the engine.`
+        )
+      }
+      this.manifest = raw
+    }
+  }
+
+  /** The packed tier's catalogue (ascending, immutable snapshot). */
+  segments(): readonly SegmentMeta[] {
+    return this.manifest.segments
+  }
+
+  /** D3's horizon marker: generations below this were reclaimed (bounded profiles only). */
+  compactedBelow(): number {
+    return this.manifest.compactedBelow
+  }
+
+  /** The covering sealed segment for `gen`, or null if it lives outside the packed tier. */
+  private coveringSegment(gen: number): SegmentMeta | null {
+    // Manifest is ascending and ranges never overlap — binary search.
+    const segs = this.manifest.segments
+    let lo = 0
+    let hi = segs.length - 1
+    while (lo <= hi) {
+      const mid = (lo + hi) >> 1
+      const s = segs[mid]
+      if (gen < s.firstGeneration) hi = mid - 1
+      else if (gen > s.lastGeneration) lo = mid + 1
+      else return s
+    }
+    return null
+  }
+
+  /** True when `gen` is packed (readable from this tier). */
+  hasGeneration(gen: number): boolean {
+    return this.coveringSegment(gen) !== null
+  }
+
+  /**
+   * Fold consecutive generations into ONE new sealed segment + sidecar and
+   * append it to the manifest atomically. Caller guarantees: `gens` is
+   * ascending, contiguous with the packed tier (first = last packed + 1 when
+   * segments exist), and already durable in the live tier. Crash between the
+   * segment write and the caller's live-tier delete leaves a DUPLICATE
+   * representation — resolved live-tier-wins by the reader; never a gap.
+   */
+  async fold(gens: FoldGeneration[]): Promise<SegmentMeta> {
+    if (gens.length === 0) {
+      throw new Error('[GenerationSegments] fold() requires at least one generation')
+    }
+    for (let i = 1; i < gens.length; i++) {
+      if (gens[i].generation <= gens[i - 1].generation) {
+        throw new Error('[GenerationSegments] fold() input must be strictly ascending')
+      }
+    }
+    const last = this.manifest.segments[this.manifest.segments.length - 1]
+    if (last && gens[0].generation <= last.lastGeneration) {
+      throw new Error(
+        `[GenerationSegments] fold() overlaps the packed tier: ${gens[0].generation} ≤ ` +
+          `sealed ${last.lastGeneration} — segments are immutable, never rewritten`
+      )
+    }
+
+    const first = gens[0].generation
+    const file = segmentFileName(first)
+    const sidecar: SidecarIndex = { version: 1, generations: [], ids: {} }
+
+    // Encode all frames, tracking offsets for the sidecar.
+    const parts: Uint8Array[] = [MAGIC]
+    let offset = MAGIC.length
+    for (const g of gens) {
+      const payload = msgpackEncode([
+        g.generation,
+        g.timestamp,
+        g.delta,
+        g.records.map((r) => [r.kind === 'noun' ? 0 : 1, r.id, r.record]),
+        0 // flags: v1 = uncompressed
+      ])
+      const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length)
+      const view = new DataView(frame.buffer)
+      view.setUint32(0, payload.length, true)
+      view.setUint32(4, crc32c(payload), true)
+      frame.set(payload, FRAME_PREFIX_BYTES)
+      sidecar.generations.push([g.generation, offset, frame.length])
+      for (const r of g.records) {
+        const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}`
+        ;(sidecar.ids[key] ??= []).push(g.generation)
+      }
+      parts.push(frame)
+      offset += frame.length
+    }
+    const total = parts.reduce((n, p) => n + p.length, 0)
+    const bytes = new Uint8Array(total)
+    let at = 0
+    for (const p of parts) {
+      bytes.set(p, at)
+      at += p.length
+    }
+
+    const meta: SegmentMeta = {
+      file,
+      firstGeneration: first,
+      lastGeneration: gens[gens.length - 1].generation,
+      frames: gens.length,
+      bytes: total,
+      checksum: crc32c(bytes)
+    }
+
+    // Durability order: segment + sidecar fsync'd BEFORE the manifest names
+    // them (a crash before the manifest = invisible orphan files, harmless);
+    // manifest last, atomically.
+    const segPath = `${SEGMENTS_PREFIX}/${file}`
+    const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(first)}`
+    await this.storage.writeRawBytes(segPath, bytes)
+    await this.storage.writeRawBytes(idxPath, msgpackEncode(sidecar))
+    await this.storage.syncRawObjects([segPath, idxPath])
+    const next: SegmentManifest = {
+      ...this.manifest,
+      segments: [...this.manifest.segments, meta]
+    }
+    await this.storage.writeRawObject(MANIFEST_PATH, next)
+    await this.storage.syncRawObjects([MANIFEST_PATH])
+    this.manifest = next
+    this.sidecars.set(file, sidecar)
+    return meta
+  }
+
+  /** Load (or rebuild, loudly) a segment's sidecar. */
+  private async sidecarFor(meta: SegmentMeta): Promise<SidecarIndex> {
+    const cached = this.sidecars.get(meta.file)
+    if (cached) return cached
+    const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(meta.firstGeneration)}`
+    const raw = await this.storage.readRawBytes(idxPath)
+    if (raw) {
+      try {
+        const idx = msgpackDecode(raw) as SidecarIndex
+        if (idx.version === 1) {
+          this.sidecars.set(meta.file, idx)
+          return idx
+        }
+      } catch {
+        // fall through to rebuild
+      }
+    }
+    // Sidecars are DERIVED: rebuild from the segment, loudly — never serve
+    // wrong offsets silently.
+    prodLog.warn(
+      `[GenerationSegments] sidecar for ${meta.file} missing or unreadable — rebuilding from the segment`
+    )
+    const rebuilt = await this.rebuildSidecar(meta)
+    await this.storage.writeRawBytes(idxPath, msgpackEncode(rebuilt))
+    this.sidecars.set(meta.file, rebuilt)
+    return rebuilt
+  }
+
+  /** One sequential read of the segment → a fresh sidecar. Verifies every frame CRC. */
+  private async rebuildSidecar(meta: SegmentMeta): Promise<SidecarIndex> {
+    const frames = await this.readAllFrames(meta)
+    const idx: SidecarIndex = { version: 1, generations: [], ids: {} }
+    for (const f of frames) {
+      idx.generations.push([f.generation, f.offset, f.frameLen])
+      for (const r of f.records) {
+        const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}`
+        ;(idx.ids[key] ??= []).push(f.generation)
+      }
+    }
+    return idx
+  }
+
+  private decodeFrame(
+    payload: Uint8Array
+  ): { generation: number; timestamp: number; delta: unknown; records: FoldGeneration['records'] } {
+    const [generation, timestamp, delta, rawRecords] = msgpackDecode(payload) as [
+      number,
+      number,
+      unknown,
+      Array<[number, string, unknown]>,
+      number
+    ]
+    return {
+      generation,
+      timestamp,
+      delta,
+      records: rawRecords.map(([kindByte, id, record]) => ({
+        kind: kindByte === 0 ? ('noun' as const) : ('verb' as const),
+        id,
+        record
+      }))
+    }
+  }
+
+  private async readAllFrames(meta: SegmentMeta): Promise<
+    Array<ReturnType<GenerationSegmentStore['decodeFrame']> & { offset: number; frameLen: number }>
+  > {
+    const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`)
+    if (!bytes) {
+      throw new Error(
+        `[GenerationSegments] sealed segment ${meta.file} is MISSING — packed history is damaged; ` +
+          `refusing to continue silently`
+      )
+    }
+    const out: Array<ReturnType<GenerationSegmentStore['decodeFrame']> & { offset: number; frameLen: number }> = []
+    let at = MAGIC.length
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+    while (at + FRAME_PREFIX_BYTES <= bytes.length) {
+      const payloadLen = view.getUint32(at, true)
+      const crc = view.getUint32(at + 4, true)
+      const payload = bytes.subarray(at + FRAME_PREFIX_BYTES, at + FRAME_PREFIX_BYTES + payloadLen)
+      if (payload.length !== payloadLen || crc32c(payload) !== crc) {
+        throw new Error(
+          `[GenerationSegments] frame CRC mismatch in ${meta.file} at offset ${at} — ` +
+            `packed history is damaged; refusing to serve it`
+        )
+      }
+      out.push({ ...this.decodeFrame(payload), offset: at, frameLen: FRAME_PREFIX_BYTES + payloadLen })
+      at += FRAME_PREFIX_BYTES + payloadLen
+    }
+    return out
+  }
+
+  /** Read one packed generation's frame via its sidecar offset (one ranged read). */
+  private async readFrame(
+    gen: number
+  ): Promise<ReturnType<GenerationSegmentStore['decodeFrame']> | null> {
+    const meta = this.coveringSegment(gen)
+    if (!meta) return null
+    const idx = await this.sidecarFor(meta)
+    // generations ascending → binary search.
+    const gens = idx.generations
+    let lo = 0
+    let hi = gens.length - 1
+    while (lo <= hi) {
+      const mid = (lo + hi) >> 1
+      if (gens[mid][0] < gen) lo = mid + 1
+      else if (gens[mid][0] > gen) hi = mid - 1
+      else {
+        const [, offset, frameLen] = gens[mid]
+        const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`)
+        if (!bytes) {
+          throw new Error(`[GenerationSegments] sealed segment ${meta.file} is MISSING`)
+        }
+        const frame = bytes.subarray(offset, offset + frameLen)
+        const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength)
+        const payloadLen = view.getUint32(0, true)
+        const crc = view.getUint32(4, true)
+        const payload = frame.subarray(FRAME_PREFIX_BYTES, FRAME_PREFIX_BYTES + payloadLen)
+        if (payload.length !== payloadLen || crc32c(payload) !== crc) {
+          throw new Error(
+            `[GenerationSegments] frame CRC mismatch for generation ${gen} in ${meta.file} — ` +
+              `packed history is damaged; refusing to serve it`
+          )
+        }
+        return this.decodeFrame(payload)
+      }
+    }
+    // In the covering range but not present: the packed tier is dense by
+    // construction (fold packs every generation it is handed, including
+    // record-less ones) — absence inside a sealed range is damage.
+    throw new Error(
+      `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
+        `range but has no frame — packed history is damaged`
+    )
+  }
+
+  /** The packed tier's delta for `gen` (null = not packed). */
+  async readDelta(gen: number): Promise<{ delta: unknown; timestamp: number } | null> {
+    const frame = await this.readFrame(gen)
+    return frame ? { delta: frame.delta, timestamp: frame.timestamp } : null
+  }
+
+  /** The packed tier's full record-set for `gen` (null = not packed). */
+  async readRecords(gen: number): Promise<FoldGeneration['records'] | null> {
+    const frame = await this.readFrame(gen)
+    return frame ? frame.records : null
+  }
+
+  /** One packed before-image (null = not packed OR no record for the id in that generation). */
+  async readRecord(gen: number, kind: 'noun' | 'verb', id: string): Promise<unknown | null> {
+    const frame = await this.readFrame(gen)
+    if (!frame) return null
+    const hit = frame.records.find((r) => r.kind === kind && r.id === id)
+    return hit ? hit.record : null
+  }
+
+  /**
+   * D3 reclaim: drop WHOLE segments whose lastGeneration < `belowGeneration`
+   * and bump `compactedBelow`. Partial segments are never dropped — the
+   * boundary waits. NEVER called under the archival profile (the caller
+   * enforces retention semantics; this method only executes boundary drops).
+   */
+  async dropSegmentsBelow(belowGeneration: number): Promise<{ dropped: number; compactedBelow: number }> {
+    const keep: SegmentMeta[] = []
+    const drop: SegmentMeta[] = []
+    for (const s of this.manifest.segments) {
+      ;(s.lastGeneration < belowGeneration ? drop : keep).push(s)
+    }
+    if (drop.length === 0) {
+      return { dropped: 0, compactedBelow: this.manifest.compactedBelow }
+    }
+    const compactedBelow = Math.max(
+      this.manifest.compactedBelow,
+      drop[drop.length - 1].lastGeneration + 1
+    )
+    // Manifest first (the drop is authoritative once named), then bytes —
+    // a crash between leaves orphan segment files invisible to the manifest,
+    // harmless and re-collectable.
+    const next: SegmentManifest = { ...this.manifest, compactedBelow, segments: keep }
+    await this.storage.writeRawObject(MANIFEST_PATH, next)
+    await this.storage.syncRawObjects([MANIFEST_PATH])
+    this.manifest = next
+    for (const s of drop) {
+      await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${s.file}`)
+      await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${sidecarFileName(s.firstGeneration)}`)
+      this.sidecars.delete(s.file)
+    }
+    return { dropped: drop.length, compactedBelow }
+  }
+
+  /**
+   * D8 rider — the packed portion of `generationDigest(g)`: a deterministic
+   * crc32c chain over sealed-segment checksums fully below `g`, plus the
+   * frame CRC of `g`'s own frame when `g` is mid-segment. O(segments), not
+   * O(generations); identical history ⇒ identical digest on any machine.
+   * The live-tier portion is composed by the caller.
+   */
+  async digestThroughPacked(g: number): Promise<number | null> {
+    let digest = 0
+    let covered = false
+    for (const s of this.manifest.segments) {
+      if (s.lastGeneration <= g) {
+        digest = crc32c(new TextEncoder().encode(`${digest}:${s.checksum}`))
+        if (s.lastGeneration === g) covered = true
+      } else if (s.firstGeneration <= g) {
+        // g is mid-segment: chain the partial prefix via g's frame CRC.
+        const frame = await this.readFrame(g)
+        if (frame === null) return null
+        const idx = await this.sidecarFor(s)
+        const upTo = idx.generations.filter(([gen]) => gen <= g)
+        for (const [gen, offset, frameLen] of upTo) {
+          digest = crc32c(new TextEncoder().encode(`${digest}:${gen}:${offset}:${frameLen}`))
+        }
+        covered = true
+        break
+      }
+    }
+    return covered || this.manifest.segments.length > 0 ? digest : null
+  }
+}
diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts
index bab9d4d9..1b1f412e 100644
--- a/src/storage/adapters/memoryStorage.ts
+++ b/src/storage/adapters/memoryStorage.ts
@@ -133,6 +133,11 @@ export class MemoryStorage extends BaseStorage {
    */
   protected async deleteObjectFromPath(path: string): Promise<void> {
     this.objectStore.delete(path)
+    // Filesystem parity: on disk, objects and raw BYTE files are both just
+    // files — unlink removes whichever exists. Without this, deleteRawObject
+    // on a raw-bytes path (fact-log/generation segments) silently no-ops on
+    // memory storage: the delete "succeeds" and the bytes remain.
+    this.rawBytesStore.delete(path)
   }
 
   /**
diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts
new file mode 100644
index 00000000..27ab85cb
--- /dev/null
+++ b/tests/unit/db/generation-segments.test.ts
@@ -0,0 +1,150 @@
+/**
+ * @module tests/unit/db/generation-segments
+ * @description The generation-segment store (Stage-2 D1+D3 file format).
+ * Laws: (1) fold → read round-trips deltas and records byte-faithfully via
+ * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen
+ * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from
+ * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong
+ * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow;
+ * (5) the packed digest is deterministic across reopen; (6) immutability —
+ * fold refuses overlap with sealed ranges.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+import {
+  GenerationSegmentStore,
+  SEGMENTS_PREFIX,
+  type FoldGeneration
+} from '../../../src/db/generationSegments.js'
+
+const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
+
+const gen = (g: number, recordCount = 2): FoldGeneration => ({
+  generation: g,
+  timestamp: 1_700_000_000_000 + g,
+  delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g },
+  records: Array.from({ length: recordCount }, (_, i) => ({
+    kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb',
+    id: UUID(g * 100 + i),
+    record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } }
+  }))
+})
+
+describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => {
+  let storage: MemoryStorage
+  let store: GenerationSegmentStore
+
+  beforeEach(async () => {
+    storage = new MemoryStorage()
+    await storage.init()
+    store = new GenerationSegmentStore(storage as any)
+    await store.open()
+  })
+
+  it('fold → read round-trips deltas and records via sidecar point-reads', async () => {
+    const meta = await store.fold([gen(1), gen(2), gen(3)])
+    expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 })
+    expect(meta.checksum).toBeGreaterThan(0)
+
+    expect(store.hasGeneration(2)).toBe(true)
+    expect(store.hasGeneration(4)).toBe(false)
+
+    const d2 = await store.readDelta(2)
+    expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 })
+    expect(d2?.timestamp).toBe(1_700_000_000_002)
+
+    const records = await store.readRecords(3)
+    expect(records).toHaveLength(2)
+    expect(records![0]).toEqual({
+      kind: 'noun',
+      id: UUID(300),
+      record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } }
+    })
+    // Point read by id, both kinds.
+    expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({
+      metadata: { noun: 'document', v: 3 },
+      vector: { v: [3, 1] }
+    })
+    expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull()
+  })
+
+  it('reopen discovers everything from the manifest alone — no listing', async () => {
+    await store.fold([gen(1), gen(2)])
+    await store.fold([gen(3), gen(4)])
+
+    const reopened = new GenerationSegmentStore(storage as any)
+    await reopened.open()
+    expect(reopened.segments()).toHaveLength(2)
+    expect(reopened.hasGeneration(4)).toBe(true)
+    expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001)
+  })
+
+  it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => {
+    const meta = await store.fold([gen(1), gen(2)])
+    const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx`
+    await storage.deleteRawObject(idxPath)
+
+    const reopened = new GenerationSegmentStore(storage as any)
+    await reopened.open()
+    // Rebuild path: still serves correct data.
+    expect((await reopened.readRecords(2))!).toHaveLength(2)
+
+    // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud.
+    const segPath = `${SEGMENTS_PREFIX}/${meta.file}`
+    const bytes = (await storage.readRawBytes(segPath))!
+    bytes[bytes.length - 3] ^= 0xff
+    await storage.writeRawBytes(segPath, bytes)
+    const damaged = new GenerationSegmentStore(storage as any)
+    await damaged.open()
+    ;(damaged as any).sidecars.clear()
+    await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes
+    await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/)
+  })
+
+  it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => {
+    await store.fold([gen(1), gen(2)])
+    await store.fold([gen(3), gen(4)])
+    await store.fold([gen(5), gen(6)])
+
+    // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops.
+    const r1 = await store.dropSegmentsBelow(4)
+    expect(r1).toEqual({ dropped: 1, compactedBelow: 3 })
+    expect(store.hasGeneration(1)).toBe(false)
+    expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole
+
+    // Bytes actually gone.
+    expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull()
+
+    // Horizon past everything: the rest drop; compactedBelow is durable.
+    const r2 = await store.dropSegmentsBelow(7)
+    expect(r2.dropped).toBe(2)
+    const reopened = new GenerationSegmentStore(storage as any)
+    await reopened.open()
+    expect(reopened.compactedBelow()).toBe(7)
+    expect(reopened.segments()).toHaveLength(0)
+  })
+
+  it('the packed digest is deterministic across reopen and changes with history', async () => {
+    await store.fold([gen(1), gen(2), gen(3)])
+    const atSeal = await store.digestThroughPacked(3)
+    const midSegment = await store.digestThroughPacked(2)
+    expect(atSeal).not.toBeNull()
+    expect(midSegment).not.toBeNull()
+    expect(midSegment).not.toBe(atSeal)
+
+    const reopened = new GenerationSegmentStore(storage as any)
+    await reopened.open()
+    expect(await reopened.digestThroughPacked(3)).toBe(atSeal)
+    expect(await reopened.digestThroughPacked(2)).toBe(midSegment)
+
+    await reopened.fold([gen(4)])
+    expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal)
+  })
+
+  it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => {
+    await store.fold([gen(1), gen(2)])
+    await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/)
+    await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/)
+    await expect(store.fold([])).rejects.toThrow(/at least one generation/)
+  })
+})

From 1201e2554330858df7a419d1c7396aa5395a8c85 Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Sun, 19 Jul 2026 16:26:10 -0700
Subject: [PATCH 82/82] =?UTF-8?q?feat:=20two-tier=20history=20reads=20+=20?=
 =?UTF-8?q?the=20repacker=20+=20generationDigest=20=E2=80=94=20D1+D3=20wir?=
 =?UTF-8?q?ed=20end-to-end?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The packed tier goes live inside GenerationStore:

- Two-tier reads: getDelta / readBeforeImage / readGenerationRecords
  fall through live-tier → sealed segments (live-tier-wins: a crash
  mid-fold leaves a duplicate representation, never a gap). Cold-open
  seeds committedRanges from the segment manifest via interval merge —
  packed generations resolve without their directories existing.
- repackHistory({timeBudgetMs, batchGenerations}): folds cold
  generations (older than the newest 1024) oldest-first into sealed
  segments, deleting per-generation directories only after segment +
  manifest are durable. Public API + automatic time-bounded pass at
  close() (before compaction, so reclaim can drop whole segments);
  re-representation only — the sole history transform under the
  archival profile. Early stop = consistent prefix, next pass resumes.
- compact(): packed generations reclaim logically in the loop and
  physically at whole-segment boundaries via dropSegmentsBelow (the
  frozen partial-segments-wait rule).
- generationDigest(g) (D8): deterministic content digest through g —
  sealed-segment checksum chain + live-tier delta hashes; O(segments +
  live window); RangeError out of range, GenerationCompactedError
  below the horizon (a gate can never silently pin reclaimed history).

Four end-to-end pins: asOf answers byte-identical across fold + cold
reopen with folded dirs physically gone; repack+reclaim composition;
digest reopen-stability/divergence/loud-horizon; budget no-op+resume.
---
 src/brainy.ts                               |  59 +++++-
 src/db/generationStore.ts                   | 217 +++++++++++++++++++-
 tests/integration/history-repacking.test.ts | 186 +++++++++++++++++
 3 files changed, 454 insertions(+), 8 deletions(-)
 create mode 100644 tests/integration/history-repacking.test.ts

diff --git a/src/brainy.ts b/src/brainy.ts
index 8ba991dd..9ab3acee 100644
--- a/src/brainy.ts
+++ b/src/brainy.ts
@@ -8265,6 +8265,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     return this.generationStore.compact(options)
   }
 
+  /**
+   * @description Repack cold generation history into sealed segments —
+   * re-representation, never deletion: every record and delta stays readable
+   * (`asOf()` unchanged); the physical file count drops by orders of
+   * magnitude. Runs automatically (time-bounded) at `close()`; call this for
+   * explicit maintenance windows on long-lived writers. The ONLY history
+   * transform permitted under the archival profile (`retention: 'all'`).
+   * @param options - `timeBudgetMs` bounds the pass (early stop = consistent
+   *   prefix, next pass resumes); `batchGenerations` sizes each fold.
+   * @returns Folded generation count and segments created.
+   */
+  async repackHistory(options?: {
+    timeBudgetMs?: number
+    batchGenerations?: number
+  }): Promise<{ foldedGenerations: number; segmentsCreated: number }> {
+    this.assertWritable('repackHistory')
+    await this.ensureInitialized()
+    await this.generationStore.flushPendingSingleOps()
+    return this.generationStore.repackHistory(options)
+  }
+
   /**
    * @description Read-only generational-history footprint for fleet audits:
    * generation count, total on-disk bytes, generation/timestamp range, the
@@ -8292,6 +8313,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
   }
 
+  /**
+   * @description A deterministic content digest of the generation log through
+   * `g` (D8 — gate-to-generation provenance): identical history produces the
+   * identical digest on any machine; divergence produces a different one.
+   * Release gates and suite verdicts pin `{generation, digest}` and verify
+   * both at execution time instead of pinning a git commit. O(segments +
+   * live-tier window), never O(all generations). Throws `RangeError` out of
+   * range and `GenerationCompactedError` below the horizon — a gate can
+   * never silently pin reclaimed history.
+   * @example
+   * const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) }
+   */
+  async generationDigest(g: number): Promise<string> {
+    await this.ensureInitialized()
+    await this.generationStore.flushPendingSingleOps()
+    return this.generationStore.generationDigest(g)
+  }
+
   /**
    * @description Drive the adaptive retention byte budget at runtime — the
    * settable input a machine-level coordinator (e.g. cor's `ResourceManager`,
@@ -16192,11 +16231,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       await this.generationStore.flushPendingSingleOps()
     }
 
-    // Phase 0b: Auto-compact generational history per config.retention (default
-    // on) BEFORE the generation store closes below. This is THE auto-compaction
-    // site (8.9.0 — flush() never compacts): time-bounded per pass, respects
-    // live Db pins and an explicit autoCompact: false; no-op on read-only
-    // instances.
+    // Phase 0b: REPACK cold history into sealed segments (D1+D3 —
+    // re-representation, never deletion; the only history transform under the
+    // archival profile), then auto-compact per config.retention. Repack runs
+    // FIRST so bounded-retention reclaim can drop whole segments. Both are
+    // time-bounded maintenance passes (8.9.0 law: flush() never pays these);
+    // both are housekeeping — failures warn, never fail a clean shutdown.
+    if (!this.isReadOnly && this.generationStore) {
+      try {
+        await this.generationStore.repackHistory({ timeBudgetMs: 5_000 })
+      } catch (error) {
+        console.warn(
+          `History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}`
+        )
+      }
+    }
     await this.autoCompactHistory()
 
     // Phase 1: Flush ALL components in parallel to persist buffered data
diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts
index 4e3738d9..aede17a4 100644
--- a/src/db/generationStore.ts
+++ b/src/db/generationStore.ts
@@ -46,6 +46,8 @@ import type {
   TxLogEntry
 } from './types.js'
 import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js'
+import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js'
+import { crc32c } from '../utils/crc32c.js'
 
 /**
  * The byte-identical before-images of every id a commit touches, read UNDER
@@ -266,6 +268,21 @@ export class GenerationStore {
    */
   private historyBytesTotal: number | null = null
 
+  /**
+   * The packed tier (D1+D3): sealed segments holding folded cold
+   * generations. Null until {@link open} wires it (and on storage adapters
+   * without raw-byte primitives — the live tier then carries everything,
+   * exactly as before the packed tier existed).
+   */
+  private segments: GenerationSegmentStore | null = null
+
+  /**
+   * Live-tier window: generations newer than `committed - REPACK_LIVE_WINDOW`
+   * are never folded — the hot tail stays in the per-generation layout the
+   * write path owns. Matches the resident chain window's scale.
+   */
+  static readonly REPACK_LIVE_WINDOW = 1024
+
   /**
    * Model-B per-write group-commit — the in-memory PENDING tier.
    *
@@ -433,6 +450,33 @@ export class GenerationStore {
       this.factLog = null
     }
 
+    // PACKED TIER (D1+D3): same capability gate as the fact log. Opening
+    // reads ONE manifest — never a listing of the packed backlog — and seeds
+    // committedRanges with the sealed ranges so packed generations resolve
+    // exactly like live ones.
+    if (storageSupportsFactLog(this.storage)) {
+      this.segments = new GenerationSegmentStore(this.storage)
+      await this.segments.open()
+      const packedRanges = this.segments
+        .segments()
+        .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
+        .filter(([lo, hi]) => lo <= hi)
+      if (packedRanges.length > 0) {
+        // Merge packed (older) + live (newer) interval sets — both ascending;
+        // coalesce adjacency so range arithmetic stays interval-exact.
+        const merged: Array<[number, number]> = []
+        for (const r of [...packedRanges, ...this.committedRanges].sort((a, b) => a[0] - b[0])) {
+          const last = merged[merged.length - 1]
+          if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1])
+          else merged.push([r[0], r[1]])
+        }
+        this.committedRanges = merged
+      }
+      this.horizonGen = Math.max(this.horizonGen, this.segments.compactedBelow() - 1)
+    } else {
+      this.segments = null
+    }
+
     // Hook single-op write batches so generation() is always meaningful.
     // Suppressed while a transact batch executes (the batch is ONE generation).
     if (!options?.readOnly) {
@@ -500,6 +544,51 @@ export class GenerationStore {
    * deltas (cache-bounded reads).
    * @returns Counts, bytes, generation range, and the compaction horizon.
    */
+  /**
+   * @description D8 (gate-to-generation provenance): a deterministic content
+   * digest of the generation log THROUGH `g` — identical history ⇒ identical
+   * digest on any machine; any divergence (different records, different
+   * order, reclaimed range) ⇒ different digest. Composed from the packed
+   * tier's sealed-segment checksum chain (O(segments)) plus the live tier's
+   * per-generation delta digests (O(live window at most)). Release gates pin
+   * {generation, digest} and verify both at execution time.
+   * @param g - The generation to digest through (≤ committed).
+   * @returns A hex digest string, stable across reopen and repacking states
+   *   ONLY for fully-packed prefixes — repacking changes representation, so
+   *   the composed digest is defined over CONTENT: live-tier gens hash their
+   *   delta + record ids, packed gens hash via frame CRCs. A gate should pin
+   *   after a repack pass for long-term stability, or re-pin on repack.
+   */
+  async generationDigest(g: number): Promise<string> {
+    if (!Number.isInteger(g) || g < 1 || g > this.committed) {
+      throw new RangeError(
+        `generationDigest(): generation ${g} is out of range [1, ${this.committed}]`
+      )
+    }
+    if (g <= this.horizonGen) {
+      throw new GenerationCompactedError(g, this.horizonGen)
+    }
+    let digest = 0
+    const enc = new TextEncoder()
+    if (this.segments) {
+      const packed = await this.segments.digestThroughPacked(g)
+      if (packed !== null) digest = packed
+    }
+    // Live-tier composition: every committed gen ≤ g not covered by a sealed
+    // segment hashes its delta content in ascending order.
+    for (const gen of this.committedGensAsc()) {
+      if (gen > g) break
+      if (this.segments?.hasGeneration(gen)) continue
+      const delta = await this.getDelta(gen)
+      digest = crc32c(
+        enc.encode(
+          `${digest}:${gen}:${delta.timestamp}:${[...delta.nouns].sort().join(',')}:${[...delta.verbs].sort().join(',')}`
+        )
+      )
+    }
+    return digest.toString(16).padStart(8, '0')
+  }
+
   async historyStats(): Promise<{
     generations: number
     bytes: number
@@ -538,14 +627,17 @@ export class GenerationStore {
     try {
       paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`)
     } catch {
-      return []
+      paths = []
     }
     const records: GenerationRecord[] = []
     for (const p of paths) {
       const record = (await this.storage.readRawObject(p)) as GenerationRecord | null
       if (record) records.push(record)
     }
-    return records
+    if (records.length > 0) return records
+    // Two-tier: folded generations serve their record-set from the segment.
+    const packed = await this.segments?.readRecords(gen)
+    return packed ? (packed.map((r) => r.record) as GenerationRecord[]) : []
   }
 
   /**
@@ -1783,9 +1875,15 @@ export class GenerationStore {
     if (pending) {
       return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null
     }
-    return (await this.storage.readRawObject(
+    const live = (await this.storage.readRawObject(
       `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json`
     )) as GenerationRecord | null
+    if (live) return live
+    // Two-tier: the packed tier serves folded generations (live-tier-wins).
+    if (this.segments?.hasGeneration(gen)) {
+      return (await this.segments.readRecord(gen, kind, id)) as GenerationRecord | null
+    }
+    return null
   }
 
   /**
@@ -2132,6 +2230,21 @@ export class GenerationStore {
       `${GENERATIONS_PREFIX}/${gen}/tx.json`
     )) as GenerationDelta | null
     if (delta === null) {
+      // Two-tier read (D1+D3): not in the live tier → the packed tier.
+      // Live-tier-wins ordering (a crash mid-fold leaves a duplicate, never
+      // a gap), so the segment lookup runs only after the live miss.
+      const packed = await this.segments?.readDelta(gen)
+      if (packed) {
+        const d = packed.delta as GenerationDelta
+        const entry = {
+          nouns: new Set(d.nouns),
+          verbs: new Set(d.verbs),
+          timestamp: packed.timestamp,
+          bytes: d.bytes ?? 0
+        }
+        this.setDelta(gen, entry)
+        return entry
+      }
       throw new Error(
         `Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` +
           `(store corrupted or records removed outside compactHistory())`
@@ -2213,6 +2326,94 @@ export class GenerationStore {
    * @param options - Retention caps (see {@link CompactHistoryOptions}).
    * @returns Count of removed record-sets and the new horizon.
    */
+  /**
+   * @description The REPACKER (D1+D3+repacking): fold cold live-tier
+   * generations into sealed segments — re-representation, never deletion.
+   * Every record and delta stays readable (asOf/chains unchanged); the
+   * per-generation directories are deleted only AFTER their segment is
+   * durable (crash between = duplicate representation, resolved
+   * live-tier-wins by every reader; never a gap). This is the transform that
+   * takes a 70k-file history to tens of segment files, and the ONLY history
+   * transform permitted under the archival profile.
+   *
+   * Folds oldest-first, contiguous from the packed boundary, in batches, and
+   * stops at the live window ({@link GenerationStore.REPACK_LIVE_WINDOW})
+   * or when `timeBudgetMs` is spent — an early stop is a consistent prefix;
+   * the next pass resumes.
+   */
+  async repackHistory(options?: { timeBudgetMs?: number; batchGenerations?: number }): Promise<{
+    foldedGenerations: number
+    segmentsCreated: number
+  }> {
+    if (!this.segments) return { foldedGenerations: 0, segmentsCreated: 0 }
+    const segments = this.segments
+    return this.withMutex(async () => {
+      const deadline =
+        options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined
+      const batchSize = options?.batchGenerations ?? 512
+      const coldCeiling = this.committed - GenerationStore.REPACK_LIVE_WINDOW
+      const packedThrough =
+        segments.segments().length > 0
+          ? segments.segments()[segments.segments().length - 1].lastGeneration
+          : 0
+
+      // Cold, unpacked, committed generations — ascending, contiguous scan.
+      const eligible: number[] = []
+      for (const gen of this.committedGensAsc()) {
+        if (gen > coldCeiling) break
+        if (gen <= packedThrough) continue // already packed (dup fold barred)
+        if (this.pendingBuffer.has(gen)) continue // un-flushed = live by definition
+        eligible.push(gen)
+      }
+
+      let folded = 0
+      let segmentsCreated = 0
+      for (let i = 0; i < eligible.length; i += batchSize) {
+        if (deadline !== undefined && Date.now() >= deadline) break
+        const batch = eligible.slice(i, i + batchSize)
+        const foldInput: FoldGeneration[] = []
+        for (const gen of batch) {
+          const delta = (await this.storage.readRawObject(
+            `${GENERATIONS_PREFIX}/${gen}/tx.json`
+          )) as GenerationDelta | null
+          if (delta === null) {
+            // Already folded by a prior crashed pass whose dirs were removed,
+            // or damage — getDelta's two-tier read decides which, loudly,
+            // when someone asks. Skip; never fold a generation we cannot read.
+            continue
+          }
+          const records: FoldGeneration['records'] = []
+          for (const [kind, ids] of [
+            ['noun', delta.nouns] as const,
+            ['verb', delta.verbs] as const
+          ]) {
+            for (const id of ids) {
+              const record = await this.storage.readRawObject(
+                `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json`
+              )
+              if (record) records.push({ kind, id, record })
+            }
+          }
+          foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
+        }
+        if (foldInput.length === 0) continue
+        await segments.fold(foldInput)
+        segmentsCreated++
+        // Segment + manifest durable → the live copies retire.
+        for (const g of foldInput) {
+          await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
+        }
+        folded += foldInput.length
+      }
+      if (folded > 0) {
+        prodLog.info(
+          `[GenerationStore] repacked ${folded} cold generation(s) into ${segmentsCreated} segment(s) — history preserved, file count reduced`
+        )
+      }
+      return { foldedGenerations: folded, segmentsCreated }
+    })
+  }
+
   async compact(options?: CompactHistoryOptions): Promise<CompactHistoryResult> {
     return this.withMutex(async () => {
       const minPinned = this.minPinnedGeneration()
@@ -2304,6 +2505,16 @@ export class GenerationStore {
         // Reclaimed generations leave the per-id chains stale → rebuild on next read.
         this.invalidateChains()
         this.horizonGen = Math.max(this.horizonGen, highestRemoved)
+        // Packed-tier reclaim (D3): a packed generation's bytes live in a
+        // sealed segment — removeRawPrefix above was a no-op for it. Drop
+        // WHOLE segments now fully below the horizon; a partially-reclaimed
+        // segment keeps its bytes until the boundary passes it (the frozen
+        // partial-segments-wait rule; logical reclamation above still holds —
+        // the generations left committedRanges and asOf below the horizon
+        // throws regardless).
+        if (this.segments) {
+          await this.segments.dropSegmentsBelow(this.horizonGen + 1)
+        }
         const manifest: GenerationManifest = {
           version: 1,
           generation: this.committed,
diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts
new file mode 100644
index 00000000..2bcee038
--- /dev/null
+++ b/tests/integration/history-repacking.test.ts
@@ -0,0 +1,186 @@
+/**
+ * @module tests/integration/history-repacking
+ * @description The D1+D3 two-tier history lifecycle end-to-end on a real
+ * brain. Laws: (1) repacking is RE-REPRESENTATION — after folding, every
+ * asOf() read below the fold boundary answers exactly as before, across a
+ * cold reopen; (2) folded per-generation directories are physically gone
+ * (the file-count cure is real, not cosmetic); (3) repack + reclaim compose:
+ * bounded retention after repacking drops whole segments and asOf below the
+ * horizon throws GenerationCompactedError; (4) repackHistory is explicit
+ * API and time-bounded (spent budget = consistent no-op).
+ *
+ * Uses a tiny REPACK_LIVE_WINDOW override so a small history has a cold
+ * tier at all (the production window is 1024).
+ */
+import { describe, it, expect, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+import * as os from 'node:os'
+import { Brainy } from '../../src/brainy.js'
+import { NounType } from '../../src/types/graphTypes.js'
+import { GenerationStore } from '../../src/db/generationStore.js'
+import { GenerationCompactedError } from '../../src/db/errors.js'
+import { SEGMENTS_PREFIX } from '../../src/db/generationSegments.js'
+
+const stub = async (text: string): Promise<number[]> => {
+  const h = text.split('').reduce((a, c) => a + c.charCodeAt(0), 0)
+  return new Array(384).fill(0).map((_, i) => Math.sin(h + i))
+}
+
+const openBrain = async (dir: string): Promise<Brainy> => {
+  const brain = new Brainy({
+    requireSubtype: false,
+    storage: { type: 'filesystem', path: dir },
+    embeddingFunction: stub
+  })
+  await brain.init()
+  return brain
+}
+
+describe('history repacking — the two-tier lifecycle', () => {
+  const dirs: string[] = []
+  const tempDir = (): string => {
+    const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-repack-'))
+    dirs.push(d)
+    return d
+  }
+  const originalWindow = GenerationStore.REPACK_LIVE_WINDOW
+
+  afterEach(() => {
+    ;(GenerationStore as any).REPACK_LIVE_WINDOW = originalWindow
+    for (const d of dirs.splice(0)) {
+      try {
+        fs.rmSync(d, { recursive: true, force: true })
+      } catch {
+        /* best effort */
+      }
+    }
+  })
+
+  it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => {
+    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
+    const dir = tempDir()
+    const brain = await openBrain(dir)
+
+    const id = await brain.add({
+      data: 'versioned-entity',
+      type: NounType.Document,
+      metadata: { v: 0 }
+    })
+    for (let v = 1; v <= 10; v++) await brain.update({ id, metadata: { v } })
+    await brain.flush()
+
+    // Ground truth BEFORE repacking: capture asOf views for early generations.
+    const before: Record<number, number> = {}
+    for (const g of [2, 4, 6]) {
+      const db = await brain.asOf(g)
+      before[g] = (await db.get(id))?.metadata?.v as number
+      await db.release()
+    }
+
+    const result = await brain.repackHistory()
+    expect(result.foldedGenerations).toBeGreaterThan(0)
+    expect(result.segmentsCreated).toBeGreaterThan(0)
+
+    // The folded per-generation directories are PHYSICALLY gone…
+    const genDirs = fs
+      .readdirSync(path.join(dir, '_generations'), { withFileTypes: true })
+      .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)).length
+    expect(genDirs).toBeLessThanOrEqual(4) // live window (3) + at most the newest
+    // …and the segment tier exists (the filesystem adapter stores objects
+    // gzipped, so the manifest may live at either spelling).
+    const segDir = path.join(dir, SEGMENTS_PREFIX)
+    expect(
+      fs.existsSync(path.join(segDir, 'manifest.json')) ||
+        fs.existsSync(path.join(segDir, 'manifest.json.gz'))
+    ).toBe(true)
+    expect(fs.readdirSync(segDir).some((f) => f.endsWith('.bgs'))).toBe(true)
+
+    // Same asOf answers from the packed tier, same process…
+    for (const g of [2, 4, 6]) {
+      const db = await brain.asOf(g)
+      expect((await db.get(id))?.metadata?.v).toBe(before[g])
+      await db.release()
+    }
+    await brain.close()
+
+    // …and across a COLD REOPEN (manifest discovery, no live dirs to list).
+    const reopened = await openBrain(dir)
+    for (const g of [2, 4, 6]) {
+      const db = await reopened.asOf(g)
+      expect((await db.get(id))?.metadata?.v).toBe(before[g])
+      await db.release()
+    }
+    expect((await reopened.get(id))?.metadata?.v).toBe(10) // live state untouched
+    await reopened.close()
+  })
+
+  it('repack + bounded reclaim compose: whole segments drop, horizon is loud', async () => {
+    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
+    const dir = tempDir()
+    const brain = await openBrain(dir)
+    const id = await brain.add({ data: 'reclaim-probe', type: NounType.Document, metadata: { v: 0 } })
+    for (let v = 1; v <= 8; v++) await brain.update({ id, metadata: { v } })
+    await brain.flush()
+    await brain.repackHistory()
+
+    // Reclaim down to the 3 newest generations — packed segments below the
+    // horizon drop whole; asOf below throws loudly.
+    const res = await brain.compactHistory({ maxGenerations: 3 })
+    expect(res.removedGenerations).toBeGreaterThan(0)
+    await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError)
+    expect((await brain.get(id))?.metadata?.v).toBe(8)
+    await brain.close()
+  })
+
+  it('generationDigest: reopen-stable, divergence-sensitive, loud below the horizon', async () => {
+    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
+    const dir = tempDir()
+    const brain = await openBrain(dir)
+    const id = await brain.add({ data: 'digest-probe', type: NounType.Document, metadata: { v: 0 } })
+    for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } })
+    await brain.flush()
+    await brain.repackHistory()
+
+    const gen = brain.generation()
+    const atHead = await brain.generationDigest(gen)
+    const atMid = await brain.generationDigest(3)
+    expect(atHead).toMatch(/^[0-9a-f]{8}$/)
+    expect(atMid).not.toBe(atHead) // more history ⇒ different digest
+    await brain.close()
+
+    // Reopen-stable: same history, same digests (packed prefix stability).
+    const reopened = await openBrain(dir)
+    expect(await reopened.generationDigest(gen)).toBe(atHead)
+    expect(await reopened.generationDigest(3)).toBe(atMid)
+
+    // New history diverges the head digest.
+    await reopened.update({ id, metadata: { v: 7 } })
+    await reopened.flush()
+    expect(await reopened.generationDigest(reopened.generation())).not.toBe(atHead)
+
+    // Below the horizon: LOUD, never a silent pin of reclaimed history.
+    await reopened.compactHistory({ maxGenerations: 2 })
+    await expect(reopened.generationDigest(1)).rejects.toBeInstanceOf(GenerationCompactedError)
+    await reopened.close()
+  })
+
+  it('a spent time budget is a consistent no-op; the next pass resumes', async () => {
+    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
+    const dir = tempDir()
+    const brain = await openBrain(dir)
+    const id = await brain.add({ data: 'budget-probe', type: NounType.Document, metadata: { v: 0 } })
+    for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } })
+    await brain.flush()
+
+    const bounded = await brain.repackHistory({ timeBudgetMs: 0 })
+    expect(bounded).toEqual({ foldedGenerations: 0, segmentsCreated: 0 })
+
+    const resumed = await brain.repackHistory()
+    expect(resumed.foldedGenerations).toBeGreaterThan(0)
+    const db = await brain.asOf(3)
+    expect((await db.get(id))?.metadata?.v).toBeDefined()
+    await db.release()
+    await brain.close()
+  })
+})