feat: generation fact log — after-image commit records, dual-written at every commit point

Every committed generation now also appends a FACT — an after-image
commit record (what each touched entity/relationship became, or a
body-less tombstone for a removal) — to an append-only, crc32c-framed
segment log under _generations/facts/. The before-image history and the
canonical tree remain authoritative; the fact log gives consumers ONE
sequential, self-verifying stream (index heals, incremental replays)
in place of a per-entity directory walk.

- Wire format: positional msgpack facts [generation, timestamp, ops,
  meta, blobHashes]; op = [kind u8, id bin16, record | nil tombstone];
  32-byte segment header (magic, formatVersion, firstGeneration,
  zeroed+verified reserved); length+crc32c frame per fact; zero-padded
  segment names so lexicographic order == generation order; JSON
  manifest with an atomic rename flip, manifest-first rotation.
- Commit protocol: facts append+fsync BEFORE the commit point inside
  the existing durability window, so a crash can only leave the log
  AHEAD of committed truth — open() truncates back (torn tails detected
  by CRC). Absent generation = never committed; a scan can never see an
  uncommitted fact. transact() facts are durable-on-return; single-op
  facts ride the group-commit flush exactly like buffered history. A
  fact-append failure fails the write, loudly — a silent gap would be a
  lie a later replay discovers.
- New public surface: brain.scanFacts() (sequential batches with heal
  telemetry: head/segments/approx up front, per-batch generation range
  + bytes + segment id, loud abort on gaps, summary cross-check) and
  brain.factSegmentPaths() (immutable sealed segments for zero-copy
  consumers; the mutable tail excluded). Exported types CommitFact,
  FactOp, FactScanBatch, FactScanHandle.
- Storage: optional binary raw-byte primitives (appendRawBytes,
  readRawBytes, writeRawBytes, rawByteSize) on StorageAdapter —
  feature-detected; filesystem + memory adapters implement them; an
  adapter without them hosts no fact log. Fact segments are byte-copied
  (never hard-linked) into snapshots. The _generations/facts/ namespace
  is registered as a protected family (rebuildable: false): no sweeper
  or GC may delete under it.
- New crc32c (Castagnoli) utility with RFC known-answer tests.

tests/integration/db-mvcc.test.ts

@@ -515,7 +515,15 @@ describe('8.0 Db API — generational MVCC', () => {
       await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 4 } }])
     ).release()
 
-    const recordsBefore = (await storage.listRawObjects('_generations')).length
+    // History record-sets only — the generation FACT LOG also lives under
+    // `_generations/` (at `facts/`) and is deliberately NOT reclaimed by
+    // history compaction (facts are the future canonical, not undo history).
+    const historyRecords = async (): Promise<number> =>
+      (await storage.listRawObjects('_generations')).filter(
+        (p: string) => !p.startsWith('_generations/facts/')
+      ).length
+
+    const recordsBefore = await historyRecords()
     expect(recordsBefore).toBeGreaterThan(0)
 
     // Compact while pinned: record-sets above the pin survive, pinned reads stay correct.
@@ -528,7 +536,7 @@ describe('8.0 Db API — generational MVCC', () => {
     const second = await brain.compactHistory()
     expect(first.removedGenerations + second.removedGenerations).toBeGreaterThan(0)
 
-    const recordsAfter = (await storage.listRawObjects('_generations')).length
+    const recordsAfter = await historyRecords()
     expect(recordsAfter).toBeLessThan(recordsBefore)
     expect(recordsAfter).toBe(0)
 

tests/integration/fact-log-dual-write.test.ts

@@ -0,0 +1,200 @@
+/**
+ * @module tests/integration/fact-log-dual-write
+ * @description The generation fact log end-to-end through real commits: every
+ * committed generation (single-op AND transact) appends its AFTER-IMAGE fact
+ * at the commit point; removals append body-less tombstones; an aborted
+ * transaction leaves no fact; facts survive reopen and continue monotonically;
+ * the scan surface (brain.scanFacts) carries the frozen telemetry shape; and
+ * the fact-log namespace is protected against prefix-nuking.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy, ProtectedArtifactError, type CommitFact } from '../../src/index.js'
+
+async function allFacts(brain: any): Promise<CommitFact[]> {
+  const scan = brain.scanFacts()
+  expect(scan).not.toBeNull()
+  const facts: CommitFact[] = []
+  for await (const batch of scan!.batches()) facts.push(...batch.facts)
+  return facts
+}
+
+describe('fact log dual-write (memory adapter)', () => {
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+  })
+
+  it('every single-op write appends its after-image fact; a remove appends a tombstone', async () => {
+    const id = await brain.add({ data: 'first', type: 'document', metadata: { rev: 1 } })
+    await brain.update({ id, metadata: { rev: 2 } })
+    await brain.remove(id)
+
+    const facts = await allFacts(brain)
+    // add + update + remove each committed a generation (the remove may span
+    // cascade ops but is ONE generation). Facts are monotonic.
+    const gens = facts.map((f) => f.generation)
+    expect([...gens].sort((a, b) => a - b)).toEqual(gens)
+    expect(facts.length).toBeGreaterThanOrEqual(3)
+
+    // The add fact carries the after-image of the new entity.
+    const addFact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null))
+    expect(addFact).toBeDefined()
+
+    // The remove fact carries a body-less tombstone for the id.
+    const removeFact = facts[facts.length - 1]
+    const tombstone = removeFact.ops.find((op) => op.id === id)
+    expect(tombstone).toBeDefined()
+    expect(tombstone!.record).toBeNull()
+    expect(tombstone!.kind).toBe('noun')
+  })
+
+  it('the update fact holds the NEW state (after-image, not before)', async () => {
+    const id = await brain.add({ data: 'versioned', type: 'document', metadata: { v: 'old' } })
+    await brain.update({ id, metadata: { v: 'new' } })
+
+    const facts = await allFacts(brain)
+    const updateFact = facts[facts.length - 1]
+    const op = updateFact.ops.find((o) => o.id === id)!
+    expect(op.record).not.toBeNull()
+    expect((op.record!.metadata as any).v).toBe('new')
+  })
+
+  it('a transact commits ONE fact carrying all its ops, with meta', async () => {
+    const receipt = await brain.transact(
+      [
+        { op: 'add', type: 'document', metadata: { part: 1 }, data: 'a' },
+        { op: 'add', type: 'document', metadata: { part: 2 }, data: 'b' }
+      ],
+      { meta: { source: 'batch-import' } }
+    )
+
+    const facts = await allFacts(brain)
+    const txFact = facts.find((f) => f.generation === receipt.generation)
+    expect(txFact).toBeDefined()
+    expect(txFact!.ops.filter((op) => op.kind === 'noun').length).toBeGreaterThanOrEqual(2)
+    expect(txFact!.meta).toEqual({ source: 'batch-import' })
+  })
+
+  it('an aborted transact leaves NO fact (absent = never committed)', async () => {
+    const id = await brain.add({ data: 'cas target', type: 'document', metadata: { n: 1 } })
+    const before = (await allFacts(brain)).length
+
+    await expect(
+      brain.transact([{ op: 'update', id, ifRev: 999, metadata: { n: 2 } }])
+    ).rejects.toThrow()
+
+    const after = await allFacts(brain)
+    expect(after.length).toBe(before)
+  })
+
+  it('fact generations line up with the transaction log', async () => {
+    await brain.add({ data: 'x', type: 'document', metadata: {} })
+    await brain.add({ data: 'y', type: 'document', metadata: {} })
+    await brain.flush()
+
+    const facts = await allFacts(brain)
+    const logGens = new Set((await brain.transactionLog()).map((e: any) => e.generation))
+    for (const f of facts) {
+      expect(logGens.has(f.generation)).toBe(true)
+    }
+  })
+
+  it('scan telemetry carries the frozen shape end-to-end', async () => {
+    for (let i = 0; i < 5; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
+
+    const scan = brain.scanFacts({ batchSize: 2 })!
+    expect(scan.headGeneration).toBeGreaterThanOrEqual(5)
+    expect(scan.approxFactCount).toBeGreaterThanOrEqual(5)
+    let batches = 0
+    for await (const b of scan.batches()) {
+      batches++
+      expect(b.factCount).toBe(b.facts.length)
+      expect(b.firstGeneration).toBe(b.facts[0].generation)
+      expect(b.lastGeneration).toBe(b.facts[b.facts.length - 1].generation)
+      expect(b.byteSize).toBeGreaterThan(0)
+      expect(typeof b.segmentId).toBe('string')
+    }
+    expect(batches).toBeGreaterThan(1)
+    expect(scan.summary().factsYielded).toBe(scan.approxFactCount)
+  })
+})
+
+describe('fact log dual-write (filesystem adapter — durability + protection)', () => {
+  let dir: string
+  let brain: any
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factlog-'))
+    brain = await open()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('facts survive close + reopen and appends continue monotonically', async () => {
+    const id = await brain.add({ data: 'persist me', type: 'document', metadata: { k: 1 } })
+    await brain.remove(id)
+    await brain.close()
+
+    brain = await open()
+    const facts = await allFacts(brain)
+    expect(facts.length).toBeGreaterThanOrEqual(2)
+    const headBefore = facts[facts.length - 1].generation
+
+    await brain.add({ data: 'after reopen', type: 'document', metadata: { k: 2 } })
+    const facts2 = await allFacts(brain)
+    expect(facts2[facts2.length - 1].generation).toBeGreaterThan(headBefore)
+  })
+
+  it('the fact segments exist on disk under _generations/facts/ with zero-padded names', async () => {
+    await brain.add({ data: 'on disk', type: 'document', metadata: {} })
+    await brain.flush()
+    const factsDir = path.join(dir, '_generations', 'facts')
+    const files = fs.readdirSync(factsDir)
+    // The manifest rides the store's JSON object discipline (gzip on disk).
+    expect(files.some((f) => f.startsWith('manifest.json'))).toBe(true)
+    const segs = files.filter((f) => /^seg-\d{20}\.bfl$/.test(f))
+    expect(segs.length).toBeGreaterThanOrEqual(1)
+  })
+
+  it('the fact-log namespace is PROTECTED: a prefix-nuke is refused', async () => {
+    await brain.add({ data: 'protected', type: 'document', metadata: {} })
+    await expect(brain.storage.removeRawPrefix('_generations/facts')).rejects.toBeInstanceOf(
+      ProtectedArtifactError
+    )
+    // Per-generation history cleanup remains unaffected (no false intersect).
+    await expect(brain.storage.removeRawPrefix('_generations/999999')).resolves.toBeUndefined()
+  })
+
+  it('transact facts are durable-on-return (no flush needed before reopen)', async () => {
+    const receipt = await brain.transact([
+      { op: 'add', type: 'document', metadata: { durable: true }, data: 'tx' }
+    ])
+    // Simulate an abrupt end: no flush(), no close() — reopen from disk.
+    brain = await open()
+    const facts = await allFacts(brain)
+    expect(facts.some((f) => f.generation === receipt.generation)).toBe(true)
+  })
+})

tests/unit/db/fact-log.test.ts

@@ -0,0 +1,189 @@
+/**
+ * @module tests/unit/db/fact-log
+ * @description The generation fact log in isolation: wire-format round-trip
+ * (positional msgpack facts, bin16 uuids, body-less tombstones), crc32c
+ * framing with torn-tail detection, open-time truncation to committed truth
+ * (the log can only ever be AHEAD after a crash; open cuts it back), rotation
+ * with a manifest-first flip, exactly-once scans with the frozen telemetry
+ * shape, and the mmap segment handoff excluding the mutable tail.
+ */
+import { describe, it, expect, beforeEach } from 'vitest'
+import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
+import {
+  FactLog,
+  FACTS_PREFIX,
+  type CommitFact,
+  type FactLogStorage,
+  storageSupportsFactLog
+} from '../../../src/db/factLog.js'
+import { crc32c } from '../../../src/utils/crc32c.js'
+
+const UUID = (n: number): string =>
+  `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
+
+const fact = (generation: number, overrides?: Partial<CommitFact>): CommitFact => ({
+  generation,
+  timestamp: 1_700_000_000_000 + generation,
+  ops: [
+    {
+      kind: 'noun',
+      id: UUID(generation),
+      record: { metadata: { noun: 'document', title: `doc ${generation}` }, vector: { v: [1, 2] } }
+    }
+  ],
+  ...overrides
+})
+
+describe('crc32c known-answer vectors', () => {
+  it('matches the RFC 3720 test vectors', () => {
+    expect(crc32c(new TextEncoder().encode('123456789'))).toBe(0xe3069283)
+    expect(crc32c(new Uint8Array(32))).toBe(0x8a9136aa)
+  })
+})
+
+describe('fact log — round-trip, framing, reconcile, rotation, scan', () => {
+  let storage: FactLogStorage
+  let log: FactLog
+
+  beforeEach(async () => {
+    const mem: any = new MemoryStorage()
+    await mem.init()
+    expect(storageSupportsFactLog(mem)).toBe(true)
+    storage = mem
+    log = new FactLog(storage)
+    await log.open(0)
+  })
+
+  it('facts round-trip byte-exactly: ops, tombstones, meta, blobHashes', async () => {
+    await log.append(fact(1))
+    await log.append(
+      fact(2, {
+        ops: [
+          { kind: 'verb', id: UUID(21), record: { metadata: { verb: 'contains' }, vector: null } },
+          { kind: 'noun', id: UUID(22), record: null } // TOMBSTONE
+        ],
+        meta: { source: 'test' },
+        blobHashes: ['abc123', 'abc123'] // multiset — duplicates preserved
+      })
+    )
+    await log.sync()
+
+    const scan = log.scanFacts()
+    expect(scan.headGeneration).toBe(2)
+    const all: CommitFact[] = []
+    for await (const batch of scan.batches()) all.push(...batch.facts)
+
+    expect(all).toHaveLength(2)
+    expect(all[0].generation).toBe(1)
+    expect(all[0].ops[0].id).toBe(UUID(1))
+    expect(all[0].ops[0].record?.metadata).toEqual({ noun: 'document', title: 'doc 1' })
+    expect(all[1].ops[0].kind).toBe('verb')
+    expect(all[1].ops[1].record).toBeNull() // the tombstone is body-less
+    expect(all[1].meta).toEqual({ source: 'test' })
+    expect(all[1].blobHashes).toEqual(['abc123', 'abc123'])
+    expect(scan.summary().factsYielded).toBe(2)
+  })
+
+  it('appends are monotonic — a replayed/duplicate generation throws', async () => {
+    await log.append(fact(5))
+    await expect(log.append(fact(5))).rejects.toThrow(/non-monotonic/)
+    await expect(log.append(fact(3))).rejects.toThrow(/non-monotonic/)
+    await expect(log.append(fact(6))).resolves.toBeUndefined() // gaps are fine (aborted reservations)
+  })
+
+  it('a torn tail (partial frame) is detected and ignored — intact prefix survives', async () => {
+    await log.append(fact(1))
+    await log.append(fact(2))
+    await log.sync()
+
+    // Simulate a crash mid-append: chop bytes off the tail file.
+    const tailPath = `${FACTS_PREFIX}/seg-${'1'.padStart(20, '0')}.bfl`
+    const bytes = (await storage.readRawBytes(tailPath))!
+    await storage.writeRawBytes(tailPath, bytes.subarray(0, bytes.length - 7))
+
+    const reopened = new FactLog(storage)
+    await reopened.open(2)
+    expect(reopened.headGeneration()).toBe(1) // fact 2's frame was torn → gone
+
+    const all: CommitFact[] = []
+    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([1])
+  })
+
+  it('open() truncates facts beyond committed truth (the crash-ahead shape)', async () => {
+    await log.append(fact(1))
+    await log.append(fact(2))
+    await log.append(fact(3))
+    await log.sync()
+
+    // The store's committed generation is 1 — facts 2..3 never committed.
+    const reopened = new FactLog(storage)
+    await reopened.open(1)
+    expect(reopened.headGeneration()).toBe(1)
+
+    const all: CommitFact[] = []
+    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([1])
+
+    // And appends continue cleanly from the truncated head.
+    await reopened.append(fact(2))
+    expect(reopened.headGeneration()).toBe(2)
+  })
+
+  it('a cleared store (committed=0) truncates everything', async () => {
+    await log.append(fact(1))
+    await log.append(fact(2))
+    await log.sync()
+    const reopened = new FactLog(storage)
+    await reopened.open(0)
+    expect(reopened.headGeneration()).toBe(0)
+  })
+
+  it('scan honors fromGeneration/toGeneration inclusively and filters kinds', async () => {
+    for (let g = 1; g <= 6; g++) await log.append(fact(g))
+    await log.sync()
+
+    const scan = log.scanFacts({ fromGeneration: 2, toGeneration: 4 })
+    const all: CommitFact[] = []
+    for await (const b of scan.batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([2, 3, 4])
+
+    const verbsOnly = log.scanFacts({ kinds: ['verb'] })
+    for await (const b of verbsOnly.batches()) {
+      for (const f of b.facts) expect(f.ops.every((op) => op.kind === 'verb')).toBe(true)
+    }
+  })
+
+  it('batch telemetry carries the frozen shape', async () => {
+    for (let g = 1; g <= 5; g++) await log.append(fact(g))
+    await log.sync()
+
+    const scan = log.scanFacts({ batchSize: 2 })
+    expect(scan.approxFactCount).toBe(5)
+    const batches = []
+    for await (const b of scan.batches()) batches.push(b)
+    expect(batches.length).toBe(3)
+    expect(batches[0]).toMatchObject({ firstGeneration: 1, lastGeneration: 2, factCount: 2 })
+    expect(batches[0].byteSize).toBeGreaterThan(0)
+    expect(typeof batches[0].segmentId).toBe('string')
+    expect(scan.summary()).toEqual({ factsYielded: 5, segmentsRead: 1 })
+  })
+
+  it('survives reopen: head and content come back from disk', async () => {
+    for (let g = 1; g <= 3; g++) await log.append(fact(g))
+    await log.sync()
+
+    const reopened = new FactLog(storage)
+    await reopened.open(3)
+    expect(reopened.headGeneration()).toBe(3)
+    const all: CommitFact[] = []
+    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
+    expect(all.map((f) => f.generation)).toEqual([1, 2, 3])
+  })
+
+  it('segmentPaths excludes the mutable tail (mmap handoff = sealed only)', async () => {
+    await log.append(fact(1))
+    await log.sync()
+    expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed
+  })
+})

tests/unit/db/generationStore.test.ts

@@ -249,7 +249,11 @@ describe('db/GenerationStore', () => {
       store.release(pinned)
       const result = await store.compact()
       expect(result.removedGenerations).toBeGreaterThan(0)
-      const remaining = await storage.listRawObjects(GENERATIONS_PREFIX)
+      // History record-sets only — the fact log (at `_generations/facts/`) is
+      // deliberately NOT reclaimed by history compaction.
+      const remaining = (await storage.listRawObjects(GENERATIONS_PREFIX)).filter(
+        (p: string) => !p.startsWith(`${GENERATIONS_PREFIX}/facts/`)
+      )
       expect(remaining).toEqual([])
     })