feat(recovery): the fold-checkpoint bound — crash folds (checkpoint, head], never the whole log twice

The fold checkpoint (_system/fold-checkpoint.json) is stamped strictly after
a canonical-sync barrier over every live entity touched since the last stamp
(syncEntityCanonical: ids → canonical paths → fsync; an absent file fsyncs
its parent directory so deletes are as durable as writes). An unclean open
under log authority now folds only (checkpoint, head]; the chain bootstraps
at an empty brain's adoption (three-phase hooks around adoptLogAuthority) or
at a brain's first whole-log fold — existing brains converge at their first
crash with zero regression. Rollback restores sync immediately; abort paths
feed the barrier; a failed barrier retains the old bound (bigger fold later,
never a lost write). Five structural pins including boundedness itself.

Also: the production-shaped write-flow gate leg (mixed traffic racing
flushes, crash mid-traffic, every ack survives — from a consumer-reported
gate miss), and two release-ceremony cures (tag-first push so the publish
never queues behind the release commit's CI run; raw-curl npmjs shasum
probe with propagation grace instead of a one-shot false divergence).

tests/integration/fold-checkpoint-bound.test.ts

@@ -0,0 +1,200 @@
+/**
+ * @module tests/integration/fold-checkpoint-bound
+ * @description The fold-checkpoint bound (crash recovery's log fold, bounded):
+ * `_system/fold-checkpoint.json` at generation G asserts every entity whose
+ * latest fact is ≤ G has DURABLE canonical bytes — each stamp strictly follows
+ * a canonical-sync barrier over every live entity touched since the last one
+ * (stamp-after-data). An unclean open then folds only `(G, head]` instead of
+ * the whole log. These pins prove the four load-bearing properties:
+ *
+ *   1. The stamp exists and tracks the committed watermark (flush + close).
+ *   2. The fold is genuinely BOUNDED — facts ≤ G are skipped — while facts in
+ *      `(G, head]` are re-applied even BELOW the manifest.
+ *   3. A failed barrier NEVER advances the stamp (the bound can lag, growing
+ *      a later fold — it can never overstate durability, losing a write).
+ *   4. A pre-checkpoint brain (the 10.0 shape) bootstraps its chain at its
+ *      first whole-log fold; a tree-authority brain never stamps at all.
+ */
+import { describe, it, expect, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as zlib from 'node:zlib'
+import { join } from 'node:path'
+import { Brainy } from '../../src/brainy.js'
+import { NounType } from '../../src/types/graphTypes.js'
+import {
+  abandonAsCrashed,
+  dropCanonicalNoun,
+  makeTempDir,
+  openBrain,
+  storeOf
+} from '../helpers/durabilityKillMatrix.js'
+
+const CHECKPOINT = join('_system', 'fold-checkpoint.json')
+
+/** Read the fold-checkpoint artifact's generation from disk, or null. */
+function readCheckpoint(dir: string): number | null {
+  for (const candidate of [join(dir, `${CHECKPOINT}.gz`), join(dir, CHECKPOINT)]) {
+    if (!fs.existsSync(candidate)) continue
+    const raw = fs.readFileSync(candidate)
+    const text = candidate.endsWith('.gz') ? zlib.gunzipSync(raw).toString('utf8') : raw.toString('utf8')
+    const parsed = JSON.parse(text) as { generation?: number }
+    return Number.isSafeInteger(parsed.generation) ? (parsed.generation as number) : null
+  }
+  return null
+}
+
+function removeArtifact(dir: string, rel: string): void {
+  for (const candidate of [join(dir, `${rel}.gz`), join(dir, rel)]) {
+    fs.rmSync(candidate, { force: true })
+  }
+}
+
+function committedOf(brain: Brainy): number {
+  return (storeOf(brain) as unknown as { committed: number }).committed
+}
+
+describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], never less durability than stamped', () => {
+  const dirs: string[] = []
+  const liveBrains: Brainy[] = []
+  afterEach(async () => {
+    vi.restoreAllMocks()
+    for (const b of liveBrains.splice(0)) await b.close().catch(() => {})
+    for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
+  })
+  function trackDir(): string {
+    const dir = makeTempDir()
+    dirs.push(dir)
+    return dir
+  }
+
+  it('a fresh adopt brain stamps at flush and again at close — the stamp tracks the committed watermark', async () => {
+    const dir = trackDir()
+    const brain = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(brain)
+    expect(brain.logAuthority().authority).toBe('log')
+
+    await brain.add({ data: 'first', type: NounType.Document, metadata: { n: 1 } })
+    await brain.add({ data: 'second', type: NounType.Document, metadata: { n: 2 } })
+    await brain.flush()
+    const afterFlush = readCheckpoint(dir)
+    expect(afterFlush).toBe(committedOf(brain))
+    expect(afterFlush!).toBeGreaterThan(0)
+
+    await brain.add({ data: 'third', type: NounType.Document, metadata: { n: 3 } })
+    const closingCommit = liveBrains.pop()!
+    await closingCommit.close()
+    // Close flushes, so the stamp advanced with it — and the clean-shutdown
+    // marker it writes afterward never vouches for bytes the stamp has not.
+    expect(readCheckpoint(dir)).toBeGreaterThanOrEqual(afterFlush!)
+  }, 120000)
+
+  it('BOUNDED fold: facts ≤ checkpoint are skipped, facts in (checkpoint, head] are re-applied even below the manifest; a failed barrier retains the old bound', async () => {
+    const dir = trackDir()
+    const brain = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(brain)
+
+    // Window 1 — flushed and stamped: the checkpoint's covered past.
+    const idA = await brain.add({ data: 'covered by the stamp', type: NounType.Document, metadata: { w: 1 } })
+    await brain.flush()
+    const checkpoint1 = readCheckpoint(dir)
+    expect(checkpoint1).toBe(committedOf(brain))
+
+    // Window 2 — committed BELOW a new manifest but with the checkpoint stamp
+    // FAILING: the barrier throws once, so the manifest advances while the
+    // stamp stays at checkpoint1 (pin 3: a failed barrier never advances it).
+    const storage = (brain as unknown as {
+      storage: { syncEntityCanonical(n: string[], v: string[]): Promise<void> }
+    }).storage
+    const realBarrier = storage.syncEntityCanonical.bind(storage)
+    let failedOnce = false
+    vi.spyOn(storage, 'syncEntityCanonical').mockImplementation(async (n: string[], v: string[]) => {
+      if (!failedOnce) {
+        failedOnce = true
+        throw new Error('injected barrier failure (device hiccup)')
+      }
+      return realBarrier(n, v)
+    })
+    const idB = await brain.add({ data: 'below manifest, above checkpoint', type: NounType.Document, metadata: { w: 2 } })
+    await brain.flush()
+    expect(failedOnce).toBe(true)
+    expect(readCheckpoint(dir)).toBe(checkpoint1) // stamp did NOT advance
+    expect(committedOf(brain)).toBeGreaterThan(checkpoint1!) // manifest DID
+
+    // Crash. Vaporize BOTH canonical records: idB's fact lives in
+    // (checkpoint, manifest] — the bounded fold MUST restore it; idA's fact
+    // is ≤ checkpoint — the fold must SKIP it (its loss here is synthetic:
+    // the stamp's barrier fsynced it, a power cut cannot take it, and the
+    // skip is exactly what makes the fold bounded instead of whole-log).
+    await abandonAsCrashed(liveBrains.pop()!)
+    dropCanonicalNoun(dir, idA)
+    dropCanonicalNoun(dir, idB)
+
+    const reopened = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(reopened)
+    const restoredB = await reopened.get(idB)
+    expect(restoredB, 'a fact above the checkpoint is re-applied even below the manifest').not.toBeNull()
+    const skippedA = await reopened.get(idA)
+    expect(skippedA, 'a fact at-or-below the checkpoint is outside the fold — the bound is real').toBeNull()
+    // And recovery re-stamped at its new committed watermark.
+    expect(readCheckpoint(dir)).toBe(committedOf(reopened))
+  }, 120000)
+
+  it('a pre-checkpoint brain (the 10.0 shape) folds the WHOLE log once, then its chain is established', async () => {
+    const dir = trackDir()
+    const brain = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(brain)
+    const idA = await brain.add({ data: 'ten-point-oh resident', type: NounType.Document, metadata: { era: '10.0' } })
+    await brain.flush()
+    await liveBrains.pop()!.close()
+
+    // Rewind the brain to the 10.0 shape: no checkpoint artifact, and an
+    // unclean shutdown (marker gone) — exactly what an existing fleet brain
+    // looks like at its first crash under 10.1.
+    removeArtifact(dir, CHECKPOINT)
+    removeArtifact(dir, join('_system', 'clean-shutdown.json'))
+    dropCanonicalNoun(dir, idA)
+
+    const reopened = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(reopened)
+    expect(await reopened.get(idA), 'no checkpoint ⇒ whole-log fold ⇒ every acked write restored').not.toBeNull()
+    const stamped = readCheckpoint(dir)
+    expect(stamped, 'the first whole-log fold is the chain’s base case — it stamps').toBe(committedOf(reopened))
+  }, 120000)
+
+  it('a tree-authority brain never stamps a checkpoint', async () => {
+    const dir = trackDir()
+    const brain = await openBrain(dir, { logAuthority: 'defer' })
+    liveBrains.push(brain)
+    expect(brain.logAuthority().authority).not.toBe('log')
+    await brain.add({ data: 'tree resident', type: NounType.Document, metadata: { n: 1 } })
+    await brain.flush()
+    await liveBrains.pop()!.close()
+    expect(readCheckpoint(dir)).toBeNull()
+  }, 120000)
+
+  it('a delete rides the barrier: the tombstoned id is in the synced set and the stamp advances past it', async () => {
+    const dir = trackDir()
+    const brain = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(brain)
+    const id = await brain.add({ data: 'short-lived', type: NounType.Document, metadata: { n: 1 } })
+    await brain.flush()
+
+    const storage = (brain as unknown as {
+      storage: { syncEntityCanonical(n: string[], v: string[]): Promise<void> }
+    }).storage
+    const seen: string[][] = []
+    const realBarrier = storage.syncEntityCanonical.bind(storage)
+    vi.spyOn(storage, 'syncEntityCanonical').mockImplementation(async (n: string[], v: string[]) => {
+      seen.push([...n])
+      return realBarrier(n, v)
+    })
+
+    await brain.remove(id)
+    await brain.flush()
+    expect(
+      seen.some((nouns) => nouns.includes(id)),
+      'the deleted id must reach the canonical barrier (absence is durable state too)'
+    ).toBe(true)
+    expect(readCheckpoint(dir)).toBe(committedOf(brain))
+  }, 120000)
+})

tests/integration/write-flow-production-shape.test.ts

@@ -0,0 +1,149 @@
+/**
+ * @module tests/integration/write-flow-production-shape
+ * @description The production-shaped WRITE-FLOW gate leg. A downstream
+ * deployment's release gate went all-green on snapshots and rehearsal reads
+ * while two write-path defects (pad-frame constructibility, a counter rewind
+ * after a successful append) waited in ordinary WRITE flows — deferred
+ * embedding retries plus background history-flush concurrency wearing the
+ * stacks. This leg runs that exact shape, permanently:
+ *
+ *   - concurrent mixed writes (adds, deferred-embed adds, updates, removes)
+ *   - racing explicit flushes (the history tier's group commit, mid-traffic)
+ *   - then the three laws: every ack is readable truth, the fact log is
+ *     STRICTLY ascending end-to-end, and no write is ever refused.
+ *
+ * Part two crashes the brain mid-traffic (no close — RAM discarded) and
+ * requires every acked write back after reopen: the at-ack contract under
+ * the same production shape, not under a synthetic single write.
+ */
+import { describe, it, expect, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import { Brainy } from '../../src/brainy.js'
+import { NounType } from '../../src/types/graphTypes.js'
+import {
+  abandonAsCrashed,
+  factGenerations,
+  makeTempDir,
+  openBrain
+} from '../helpers/durabilityKillMatrix.js'
+
+describe('write-flow production shape — the pair gate leg from a consumer-reported miss', () => {
+  const dirs: string[] = []
+  const liveBrains: Brainy[] = []
+  afterEach(async () => {
+    for (const b of liveBrains.splice(0)) await b.close().catch(() => {})
+    for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
+  })
+  function trackDir(): string {
+    const dir = makeTempDir()
+    dirs.push(dir)
+    return dir
+  }
+
+  async function runTrafficWave(
+    brain: Brainy,
+    wave: number,
+    perWave: number
+  ): Promise<{ kept: string[]; removed: string[] }> {
+    const kept: string[] = []
+    const removed: string[] = []
+    const work: Promise<unknown>[] = []
+    for (let i = 0; i < perWave; i++) {
+      const n = wave * perWave + i
+      if (i % 4 === 0) {
+        // Deferred-embed add — the retry-marker flow that wore the defect.
+        work.push(
+          brain
+            .add({ data: `deferred payload ${n}`, type: NounType.Document, metadata: { n, defer: true }, deferEmbedding: true })
+            .then((id) => void kept.push(id))
+        )
+      } else if (i % 4 === 1) {
+        // Add, then update it in the same wave (two generations, same id).
+        work.push(
+          brain.add({ data: `versioned payload ${n}`, type: NounType.Document, metadata: { n, v: 1 } }).then(async (id) => {
+            kept.push(id)
+            await brain.update({ id, metadata: { n, v: 2 } })
+          })
+        )
+      } else if (i % 4 === 2) {
+        // Add, then remove — a durable tombstone is an ack too.
+        work.push(
+          brain.add({ data: `ephemeral payload ${n}`, type: NounType.Document, metadata: { n } }).then(async (id) => {
+            await brain.remove(id)
+            removed.push(id)
+          })
+        )
+      } else {
+        work.push(
+          brain.add({ data: `plain payload ${n}`, type: NounType.Document, metadata: { n } }).then((id) => void kept.push(id))
+        )
+      }
+      // Race the history tier's group commit against live traffic.
+      if (i % 5 === 3) work.push(brain.flush())
+    }
+    // NO REFUSALS: every promise must resolve — a single rejection here is
+    // the refusal-loop costume this leg exists to catch.
+    await Promise.all(work)
+    return { kept, removed }
+  }
+
+  it('three waves of mixed traffic with racing flushes: every ack is truth, the log is strictly ascending, nothing refused', async () => {
+    const dir = trackDir()
+    const brain = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(brain)
+    expect(brain.logAuthority().authority).toBe('log')
+
+    const kept: string[] = []
+    const removed: string[] = []
+    for (let wave = 0; wave < 3; wave++) {
+      const result = await runTrafficWave(brain, wave, 20)
+      kept.push(...result.kept)
+      removed.push(...result.removed)
+    }
+    await brain.flush()
+
+    for (const id of kept) {
+      expect(await brain.get(id), `acked write ${id} must be readable truth`).not.toBeNull()
+    }
+    for (const id of removed) {
+      expect(await brain.get(id), `acked remove ${id} must hold`).toBeNull()
+    }
+
+    const gens = await factGenerations(brain)
+    expect(gens.length).toBeGreaterThan(0)
+    for (let i = 1; i < gens.length; i++) {
+      expect(gens[i], 'fact log strictly ascending end-to-end').toBeGreaterThan(gens[i - 1])
+    }
+
+    // Clean reopen: the same truth survives a restart.
+    await liveBrains.pop()!.close()
+    const reopened = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(reopened)
+    for (const id of kept.slice(0, 10)) {
+      expect(await reopened.get(id)).not.toBeNull()
+    }
+  }, 240000)
+
+  it('crash mid-traffic: every acked write survives the reopen (the at-ack law under the production shape)', async () => {
+    const dir = trackDir()
+    const brain = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(brain)
+
+    const { kept, removed } = await runTrafficWave(brain, 0, 24)
+    // No close, no flush — the process "dies" holding its RAM.
+    await abandonAsCrashed(liveBrains.pop()!)
+
+    const reopened = await openBrain(dir, { logAuthority: 'adopt' })
+    liveBrains.push(reopened)
+    for (const id of kept) {
+      expect(await reopened.get(id), `acked write ${id} must survive the crash`).not.toBeNull()
+    }
+    for (const id of removed) {
+      expect(await reopened.get(id), `acked remove ${id} must survive the crash`).toBeNull()
+    }
+    const gens = await factGenerations(reopened)
+    for (let i = 1; i < gens.length; i++) {
+      expect(gens[i], 'fact log strictly ascending after recovery').toBeGreaterThan(gens[i - 1])
+    }
+  }, 240000)
+})