/** * @module tests/integration/fold-checkpoint-bound * @description The fold-checkpoint bound (crash recovery's log fold, bounded): * `_system/fold-checkpoint.json` at generation G asserts every entity whose * latest fact is ≤ G has DURABLE canonical bytes — each stamp strictly follows * a canonical-sync barrier over every live entity touched since the last one * (stamp-after-data). An unclean open then folds only `(G, head]` instead of * the whole log. These pins prove the four load-bearing properties: * * 1. The stamp exists and tracks the committed watermark (flush + close). * 2. The fold is genuinely BOUNDED — facts ≤ G are skipped — while facts in * `(G, head]` are re-applied even BELOW the manifest. * 3. A failed barrier NEVER advances the stamp (the bound can lag, growing * a later fold — it can never overstate durability, losing a write). * 4. A pre-checkpoint brain (the 10.0 shape) bootstraps its chain at its * first whole-log fold; a tree-authority brain never stamps at all. */ import { describe, it, expect, afterEach, vi } from 'vitest' import * as fs from 'node:fs' import * as zlib from 'node:zlib' import { join } from 'node:path' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { abandonAsCrashed, dropCanonicalNoun, makeTempDir, openBrain, storeOf } from '../helpers/durabilityKillMatrix.js' const CHECKPOINT = join('_system', 'fold-checkpoint.json') /** Read the fold-checkpoint artifact's generation from disk, or null. */ function readCheckpoint(dir: string): number | null { for (const candidate of [join(dir, `${CHECKPOINT}.gz`), join(dir, CHECKPOINT)]) { if (!fs.existsSync(candidate)) continue const raw = fs.readFileSync(candidate) const text = candidate.endsWith('.gz') ? zlib.gunzipSync(raw).toString('utf8') : raw.toString('utf8') const parsed = JSON.parse(text) as { generation?: number } return Number.isSafeInteger(parsed.generation) ? (parsed.generation as number) : null } return null } function removeArtifact(dir: string, rel: string): void { for (const candidate of [join(dir, `${rel}.gz`), join(dir, rel)]) { fs.rmSync(candidate, { force: true }) } } function committedOf(brain: Brainy): number { return (storeOf(brain) as unknown as { committed: number }).committed } describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], never less durability than stamped', () => { const dirs: string[] = [] const liveBrains: Brainy[] = [] afterEach(async () => { vi.restoreAllMocks() for (const b of liveBrains.splice(0)) await b.close().catch(() => {}) for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) }) function trackDir(): string { const dir = makeTempDir() dirs.push(dir) return dir } it('a fresh adopt brain stamps at flush and again at close — the stamp tracks the committed watermark', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'adopt' }) liveBrains.push(brain) expect(brain.logAuthority().authority).toBe('log') await brain.add({ data: 'first', type: NounType.Document, metadata: { n: 1 } }) await brain.add({ data: 'second', type: NounType.Document, metadata: { n: 2 } }) await brain.flush() const afterFlush = readCheckpoint(dir) expect(afterFlush).toBe(committedOf(brain)) expect(afterFlush!).toBeGreaterThan(0) await brain.add({ data: 'third', type: NounType.Document, metadata: { n: 3 } }) const closingCommit = liveBrains.pop()! await closingCommit.close() // Close flushes, so the stamp advanced with it — and the clean-shutdown // marker it writes afterward never vouches for bytes the stamp has not. expect(readCheckpoint(dir)).toBeGreaterThanOrEqual(afterFlush!) }, 120000) it('BOUNDED fold: facts ≤ checkpoint are skipped, facts in (checkpoint, head] are re-applied even below the manifest; a failed barrier retains the old bound', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'adopt' }) liveBrains.push(brain) // Window 1 — flushed and stamped: the checkpoint's covered past. const idA = await brain.add({ data: 'covered by the stamp', type: NounType.Document, metadata: { w: 1 } }) await brain.flush() const checkpoint1 = readCheckpoint(dir) expect(checkpoint1).toBe(committedOf(brain)) // Window 2 — committed BELOW a new manifest but with the checkpoint stamp // FAILING: the barrier throws once, so the manifest advances while the // stamp stays at checkpoint1 (pin 3: a failed barrier never advances it). const storage = (brain as unknown as { storage: { syncEntityCanonical(n: string[], v: string[]): Promise } }).storage const realBarrier = storage.syncEntityCanonical.bind(storage) let failedOnce = false vi.spyOn(storage, 'syncEntityCanonical').mockImplementation(async (n: string[], v: string[]) => { if (!failedOnce) { failedOnce = true throw new Error('injected barrier failure (device hiccup)') } return realBarrier(n, v) }) const idB = await brain.add({ data: 'below manifest, above checkpoint', type: NounType.Document, metadata: { w: 2 } }) await brain.flush() expect(failedOnce).toBe(true) expect(readCheckpoint(dir)).toBe(checkpoint1) // stamp did NOT advance expect(committedOf(brain)).toBeGreaterThan(checkpoint1!) // manifest DID // Crash. Vaporize BOTH canonical records: idB's fact lives in // (checkpoint, manifest] — the bounded fold MUST restore it; idA's fact // is ≤ checkpoint — the fold must SKIP it (its loss here is synthetic: // the stamp's barrier fsynced it, a power cut cannot take it, and the // skip is exactly what makes the fold bounded instead of whole-log). await abandonAsCrashed(liveBrains.pop()!) dropCanonicalNoun(dir, idA) dropCanonicalNoun(dir, idB) const reopened = await openBrain(dir, { logAuthority: 'adopt' }) liveBrains.push(reopened) const restoredB = await reopened.get(idB) expect(restoredB, 'a fact above the checkpoint is re-applied even below the manifest').not.toBeNull() const skippedA = await reopened.get(idA) expect(skippedA, 'a fact at-or-below the checkpoint is outside the fold — the bound is real').toBeNull() // And recovery re-stamped at its new committed watermark. expect(readCheckpoint(dir)).toBe(committedOf(reopened)) }, 120000) it('a pre-checkpoint brain (the 10.0 shape) folds the WHOLE log once, then its chain is established', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'adopt' }) liveBrains.push(brain) const idA = await brain.add({ data: 'ten-point-oh resident', type: NounType.Document, metadata: { era: '10.0' } }) await brain.flush() await liveBrains.pop()!.close() // Rewind the brain to the 10.0 shape: no checkpoint artifact, and an // unclean shutdown (marker gone) — exactly what an existing fleet brain // looks like at its first crash under 10.1. removeArtifact(dir, CHECKPOINT) removeArtifact(dir, join('_system', 'clean-shutdown.json')) dropCanonicalNoun(dir, idA) const reopened = await openBrain(dir, { logAuthority: 'adopt' }) liveBrains.push(reopened) expect(await reopened.get(idA), 'no checkpoint ⇒ whole-log fold ⇒ every acked write restored').not.toBeNull() const stamped = readCheckpoint(dir) expect(stamped, 'the first whole-log fold is the chain’s base case — it stamps').toBe(committedOf(reopened)) }, 120000) it('a tree-authority brain never stamps a checkpoint', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'defer' }) liveBrains.push(brain) expect(brain.logAuthority().authority).not.toBe('log') await brain.add({ data: 'tree resident', type: NounType.Document, metadata: { n: 1 } }) await brain.flush() await liveBrains.pop()!.close() expect(readCheckpoint(dir)).toBeNull() }, 120000) it('a delete rides the barrier: the tombstoned id is in the synced set and the stamp advances past it', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'adopt' }) liveBrains.push(brain) const id = await brain.add({ data: 'short-lived', type: NounType.Document, metadata: { n: 1 } }) await brain.flush() const storage = (brain as unknown as { storage: { syncEntityCanonical(n: string[], v: string[]): Promise } }).storage const seen: string[][] = [] const realBarrier = storage.syncEntityCanonical.bind(storage) vi.spyOn(storage, 'syncEntityCanonical').mockImplementation(async (n: string[], v: string[]) => { seen.push([...n]) return realBarrier(n, v) }) await brain.remove(id) await brain.flush() expect( seen.some((nouns) => nouns.includes(id)), 'the deleted id must reach the canonical barrier (absence is durable state too)' ).toBe(true) expect(readCheckpoint(dir)).toBe(committedOf(brain)) }, 120000) })