/** * @module tests/integration/durability-kill-matrix * @description THE DURABILITY KILL MATRIX — for every step of the commit * path, inject a crash AT that step (the generation store's test-only fault * injector), then reopen the same storage directory with a brand-new Brainy * and assert the recovery contract BY CONSTRUCTION, not by timing: * * - an ACKED write survives the crash (never a lost ack), and * - an UN-ACKED write leaves no torn state (fully present or fully absent, * never half). * * The crash simulation is honest process death: the crashed brain is NEVER * closed — `abandonAsCrashed` discards its buffered RAM state exactly as a * dead process would, and recovery on the next open is the only repair that * runs. File bytes already handed to the OS survive (process-crash model); * one row additionally models POWER LOSS by removing an entity's un-fsynced * canonical files (legal: single-op canonical writes are tmp+rename without * fsync). * * Matrix rows (fault point → durability barrier position): * * BEFORE the barrier (nothing durable records the write): * singleop-after-execute · singleop-after-fact-append · flush-after-staging * AFTER partial durability (staged/synced bytes exist, manifest did not advance): * flush-before-manifest · before-manifest-rename (transact) · * transact-after-fact-sync * AFTER the commit point: * after-manifest-rename (transact) * MODE VARIANTS: singleop-after-fact-append under durable-at-ack. * DISK FULL: one ENOSPC'd append — loud typed rejection, reads keep * serving, a later write succeeds. * * Where the observed recovery contract differs from the ideal, the pin states * the OBSERVED behavior with a comment; where the observed behavior violates * "never a torn state / never a lost ack", the pin asserts the CONTRACT and * is marked `.fails` — a release-blocking finding, deliberately not weakened. */ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import { join } from 'node:path' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { abandonAsCrashed, armCrash, dropCanonicalNoun, factGenerations, failNextAppendWithEnospc, generationDirExists, makeTempDir, openBrain, storeOf, uid, vec } from '../helpers/durabilityKillMatrix.js' describe('durability kill matrix — crash at every commit-path step, recover by reopen', () => { const dirs: string[] = [] const liveBrains: Brainy[] = [] // Crashed brains are deliberately NEVER closed (a dead process cannot // close); they are severed by abandonAsCrashed inside each test. function trackDir(): string { const dir = makeTempDir() dirs.push(dir) return dir } async function openLive(dir: string): Promise { const brain = await openBrain(dir) liveBrains.push(brain) return brain } afterEach(async () => { for (const brain of liveBrains.splice(0)) { try { await brain.close() } catch { // already closed / crashed mid-close — teardown only } } for (const dir of dirs.splice(0)) { await fs.promises.rm(dir, { recursive: true, force: true }) } }) /** Baseline arrangement: one durable row + explicit flush = the durable floor. */ async function arrangeBaseline(label: string): Promise<{ dir: string brain: Brainy baselineId: string floor: number }> { const dir = trackDir() const brain = await openBrain(dir) // NOT tracked live — most rows crash it const baselineId = uid(`${label}-baseline`) await brain.add({ id: baselineId, data: 'baseline row', type: NounType.Document, vector: vec(1), metadata: { v: 1 } }) await brain.flush() return { dir, brain, baselineId, floor: storeOf(brain).committedGeneration() } } /** * Flip a brain to durable-at-ack (log-authority) mode. * * NOT via `adoptLogAuthority()` (and the helper opens every brain with * `logAuthority: 'defer'`, opting out of the 10.0.0 adopt-at-open fleet * default): the sanctioned path runs the oracle and a baseline backfill, * which appends its own generation — shifting the floor arithmetic every * row pins. This helper flips the SAME switch the sanctioned path flips * (`setLogDurability('at-ack')`) and persists the SAME authority artifact, * so a reopened brain also runs in log-authority mode. The durability * semantics under test are governed entirely by that switch. */ async function flipToAtAck(brain: Brainy): Promise { const storage = ( brain as unknown as { storage: { writeRawObject(p: string, d: unknown): Promise syncRawObjects(p: string[]): Promise } } ).storage await storage.writeRawObject('_system/log-authority.json', { authority: 'log', flippedAt: Date.now() }) await storage.syncRawObjects(['_system/log-authority.json']) storeOf(brain).setLogDurability('at-ack') } // ========================================================================== // Rows BEFORE the durability barrier — the write never became durable-acked // ========================================================================== it('singleop-after-execute — un-acked write is atomic (present-whole), baseline and log stay at the floor', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('sae') const crashedId = uid('sae-crashed') const arm = armCrash(brain, 'singleop-after-execute') await expect( brain.add({ id: crashedId, data: 'never acked', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) ).rejects.toThrow('simulated process crash at singleop-after-execute') expect(arm.fired).toContain('singleop-after-execute') await abandonAsCrashed(brain) const reopened = await openLive(dir) // Baseline intact. expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) // The log holds nothing beyond the committed watermark (no fact was ever // appended for the crashed write). expect(await factGenerations(reopened)).toEqual([floor]) expect(storeOf(reopened).committedGeneration()).toBe(floor) // The un-acked write: Model-B applies the live canonical write BEFORE the // ack, so under process death its bytes survive — the row is PRESENT and // WHOLE by id (atomic, not torn). Under power loss the same un-fsynced // bytes may instead vanish entirely; both end states are atomic. NOTE the // divergence: the row is get()-visible but find()-invisible (no index // entry survived, no generation/fact records it, and no repair is pending // — a permanent canonical orphan; see the suite report). const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null expect(orphan).not.toBeNull() expect(orphan!.metadata.v).toBe(2) // whole, byte-consistent — never torn const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }> expect(found.map((f) => f.id)).toContain(baselineId) expect(found.map((f) => f.id)).not.toContain(crashedId) // A fresh write succeeds with a monotonic generation. The crashed // generation number is REUSED (nothing durable references it): the // counter reopened at the floor. expect(reopened.generation()).toBe(floor) const freshId = uid('sae-fresh') await reopened.add({ id: freshId, data: 'fresh after recovery', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(floor + 1) expect(((await reopened.get(freshId)) as { metadata: { v: number } }).metadata.v).toBe(3) }) it('singleop-after-fact-append (deferred mode) — the appended fact is truncated back at reopen', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('sfa') const crashedId = uid('sfa-crashed') const arm = armCrash(brain, 'singleop-after-fact-append') await expect( brain.add({ id: crashedId, data: 'never acked', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) ).rejects.toThrow('simulated process crash at singleop-after-fact-append') expect(arm.fired).toContain('singleop-after-fact-append') await abandonAsCrashed(brain) const reopened = await openLive(dir) // The fact WAS appended to the log file before the crash (process death // keeps file bytes) — open() must truncate it back to the manifest // watermark, and does. expect(await factGenerations(reopened)).toEqual([floor]) expect(storeOf(reopened).committedGeneration()).toBe(floor) // Baseline intact; un-acked row atomic (present-whole via canonical, as // in the singleop-after-execute row). expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null expect(orphan).not.toBeNull() expect(orphan!.metadata.v).toBe(2) // Fresh write with a monotonic generation (crashed number reused — the // truncated fact freed it). expect(reopened.generation()).toBe(floor) const freshId = uid('sfa-fresh') await reopened.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(floor + 1) expect(await factGenerations(reopened)).toEqual([floor, floor + 1]) }) it('flush-after-staging — the ACKED write survives (drop-without-restore); only the window history is lost', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('fas') const ackedId = uid('fas-acked') await brain.add({ id: ackedId, data: 'acked before flush', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) const ackedGen = storeOf(brain).generation() const arm = armCrash(brain, 'flush-after-staging') await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-after-staging') expect(arm.fired).toContain('flush-after-staging') // The crashed flush left the staged record-set dir on disk, above the manifest. expect(generationDirExists(dir, ackedGen)).toBe(true) await abandonAsCrashed(brain) const reopened = await openLive(dir) // Recovery DROPPED the staged group-commit dir WITHOUT restoring its // before-images — restoring would silently revert an acknowledged write. expect(generationDirExists(dir, ackedGen)).toBe(false) expect(storeOf(reopened).committedGeneration()).toBe(floor) // NEVER A LOST ACK: the acknowledged write is present and whole. const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null expect(acked).not.toBeNull() expect(acked!.metadata.v).toBe(2) expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) // Recovery rolled generations back → index reconciliation ran → the acked // row is find()-visible too. const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }> expect(found.map((f) => f.id)).toEqual(expect.arrayContaining([baselineId, ackedId])) // The window's HISTORY is the documented cost: its fact is truncated back // (the acked row now lives only in canonical bytes, not the log). expect(await factGenerations(reopened)).toEqual([floor]) // The crashed generation number is NOT reused (its dropped dir was seen // at open): fresh writes continue above it. expect(reopened.generation()).toBe(ackedGen) const freshId = uid('fas-fresh') await reopened.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1) }) // ========================================================================== // Rows AFTER partial durability — staged/synced bytes exist, no manifest // ========================================================================== it('flush-before-manifest — staged bytes + synced facts above the manifest are dropped/truncated; the acked write stays', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('fbm') const ackedId = uid('fbm-acked') await brain.add({ id: ackedId, data: 'acked before flush', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) const ackedGen = storeOf(brain).generation() const arm = armCrash(brain, 'flush-before-manifest') await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-before-manifest') // The earlier flush phase passed through untripped before the target fired. expect(arm.fired).toContain('flush-after-staging') expect(arm.fired).toContain('flush-before-manifest') expect(generationDirExists(dir, ackedGen)).toBe(true) await abandonAsCrashed(brain) const reopened = await openLive(dir) // Per the recovery contract in open(): groupCommit record-sets above the // manifest are dropped WITHOUT restore, and the (fsynced!) facts above // the manifest are truncated back. The acked live write stays. expect(generationDirExists(dir, ackedGen)).toBe(false) expect(storeOf(reopened).committedGeneration()).toBe(floor) expect(await factGenerations(reopened)).toEqual([floor]) const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null expect(acked).not.toBeNull() // never a lost ack expect(acked!.metadata.v).toBe(2) expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) // Fresh write above the crashed generation (number not reused). expect(reopened.generation()).toBe(ackedGen) const freshId = uid('fbm-fresh') await reopened.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1) }) it('before-manifest-rename (transact) — fully staged, never committed: rolled back byte-identically', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('bmr') const newId = uid('bmr-new') const arm = armCrash(brain, 'before-manifest-rename') await expect( brain.transact([ { op: 'update', id: baselineId, metadata: { v: 2 } }, { op: 'add', id: newId, type: NounType.Document, data: 'uncommitted', vector: vec(2), metadata: { v: 2 } } ]) ).rejects.toThrow('simulated process crash at before-manifest-rename') expect(arm.fired).toContain('before-manifest-rename') const txGen = storeOf(brain).generation() expect(generationDirExists(dir, txGen)).toBe(true) await abandonAsCrashed(brain) const reopened = await openLive(dir) // Rolled back cleanly: the update is undone, the add is ABSENT everywhere. expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) expect(await reopened.get(newId)).toBeNull() const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }> expect(found.map((f) => f.id)).not.toContain(newId) expect(generationDirExists(dir, txGen)).toBe(false) expect(storeOf(reopened).committedGeneration()).toBe(floor) expect(await factGenerations(reopened)).toEqual([floor]) // The crashed generation number is never reissued (counter persisted // before the crash point). expect(reopened.generation()).toBe(txGen) const freshId = uid('bmr-fresh') await reopened.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1) }) it('transact-after-fact-sync — the fsynced fact of an uncommitted transact is truncated back; rollback is clean', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('tfs') const newId = uid('tfs-new') const arm = armCrash(brain, 'transact-after-fact-sync') await expect( brain.transact([ { op: 'update', id: baselineId, metadata: { v: 2 } }, { op: 'add', id: newId, type: NounType.Document, data: 'uncommitted', vector: vec(2), metadata: { v: 2 } } ]) ).rejects.toThrow('simulated process crash at transact-after-fact-sync') expect(arm.fired).toContain('transact-after-fact-sync') const txGen = storeOf(brain).generation() await abandonAsCrashed(brain) const reopened = await openLive(dir) // The batch's fact was appended AND fsynced before the crash — open() // must truncate it back to the manifest watermark (the generation never // committed), and the before-images must restore byte-identically. expect(await factGenerations(reopened)).toEqual([floor]) expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) expect(await reopened.get(newId)).toBeNull() expect(storeOf(reopened).committedGeneration()).toBe(floor) expect(generationDirExists(dir, txGen)).toBe(false) // Counter: the staged dir was seen at open, so the number is not reused. expect(reopened.generation()).toBe(txGen) const freshId = uid('tfs-fresh') await reopened.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1) }) // ========================================================================== // Row AFTER the commit point — the transaction must be kept // ========================================================================== it('after-manifest-rename (transact) — the manifest rename landed: the transaction is COMMITTED and fully present', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('amr') const newId = uid('amr-new') const arm = armCrash(brain, 'after-manifest-rename') await expect( brain.transact([ { op: 'update', id: baselineId, metadata: { v: 2 } }, { op: 'add', id: newId, type: NounType.Document, data: 'committed by the rename', vector: vec(2), metadata: { v: 2 } } ]) ).rejects.toThrow('simulated process crash at after-manifest-rename') expect(arm.fired).toContain('after-manifest-rename') const txGen = storeOf(brain).generation() await abandonAsCrashed(brain) const reopened = await openLive(dir) // COMMITTED: both operations present, atomically. expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(2) const added = (await reopened.get(newId)) as { metadata: { v: number } } | null expect(added).not.toBeNull() expect(added!.metadata.v).toBe(2) expect(storeOf(reopened).committedGeneration()).toBe(txGen) // The fact was synced before the commit point and sits at/below the // manifest — it is KEPT. expect(await factGenerations(reopened)).toEqual([floor, txGen]) // Fresh writes continue above the committed generation. const freshId = uid('amr-fresh') await reopened.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1) }) // ========================================================================== // Durable-at-ack (log-authority) mode variants // ========================================================================== it('singleop-after-fact-append (at-ack mode) — the intact fact is REPLAYED at reopen; the write commits', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('aaf') await flipToAtAck(brain) const crashedId = uid('aaf-crashed') const arm = armCrash(brain, 'singleop-after-fact-append') await expect( brain.add({ id: crashedId, data: 'fact fsynced, never acked', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) ).rejects.toThrow('simulated process crash at singleop-after-fact-append') expect(arm.fired).toContain('singleop-after-fact-append') await abandonAsCrashed(brain) const reopened = await openLive(dir) // LOG-AUTHORITY RECOVERY CONTRACT: under 'log' authority, an intact // fact above the manifest is adopted at open — REPLAYED into canonical // and committed — never truncated. (At-least-once at the fact layer: a // crashed-pre-ack write whose fact survived intact becomes committed; // that is a valid write landing, never a torn or lost state.) expect(await factGenerations(reopened)).toEqual([floor, floor + 1]) expect(storeOf(reopened).committedGeneration()).toBe(floor + 1) expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) const replayed = (await reopened.get(crashedId)) as { metadata: { v: number } } | null expect(replayed).not.toBeNull() expect(replayed!.metadata.v).toBe(2) // Fresh write lands monotonically ABOVE the replayed generation. const freshId = uid('aaf-fresh') await reopened.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await reopened.flush() expect(storeOf(reopened).committedGeneration()).toBe(floor + 2) }) // THE AT-ACK CONTRACT, END TO END (was a release-blocking finding; fixed // by log-authority replay-at-open): under power loss the un-fsynced // tmp+rename canonical bytes legally vanish while the fsynced fact // survives — recovery REPLAYS that fact into canonical, so the acked // write lives. This is the sentence 'durable-at-ack' actually promises. it( 'at-ack POWER LOSS — an ACKED write whose fact is fsynced SURVIVES reopen via log replay', async () => { const { dir, brain, baselineId } = await arrangeBaseline('apl') await flipToAtAck(brain) const ackedId = uid('apl-acked') // No fault injector: this write ACKS normally — in at-ack mode the ack // returned only after a covering log fsync. await brain.add({ id: ackedId, data: 'acked, fact fsynced', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) // Crash before any flush: RAM is gone… await abandonAsCrashed(brain) // …and power loss takes the un-fsynced canonical rename with it. The // fsynced fact log survives — it is the write's only durable copy. dropCanonicalNoun(dir, ackedId) const reopened = await openLive(dir) expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) // THE AT-ACK CONTRACT: the acknowledged write survives the crash. // Observed today: open() truncates its fact back to the manifest // watermark and the write is gone everywhere. const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null expect(acked).not.toBeNull() expect(acked!.metadata.v).toBe(2) } ) // ========================================================================== // Disk full — one ENOSPC'd append // ========================================================================== it('disk full — an ENOSPC append rejects loudly and typed; reads keep serving; a later write succeeds', async () => { const { dir, brain, baselineId, floor } = await arrangeBaseline('nospc') liveBrains.push(brain) // this row never crashes the brain void dir const failedId = uid('nospc-failed') const probe = failNextAppendWithEnospc(brain) // LOUD, TYPED, never a silent success: the raw ENOSPC surfaces to the // caller with its errno code intact. await expect( brain.add({ id: failedId, data: 'no space', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) ).rejects.toMatchObject({ code: 'ENOSPC' }) expect(probe.failed()).toBe(1) // The store still serves reads. expect(((await brain.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) // Space "restored" (the failing patch self-cleared): a later write succeeds // end to end, including its fact and an explicit durability barrier. const laterId = uid('nospc-later') await brain.add({ id: laterId, data: 'space restored', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) await brain.flush() expect(((await brain.get(laterId)) as { metadata: { v: number } }).metadata.v).toBe(3) expect(storeOf(brain).committedGeneration()).toBeGreaterThan(floor) // FIXED BEHAVIOR (was: the rejected generation stayed buffered and the // next flush committed it with NO fact — a silent log gap): the failure // path un-buffers the generation and returns the counter reservation, // so the later write takes floor+1 and the log is gap-free. expect(storeOf(brain).committedGeneration()).toBe(floor + 1) expect(await factGenerations(brain)).toEqual([floor, floor + 1]) // Canonical residue of the rejected write (execute ran before the // append failed) is the documented Model-B crash-equivalent orphan — // uncommitted, absent from the log, same shape as a crash at execute. expect(((await brain.get(failedId)) as { metadata: { v: number } } | null)?.metadata.v).toBe(2) }) // THE NO-SILENT-COMMIT CONTRACT (was a release-blocking finding; fixed by // un-buffering on append failure): a loudly-rejected write never becomes // durably committed and the log never carries a gap. Canonical residue // (the execute-before-commit orphan) is the documented Model-B // crash-equivalent, pinned in the row above — NOT a commit. it('disk full — a write rejected for a failed fact append is NOT silently committed', async () => { const { brain, floor } = await arrangeBaseline('nogap') liveBrains.push(brain) const failedId = uid('nogap-failed') failNextAppendWithEnospc(brain) await expect( brain.add({ id: failedId, data: 'no space', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) ).rejects.toMatchObject({ code: 'ENOSPC' }) await brain.flush() // THE CONTRACT: nothing was committed behind the caller's back — the // log carries no gap and no generation for the rejected write. (get() // still serves the canonical execute-residue orphan — the documented // Model-B crash-equivalent, pinned in the row above.) expect(storeOf(brain).committedGeneration()).toBe(floor) expect(await factGenerations(brain)).toEqual([floor]) }) // ========================================================================== // Block-layer power-loss findings (first dm-flakey run) — the three cures // ========================================================================== it('at-ack POWER LOSS BELOW THE MANIFEST — an unclean open folds the WHOLE log; acked writes committed before the flush still survive vanished canonical', async () => { const { dir, brain, baselineId } = await arrangeBaseline('wlf') await flipToAtAck(brain) const ackedA = uid('wlf-a') const ackedB = uid('wlf-b') await brain.add({ id: ackedA, data: 'below manifest one', type: NounType.Document, vector: vec(2), metadata: { v: 2 } }) await brain.add({ id: ackedB, data: 'below manifest two', type: NounType.Document, vector: vec(3), metadata: { v: 3 } }) // The group-commit flush advances the manifest OVER these generations — // but live canonical bytes are tmp+rename without per-file fsync, so a // power cut can still take them. The fsynced facts are the durable copy. await (brain as unknown as { flush(): Promise }).flush() await abandonAsCrashed(brain) // no clean close → no clean-shutdown marker dropCanonicalNoun(dir, ackedA) dropCanonicalNoun(dir, ackedB) const reopened = await openLive(dir) // The whole-log fold restores BOTH rows from facts ≤ manifest. expect(((await reopened.get(ackedA)) as { metadata: { v: number } }).metadata.v).toBe(2) expect(((await reopened.get(ackedB)) as { metadata: { v: number } }).metadata.v).toBe(3) expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1) }) it('clean-shutdown marker: a clean close writes it, the next open consumes it (no fold on the happy path)', async () => { const { dir, brain } = await arrangeBaseline('csm') await flipToAtAck(brain) await brain.close() liveBrains.splice(liveBrains.indexOf(brain), 1) // The adapter stores raw objects gzipped — accept either spelling. const markerExists = () => fs.existsSync(join(dir, '_system', 'clean-shutdown.json')) || fs.existsSync(join(dir, '_system', 'clean-shutdown.json.gz')) expect(markerExists(), 'clean close stamps the marker').toBe(true) const reopened = await openLive(dir) expect(markerExists(), 'open consumes the marker').toBe(false) await reopened.close() liveBrains.splice(liveBrains.indexOf(reopened), 1) expect(markerExists(), 'the next clean close re-stamps it').toBe(true) }) it('torn writer lock (empty file) — open treats it as stale and recovers; never a permanent lockout', async () => { const { dir, brain } = await arrangeBaseline('tlk') await brain.close() liveBrains.splice(liveBrains.indexOf(brain), 1) // The power-loss shape: the lock file exists but is EMPTY (torn write). fs.writeFileSync(join(dir, 'locks', '_writer.lock'), '') const reopened = await openLive(dir) // must not throw 'contended' const fresh = uid('tlk-fresh') await reopened.add({ id: fresh, data: 'lock recovered', type: NounType.Document, vector: vec(4), metadata: { v: 4 } }) expect(await reopened.get(fresh)).not.toBeNull() }) it('pair guard: a metadata index without stampWatermark never crashes flush', async () => { const { brain } = await arrangeBaseline('psg') liveBrains.push(brain) // The native pair swaps the metadata manager; the replacement may not // carry the stamp method — flush must treat that as verdict-side rescan, // never a TypeError at the fan-out. ;(brain as unknown as { metadataIndex: { stampWatermark?: unknown } }).metadataIndex.stampWatermark = undefined await expect((brain as unknown as { flush(): Promise }).flush()).resolves.toBeUndefined() }) })