open-brainy/tests/integration/durability-kill-matrix.test.ts

/**
 * @module tests/integration/durability-kill-matrix
 * @description THE DURABILITY KILL MATRIX — for every step of the commit
 * path, inject a crash AT that step (the generation store's test-only fault
 * injector), then reopen the same storage directory with a brand-new Brainy
 * and assert the recovery contract BY CONSTRUCTION, not by timing:
 *
 *   - an ACKED write survives the crash (never a lost ack), and
 *   - an UN-ACKED write leaves no torn state (fully present or fully absent,
 *     never half).
 *
 * The crash simulation is honest process death: the crashed brain is NEVER
 * closed — `abandonAsCrashed` discards its buffered RAM state exactly as a
 * dead process would, and recovery on the next open is the only repair that
 * runs. File bytes already handed to the OS survive (process-crash model);
 * one row additionally models POWER LOSS by removing an entity's un-fsynced
 * canonical files (legal: single-op canonical writes are tmp+rename without
 * fsync).
 *
 * Matrix rows (fault point → durability barrier position):
 *
 *   BEFORE the barrier (nothing durable records the write):
 *     singleop-after-execute · singleop-after-fact-append · flush-after-staging
 *   AFTER partial durability (staged/synced bytes exist, manifest did not advance):
 *     flush-before-manifest · before-manifest-rename (transact) ·
 *     transact-after-fact-sync
 *   AFTER the commit point:
 *     after-manifest-rename (transact)
 *   MODE VARIANTS: singleop-after-fact-append under durable-at-ack.
 *   DISK FULL: one ENOSPC'd append — loud typed rejection, reads keep
 *     serving, a later write succeeds.
 *
 * Where the observed recovery contract differs from the ideal, the pin states
 * the OBSERVED behavior with a comment; where the observed behavior violates
 * "never a torn state / never a lost ack", the pin asserts the CONTRACT and
 * is marked `.fails` — a release-blocking finding, deliberately not weakened.
 */
import { describe, it, expect, afterEach } from 'vitest'
import * as fs from 'node:fs'
import { join } from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import {
  abandonAsCrashed,
  armCrash,
  dropCanonicalNoun,
  factGenerations,
  failNextAppendWithEnospc,
  generationDirExists,
  makeTempDir,
  openBrain,
  storeOf,
  uid,
  vec
} from '../helpers/durabilityKillMatrix.js'

describe('durability kill matrix — crash at every commit-path step, recover by reopen', () => {
  const dirs: string[] = []
  const liveBrains: Brainy[] = []
  // Crashed brains are deliberately NEVER closed (a dead process cannot
  // close); they are severed by abandonAsCrashed inside each test.

  function trackDir(): string {
    const dir = makeTempDir()
    dirs.push(dir)
    return dir
  }

  async function openLive(dir: string): Promise<Brainy> {
    const brain = await openBrain(dir)
    liveBrains.push(brain)
    return brain
  }

  afterEach(async () => {
    for (const brain of liveBrains.splice(0)) {
      try {
        await brain.close()
      } catch {
        // already closed / crashed mid-close — teardown only
      }
    }
    for (const dir of dirs.splice(0)) {
      await fs.promises.rm(dir, { recursive: true, force: true })
    }
  })

  /** Baseline arrangement: one durable row + explicit flush = the durable floor. */
  async function arrangeBaseline(label: string): Promise<{
    dir: string
    brain: Brainy
    baselineId: string
    floor: number
  }> {
    const dir = trackDir()
    const brain = await openBrain(dir) // NOT tracked live — most rows crash it
    const baselineId = uid(`${label}-baseline`)
    await brain.add({
      id: baselineId,
      data: 'baseline row',
      type: NounType.Document,
      vector: vec(1),
      metadata: { v: 1 }
    })
    await brain.flush()
    return { dir, brain, baselineId, floor: storeOf(brain).committedGeneration() }
  }

  /**
   * Flip a brain to durable-at-ack (log-authority) mode.
   *
   * NOT via `adoptLogAuthority()` (and the helper opens every brain with
   * `logAuthority: 'defer'`, opting out of the 10.0.0 adopt-at-open fleet
   * default): the sanctioned path runs the oracle and a baseline backfill,
   * which appends its own generation — shifting the floor arithmetic every
   * row pins. This helper flips the SAME switch the sanctioned path flips
   * (`setLogDurability('at-ack')`) and persists the SAME authority artifact,
   * so a reopened brain also runs in log-authority mode. The durability
   * semantics under test are governed entirely by that switch.
   */
  async function flipToAtAck(brain: Brainy): Promise<void> {
    const storage = (
      brain as unknown as {
        storage: {
          writeRawObject(p: string, d: unknown): Promise<void>
          syncRawObjects(p: string[]): Promise<void>
        }
      }
    ).storage
    await storage.writeRawObject('_system/log-authority.json', {
      authority: 'log',
      flippedAt: Date.now()
    })
    await storage.syncRawObjects(['_system/log-authority.json'])
    storeOf(brain).setLogDurability('at-ack')
  }

  // ==========================================================================
  // Rows BEFORE the durability barrier — the write never became durable-acked
  // ==========================================================================

  it('singleop-after-execute — un-acked write is atomic (present-whole), baseline and log stay at the floor', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('sae')
    const crashedId = uid('sae-crashed')
    const arm = armCrash(brain, 'singleop-after-execute')
    await expect(
      brain.add({
        id: crashedId,
        data: 'never acked',
        type: NounType.Document,
        vector: vec(2),
        metadata: { v: 2 }
      })
    ).rejects.toThrow('simulated process crash at singleop-after-execute')
    expect(arm.fired).toContain('singleop-after-execute')
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // Baseline intact.
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    // The log holds nothing beyond the committed watermark (no fact was ever
    // appended for the crashed write).
    expect(await factGenerations(reopened)).toEqual([floor])
    expect(storeOf(reopened).committedGeneration()).toBe(floor)
    // The un-acked write: Model-B applies the live canonical write BEFORE the
    // ack, so under process death its bytes survive — the row is PRESENT and
    // WHOLE by id (atomic, not torn). Under power loss the same un-fsynced
    // bytes may instead vanish entirely; both end states are atomic. NOTE the
    // divergence: the row is get()-visible but find()-invisible (no index
    // entry survived, no generation/fact records it, and no repair is pending
    // — a permanent canonical orphan; see the suite report).
    const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null
    expect(orphan).not.toBeNull()
    expect(orphan!.metadata.v).toBe(2) // whole, byte-consistent — never torn
    const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }>
    expect(found.map((f) => f.id)).toContain(baselineId)
    expect(found.map((f) => f.id)).not.toContain(crashedId)
    // A fresh write succeeds with a monotonic generation. The crashed
    // generation number is REUSED (nothing durable references it): the
    // counter reopened at the floor.
    expect(reopened.generation()).toBe(floor)
    const freshId = uid('sae-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh after recovery',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(floor + 1)
    expect(((await reopened.get(freshId)) as { metadata: { v: number } }).metadata.v).toBe(3)
  })

  it('singleop-after-fact-append (deferred mode) — the appended fact is truncated back at reopen', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('sfa')
    const crashedId = uid('sfa-crashed')
    const arm = armCrash(brain, 'singleop-after-fact-append')
    await expect(
      brain.add({
        id: crashedId,
        data: 'never acked',
        type: NounType.Document,
        vector: vec(2),
        metadata: { v: 2 }
      })
    ).rejects.toThrow('simulated process crash at singleop-after-fact-append')
    expect(arm.fired).toContain('singleop-after-fact-append')
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // The fact WAS appended to the log file before the crash (process death
    // keeps file bytes) — open() must truncate it back to the manifest
    // watermark, and does.
    expect(await factGenerations(reopened)).toEqual([floor])
    expect(storeOf(reopened).committedGeneration()).toBe(floor)
    // Baseline intact; un-acked row atomic (present-whole via canonical, as
    // in the singleop-after-execute row).
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    const orphan = (await reopened.get(crashedId)) as { metadata: { v: number } } | null
    expect(orphan).not.toBeNull()
    expect(orphan!.metadata.v).toBe(2)
    // Fresh write with a monotonic generation (crashed number reused — the
    // truncated fact freed it).
    expect(reopened.generation()).toBe(floor)
    const freshId = uid('sfa-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(floor + 1)
    expect(await factGenerations(reopened)).toEqual([floor, floor + 1])
  })

  it('flush-after-staging — the ACKED write survives (drop-without-restore); only the window history is lost', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('fas')
    const ackedId = uid('fas-acked')
    await brain.add({
      id: ackedId,
      data: 'acked before flush',
      type: NounType.Document,
      vector: vec(2),
      metadata: { v: 2 }
    })
    const ackedGen = storeOf(brain).generation()
    const arm = armCrash(brain, 'flush-after-staging')
    await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-after-staging')
    expect(arm.fired).toContain('flush-after-staging')
    // The crashed flush left the staged record-set dir on disk, above the manifest.
    expect(generationDirExists(dir, ackedGen)).toBe(true)
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // Recovery DROPPED the staged group-commit dir WITHOUT restoring its
    // before-images — restoring would silently revert an acknowledged write.
    expect(generationDirExists(dir, ackedGen)).toBe(false)
    expect(storeOf(reopened).committedGeneration()).toBe(floor)
    // NEVER A LOST ACK: the acknowledged write is present and whole.
    const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null
    expect(acked).not.toBeNull()
    expect(acked!.metadata.v).toBe(2)
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    // Recovery rolled generations back → index reconciliation ran → the acked
    // row is find()-visible too.
    const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }>
    expect(found.map((f) => f.id)).toEqual(expect.arrayContaining([baselineId, ackedId]))
    // The window's HISTORY is the documented cost: its fact is truncated back
    // (the acked row now lives only in canonical bytes, not the log).
    expect(await factGenerations(reopened)).toEqual([floor])
    // The crashed generation number is NOT reused (its dropped dir was seen
    // at open): fresh writes continue above it.
    expect(reopened.generation()).toBe(ackedGen)
    const freshId = uid('fas-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1)
  })

  // ==========================================================================
  // Rows AFTER partial durability — staged/synced bytes exist, no manifest
  // ==========================================================================

  it('flush-before-manifest — staged bytes + synced facts above the manifest are dropped/truncated; the acked write stays', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('fbm')
    const ackedId = uid('fbm-acked')
    await brain.add({
      id: ackedId,
      data: 'acked before flush',
      type: NounType.Document,
      vector: vec(2),
      metadata: { v: 2 }
    })
    const ackedGen = storeOf(brain).generation()
    const arm = armCrash(brain, 'flush-before-manifest')
    await expect(brain.flush()).rejects.toThrow('simulated process crash at flush-before-manifest')
    // The earlier flush phase passed through untripped before the target fired.
    expect(arm.fired).toContain('flush-after-staging')
    expect(arm.fired).toContain('flush-before-manifest')
    expect(generationDirExists(dir, ackedGen)).toBe(true)
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // Per the recovery contract in open(): groupCommit record-sets above the
    // manifest are dropped WITHOUT restore, and the (fsynced!) facts above
    // the manifest are truncated back. The acked live write stays.
    expect(generationDirExists(dir, ackedGen)).toBe(false)
    expect(storeOf(reopened).committedGeneration()).toBe(floor)
    expect(await factGenerations(reopened)).toEqual([floor])
    const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null
    expect(acked).not.toBeNull() // never a lost ack
    expect(acked!.metadata.v).toBe(2)
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    // Fresh write above the crashed generation (number not reused).
    expect(reopened.generation()).toBe(ackedGen)
    const freshId = uid('fbm-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(ackedGen + 1)
  })

  it('before-manifest-rename (transact) — fully staged, never committed: rolled back byte-identically', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('bmr')
    const newId = uid('bmr-new')
    const arm = armCrash(brain, 'before-manifest-rename')
    await expect(
      brain.transact([
        { op: 'update', id: baselineId, metadata: { v: 2 } },
        {
          op: 'add',
          id: newId,
          type: NounType.Document,
          data: 'uncommitted',
          vector: vec(2),
          metadata: { v: 2 }
        }
      ])
    ).rejects.toThrow('simulated process crash at before-manifest-rename')
    expect(arm.fired).toContain('before-manifest-rename')
    const txGen = storeOf(brain).generation()
    expect(generationDirExists(dir, txGen)).toBe(true)
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // Rolled back cleanly: the update is undone, the add is ABSENT everywhere.
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    expect(await reopened.get(newId)).toBeNull()
    const found = (await reopened.find({ type: NounType.Document, limit: 10 })) as Array<{ id: string }>
    expect(found.map((f) => f.id)).not.toContain(newId)
    expect(generationDirExists(dir, txGen)).toBe(false)
    expect(storeOf(reopened).committedGeneration()).toBe(floor)
    expect(await factGenerations(reopened)).toEqual([floor])
    // The crashed generation number is never reissued (counter persisted
    // before the crash point).
    expect(reopened.generation()).toBe(txGen)
    const freshId = uid('bmr-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1)
  })

  it('transact-after-fact-sync — the fsynced fact of an uncommitted transact is truncated back; rollback is clean', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('tfs')
    const newId = uid('tfs-new')
    const arm = armCrash(brain, 'transact-after-fact-sync')
    await expect(
      brain.transact([
        { op: 'update', id: baselineId, metadata: { v: 2 } },
        {
          op: 'add',
          id: newId,
          type: NounType.Document,
          data: 'uncommitted',
          vector: vec(2),
          metadata: { v: 2 }
        }
      ])
    ).rejects.toThrow('simulated process crash at transact-after-fact-sync')
    expect(arm.fired).toContain('transact-after-fact-sync')
    const txGen = storeOf(brain).generation()
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // The batch's fact was appended AND fsynced before the crash — open()
    // must truncate it back to the manifest watermark (the generation never
    // committed), and the before-images must restore byte-identically.
    expect(await factGenerations(reopened)).toEqual([floor])
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    expect(await reopened.get(newId)).toBeNull()
    expect(storeOf(reopened).committedGeneration()).toBe(floor)
    expect(generationDirExists(dir, txGen)).toBe(false)
    // Counter: the staged dir was seen at open, so the number is not reused.
    expect(reopened.generation()).toBe(txGen)
    const freshId = uid('tfs-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1)
  })

  // ==========================================================================
  // Row AFTER the commit point — the transaction must be kept
  // ==========================================================================

  it('after-manifest-rename (transact) — the manifest rename landed: the transaction is COMMITTED and fully present', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('amr')
    const newId = uid('amr-new')
    const arm = armCrash(brain, 'after-manifest-rename')
    await expect(
      brain.transact([
        { op: 'update', id: baselineId, metadata: { v: 2 } },
        {
          op: 'add',
          id: newId,
          type: NounType.Document,
          data: 'committed by the rename',
          vector: vec(2),
          metadata: { v: 2 }
        }
      ])
    ).rejects.toThrow('simulated process crash at after-manifest-rename')
    expect(arm.fired).toContain('after-manifest-rename')
    const txGen = storeOf(brain).generation()
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // COMMITTED: both operations present, atomically.
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(2)
    const added = (await reopened.get(newId)) as { metadata: { v: number } } | null
    expect(added).not.toBeNull()
    expect(added!.metadata.v).toBe(2)
    expect(storeOf(reopened).committedGeneration()).toBe(txGen)
    // The fact was synced before the commit point and sits at/below the
    // manifest — it is KEPT.
    expect(await factGenerations(reopened)).toEqual([floor, txGen])
    // Fresh writes continue above the committed generation.
    const freshId = uid('amr-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(txGen + 1)
  })

  // ==========================================================================
  // Durable-at-ack (log-authority) mode variants
  // ==========================================================================

  it('singleop-after-fact-append (at-ack mode) — the intact fact is REPLAYED at reopen; the write commits', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('aaf')
    await flipToAtAck(brain)
    const crashedId = uid('aaf-crashed')
    const arm = armCrash(brain, 'singleop-after-fact-append')
    await expect(
      brain.add({
        id: crashedId,
        data: 'fact fsynced, never acked',
        type: NounType.Document,
        vector: vec(2),
        metadata: { v: 2 }
      })
    ).rejects.toThrow('simulated process crash at singleop-after-fact-append')
    expect(arm.fired).toContain('singleop-after-fact-append')
    await abandonAsCrashed(brain)

    const reopened = await openLive(dir)
    // LOG-AUTHORITY RECOVERY CONTRACT: under 'log' authority, an intact
    // fact above the manifest is adopted at open — REPLAYED into canonical
    // and committed — never truncated. (At-least-once at the fact layer: a
    // crashed-pre-ack write whose fact survived intact becomes committed;
    // that is a valid write landing, never a torn or lost state.)
    expect(await factGenerations(reopened)).toEqual([floor, floor + 1])
    expect(storeOf(reopened).committedGeneration()).toBe(floor + 1)
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    const replayed = (await reopened.get(crashedId)) as { metadata: { v: number } } | null
    expect(replayed).not.toBeNull()
    expect(replayed!.metadata.v).toBe(2)
    // Fresh write lands monotonically ABOVE the replayed generation.
    const freshId = uid('aaf-fresh')
    await reopened.add({
      id: freshId,
      data: 'fresh',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await reopened.flush()
    expect(storeOf(reopened).committedGeneration()).toBe(floor + 2)
  })

  // THE AT-ACK CONTRACT, END TO END (was a release-blocking finding; fixed
  // by log-authority replay-at-open): under power loss the un-fsynced
  // tmp+rename canonical bytes legally vanish while the fsynced fact
  // survives — recovery REPLAYS that fact into canonical, so the acked
  // write lives. This is the sentence 'durable-at-ack' actually promises.
  it(
    'at-ack POWER LOSS — an ACKED write whose fact is fsynced SURVIVES reopen via log replay',
    async () => {
      const { dir, brain, baselineId } = await arrangeBaseline('apl')
      await flipToAtAck(brain)
      const ackedId = uid('apl-acked')
      // No fault injector: this write ACKS normally — in at-ack mode the ack
      // returned only after a covering log fsync.
      await brain.add({
        id: ackedId,
        data: 'acked, fact fsynced',
        type: NounType.Document,
        vector: vec(2),
        metadata: { v: 2 }
      })
      // Crash before any flush: RAM is gone…
      await abandonAsCrashed(brain)
      // …and power loss takes the un-fsynced canonical rename with it. The
      // fsynced fact log survives — it is the write's only durable copy.
      dropCanonicalNoun(dir, ackedId)

      const reopened = await openLive(dir)
      expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
      // THE AT-ACK CONTRACT: the acknowledged write survives the crash.
      // Observed today: open() truncates its fact back to the manifest
      // watermark and the write is gone everywhere.
      const acked = (await reopened.get(ackedId)) as { metadata: { v: number } } | null
      expect(acked).not.toBeNull()
      expect(acked!.metadata.v).toBe(2)
    }
  )

  // ==========================================================================
  // Disk full — one ENOSPC'd append
  // ==========================================================================

  it('disk full — an ENOSPC append rejects loudly and typed; reads keep serving; a later write succeeds', async () => {
    const { dir, brain, baselineId, floor } = await arrangeBaseline('nospc')
    liveBrains.push(brain) // this row never crashes the brain
    void dir
    const failedId = uid('nospc-failed')
    const probe = failNextAppendWithEnospc(brain)
    // LOUD, TYPED, never a silent success: the raw ENOSPC surfaces to the
    // caller with its errno code intact.
    await expect(
      brain.add({
        id: failedId,
        data: 'no space',
        type: NounType.Document,
        vector: vec(2),
        metadata: { v: 2 }
      })
    ).rejects.toMatchObject({ code: 'ENOSPC' })
    expect(probe.failed()).toBe(1)
    // The store still serves reads.
    expect(((await brain.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
    // Space "restored" (the failing patch self-cleared): a later write succeeds
    // end to end, including its fact and an explicit durability barrier.
    const laterId = uid('nospc-later')
    await brain.add({
      id: laterId,
      data: 'space restored',
      type: NounType.Document,
      vector: vec(3),
      metadata: { v: 3 }
    })
    await brain.flush()
    expect(((await brain.get(laterId)) as { metadata: { v: number } }).metadata.v).toBe(3)
    expect(storeOf(brain).committedGeneration()).toBeGreaterThan(floor)
    // FIXED BEHAVIOR (was: the rejected generation stayed buffered and the
    // next flush committed it with NO fact — a silent log gap): the failure
    // path un-buffers the generation and returns the counter reservation,
    // so the later write takes floor+1 and the log is gap-free.
    expect(storeOf(brain).committedGeneration()).toBe(floor + 1)
    expect(await factGenerations(brain)).toEqual([floor, floor + 1])
    // Canonical residue of the rejected write (execute ran before the
    // append failed) is the documented Model-B crash-equivalent orphan —
    // uncommitted, absent from the log, same shape as a crash at execute.
    expect(((await brain.get(failedId)) as { metadata: { v: number } } | null)?.metadata.v).toBe(2)
  })

  // THE NO-SILENT-COMMIT CONTRACT (was a release-blocking finding; fixed by
  // un-buffering on append failure): a loudly-rejected write never becomes
  // durably committed and the log never carries a gap. Canonical residue
  // (the execute-before-commit orphan) is the documented Model-B
  // crash-equivalent, pinned in the row above — NOT a commit.
  it('disk full — a write rejected for a failed fact append is NOT silently committed', async () => {
    const { brain, floor } = await arrangeBaseline('nogap')
    liveBrains.push(brain)
    const failedId = uid('nogap-failed')
    failNextAppendWithEnospc(brain)
    await expect(
      brain.add({
        id: failedId,
        data: 'no space',
        type: NounType.Document,
        vector: vec(2),
        metadata: { v: 2 }
      })
    ).rejects.toMatchObject({ code: 'ENOSPC' })
    await brain.flush()
    // THE CONTRACT: nothing was committed behind the caller's back — the
    // log carries no gap and no generation for the rejected write. (get()
    // still serves the canonical execute-residue orphan — the documented
    // Model-B crash-equivalent, pinned in the row above.)
    expect(storeOf(brain).committedGeneration()).toBe(floor)
    expect(await factGenerations(brain)).toEqual([floor])
  })

  // ==========================================================================
  // Block-layer power-loss findings (first dm-flakey run) — the three cures
  // ==========================================================================

  it('at-ack POWER LOSS BELOW THE MANIFEST — an unclean open folds the WHOLE log; acked writes committed before the flush still survive vanished canonical', async () => {
    const { dir, brain, baselineId } = await arrangeBaseline('wlf')
    await flipToAtAck(brain)
    const ackedA = uid('wlf-a')
    const ackedB = uid('wlf-b')
    await brain.add({ id: ackedA, data: 'below manifest one', type: NounType.Document, vector: vec(2), metadata: { v: 2 } })
    await brain.add({ id: ackedB, data: 'below manifest two', type: NounType.Document, vector: vec(3), metadata: { v: 3 } })
    // The group-commit flush advances the manifest OVER these generations —
    // but live canonical bytes are tmp+rename without per-file fsync, so a
    // power cut can still take them. The fsynced facts are the durable copy.
    await (brain as unknown as { flush(): Promise<void> }).flush()
    await abandonAsCrashed(brain) // no clean close → no clean-shutdown marker
    dropCanonicalNoun(dir, ackedA)
    dropCanonicalNoun(dir, ackedB)

    const reopened = await openLive(dir)
    // The whole-log fold restores BOTH rows from facts ≤ manifest.
    expect(((await reopened.get(ackedA)) as { metadata: { v: number } }).metadata.v).toBe(2)
    expect(((await reopened.get(ackedB)) as { metadata: { v: number } }).metadata.v).toBe(3)
    expect(((await reopened.get(baselineId)) as { metadata: { v: number } }).metadata.v).toBe(1)
  })

  it('clean-shutdown marker: a clean close writes it, the next open consumes it (no fold on the happy path)', async () => {
    const { dir, brain } = await arrangeBaseline('csm')
    await flipToAtAck(brain)
    await brain.close()
    liveBrains.splice(liveBrains.indexOf(brain), 1)
    // The adapter stores raw objects gzipped — accept either spelling.
    const markerExists = () =>
      fs.existsSync(join(dir, '_system', 'clean-shutdown.json')) ||
      fs.existsSync(join(dir, '_system', 'clean-shutdown.json.gz'))
    expect(markerExists(), 'clean close stamps the marker').toBe(true)

    const reopened = await openLive(dir)
    expect(markerExists(), 'open consumes the marker').toBe(false)
    await reopened.close()
    liveBrains.splice(liveBrains.indexOf(reopened), 1)
    expect(markerExists(), 'the next clean close re-stamps it').toBe(true)
  })

  it('torn writer lock (empty file) — open treats it as stale and recovers; never a permanent lockout', async () => {
    const { dir, brain } = await arrangeBaseline('tlk')
    await brain.close()
    liveBrains.splice(liveBrains.indexOf(brain), 1)
    // The power-loss shape: the lock file exists but is EMPTY (torn write).
    fs.writeFileSync(join(dir, 'locks', '_writer.lock'), '')

    const reopened = await openLive(dir) // must not throw 'contended'
    const fresh = uid('tlk-fresh')
    await reopened.add({ id: fresh, data: 'lock recovered', type: NounType.Document, vector: vec(4), metadata: { v: 4 } })
    expect(await reopened.get(fresh)).not.toBeNull()
  })

  it('pair guard: a metadata index without stampWatermark never crashes flush', async () => {
    const { brain } = await arrangeBaseline('psg')
    liveBrains.push(brain)
    // The native pair swaps the metadata manager; the replacement may not
    // carry the stamp method — flush must treat that as verdict-side rescan,
    // never a TypeError at the fan-out.
    ;(brain as unknown as { metadataIndex: { stampWatermark?: unknown } }).metadataIndex.stampWatermark = undefined
    await expect((brain as unknown as { flush(): Promise<void> }).flush()).resolves.toBeUndefined()
  })
})