fix(recovery): a torn generation-log tail is a terminal verdict, never a wait

Two halves of one defect, found by a seeded-SIGKILL crash lane.

THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation()
— the ALLOCATED counter, a number a write in flight has claimed and may
never commit — while the JSDoc beside it already said the source is the
committed generation. Every crash inside a write window therefore produced
a spurious verdict at the next open: either 'sourceGeneration N is ahead of
the log head N-1' (the allocated generation died with the process) or
'rollup invariant nounCount: stamped X, observed Y' (the recovery fold
folded facts the stamp's counts predate). Both told the operator to run
repairIndex() — a whole-store recount — for a store that was coherent.
Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store
raised one of the two. The stamp and the open now both read
committedGeneration(), which is what every other open-time watermark in the
class already reasons about.

THE TERMINAL VERDICT. A stamp still ahead of committed truth after the
recovery fold witnesses a generation that is not in the log — the stamp's
fsync outlived the tail's, and there is nothing to arrive. That is its own
verdict state now ('torn'), never folded in with 'incoherent': the two have
opposite cures. A writer open demotes it — the unusable stamped surface is
re-derived at the committed generation from the live counters, O(1),
straight-line, no loop and no await on external progress, narrated with
both count sets, the stamp's path and its committedAt. A read-only open
cannot re-stamp, so it says so and names the cure instead of guessing, and
still serves. Neither branch waits, and neither locks an owner out of a
canonical tree the stamp only describes.

Pins: the verifier returns the torn verdict with both generations; a
fabricated head-behind-source store narrates precisely, demotes inside a
bounded open, serves its rows, and is quiet at the next open (the demotion
converges); a read-only open narrates the same verdict and leaves the bytes
untouched.

(cherry picked from commit 298cb6daca)
This commit is contained in:
David Snelling 2026-08-31 09:07:18 -07:00 committed by David Snelling
parent ff39941b0a
commit c99308710a
3 changed files with 241 additions and 16 deletions

View file

@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => {
const invariants = (stamp.members as any).invariants
expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
expect(stamp.sourceGeneration).toBe(brain.generation())
// THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the
// counter labelled the stamp with a generation a write in flight had merely
// claimed, so every crash inside a write window produced a spurious verdict
// at the next open (see the torn-tail pins below).
expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration())
expect(stamp.generation).toBeGreaterThanOrEqual(1)
})
@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => {
expect(stillIncoherent).toEqual([])
})
/**
* Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's
* committed watermark the durable shape a torn generation-log tail leaves
* behind (the stamp's fsync outlived the tail's). Fabricated rather than
* crash-produced so the pin is deterministic; the seeded-SIGKILL lane
* (`scripts/crash-consistency.mjs` in the engine repo) produces the same
* shape from a real abrupt termination.
*/
const fabricateTear = (ahead: number): FamilyStamp => {
const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)
const zlib = require('node:zlib')
const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp
const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead }
fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn)))
return torn
}
it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => {
for (let i = 0; i < 3; i++)
await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } })
await brain.close()
const torn = fabricateTear(5)
const warn = vi.spyOn(prodLog, 'warn')
const startedAt = Date.now()
brain = await open()
const openMs = Date.now() - startedAt
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
expect(tearLines.length).toBe(1)
const said = String(tearLines[0][0])
// Narrated PRECISELY: both generations, the file, and the named cure.
expect(said).toContain(`source generation ${torn.sourceGeneration}`)
expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`)
expect(said).toContain(ENTITY_TREE_STAMP_PATH)
expect(said).toContain('DEMOTED')
expect(said).toMatch(/repairIndex\(\)/)
// Terminal, not a wait: the demotion is O(1) straight-line work, so a tear
// cannot turn an open into the 8-minute spin this class was reported as.
expect(openMs).toBeLessThan(30_000)
// The store SERVES — a tear in a stamp never locks an owner out of the
// canonical tree the stamp merely describes.
expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3)
// The demotion CONVERGED: the stamp now names committed truth, and the
// next open is quiet. A verdict that re-narrates every open is a wait
// wearing a different hat.
const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration())
await brain.close()
const warn2 = vi.spyOn(prodLog, 'warn')
brain = await open()
expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([])
})
it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => {
await brain.add({ data: 'ro', type: 'document', metadata: {} })
await brain.close()
const torn = fabricateTear(3)
const warn = vi.spyOn(prodLog, 'warn')
const reader: any = await Brainy.openReadOnly({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true,
dimensions: 384
})
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
expect(tearLines.length).toBe(1)
const said = String(tearLines[0][0])
expect(said).toContain('READ-ONLY')
expect(said).toContain('UNVERIFIED')
expect(said).toMatch(/repairIndex\(\)/)
await reader.close()
// A reader never rewrites the store: read the bytes back off disk (not
// through a writer open, which would demote them) — the torn stamp is
// exactly as it was found.
const onDisk = JSON.parse(
require('node:zlib')
.gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
.toString('utf-8')
) as FamilyStamp
expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration)
expect(onDisk.generation).toBe(torn.generation)
brain = await open()
})
it('the one verifier handles both member modes', () => {
const rollup: FamilyStamp = {
family: 'x',
@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => {
stampSource: 5,
head: 9
})
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head
// AHEAD is its own class — a torn generation-log tail, never folded in
// with `incoherent`: the two have opposite cures (recount vs demote).
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({
state: 'torn',
stampSource: 5,
head: 3
})
expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
const enumerated: FamilyStamp = {