fix(recovery): a torn generation-log tail is a terminal verdict, never a wait
Two halves of one defect, found by a seeded-SIGKILL crash lane.
THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation()
— the ALLOCATED counter, a number a write in flight has claimed and may
never commit — while the JSDoc beside it already said the source is the
committed generation. Every crash inside a write window therefore produced
a spurious verdict at the next open: either 'sourceGeneration N is ahead of
the log head N-1' (the allocated generation died with the process) or
'rollup invariant nounCount: stamped X, observed Y' (the recovery fold
folded facts the stamp's counts predate). Both told the operator to run
repairIndex() — a whole-store recount — for a store that was coherent.
Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store
raised one of the two. The stamp and the open now both read
committedGeneration(), which is what every other open-time watermark in the
class already reasons about.
THE TERMINAL VERDICT. A stamp still ahead of committed truth after the
recovery fold witnesses a generation that is not in the log — the stamp's
fsync outlived the tail's, and there is nothing to arrive. That is its own
verdict state now ('torn'), never folded in with 'incoherent': the two have
opposite cures. A writer open demotes it — the unusable stamped surface is
re-derived at the committed generation from the live counters, O(1),
straight-line, no loop and no await on external progress, narrated with
both count sets, the stamp's path and its committedAt. A read-only open
cannot re-stamp, so it says so and names the cure instead of guessing, and
still serves. Neither branch waits, and neither locks an owner out of a
canonical tree the stamp only describes.
Pins: the verifier returns the torn verdict with both generations; a
fabricated head-behind-source store narrates precisely, demotes inside a
bounded open, serves its rows, and is quiet at the next open (the demotion
converges); a read-only open narrates the same verdict and leaves the bytes
untouched.
(cherry picked from commit 298cb6daca)
This commit is contained in:
parent
ff39941b0a
commit
c99308710a
3 changed files with 241 additions and 16 deletions
|
|
@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => {
|
|||
const invariants = (stamp.members as any).invariants
|
||||
expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
|
||||
expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
|
||||
expect(stamp.sourceGeneration).toBe(brain.generation())
|
||||
// THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the
|
||||
// counter labelled the stamp with a generation a write in flight had merely
|
||||
// claimed, so every crash inside a write window produced a spurious verdict
|
||||
// at the next open (see the torn-tail pins below).
|
||||
expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
||||
expect(stamp.generation).toBeGreaterThanOrEqual(1)
|
||||
})
|
||||
|
||||
|
|
@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => {
|
|||
expect(stillIncoherent).toEqual([])
|
||||
})
|
||||
|
||||
/**
|
||||
* Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's
|
||||
* committed watermark — the durable shape a torn generation-log tail leaves
|
||||
* behind (the stamp's fsync outlived the tail's). Fabricated rather than
|
||||
* crash-produced so the pin is deterministic; the seeded-SIGKILL lane
|
||||
* (`scripts/crash-consistency.mjs` in the engine repo) produces the same
|
||||
* shape from a real abrupt termination.
|
||||
*/
|
||||
const fabricateTear = (ahead: number): FamilyStamp => {
|
||||
const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)
|
||||
const zlib = require('node:zlib')
|
||||
const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp
|
||||
const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead }
|
||||
fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn)))
|
||||
return torn
|
||||
}
|
||||
|
||||
it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => {
|
||||
for (let i = 0; i < 3; i++)
|
||||
await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } })
|
||||
await brain.close()
|
||||
const torn = fabricateTear(5)
|
||||
|
||||
const warn = vi.spyOn(prodLog, 'warn')
|
||||
const startedAt = Date.now()
|
||||
brain = await open()
|
||||
const openMs = Date.now() - startedAt
|
||||
|
||||
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
||||
expect(tearLines.length).toBe(1)
|
||||
const said = String(tearLines[0][0])
|
||||
// Narrated PRECISELY: both generations, the file, and the named cure.
|
||||
expect(said).toContain(`source generation ${torn.sourceGeneration}`)
|
||||
expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`)
|
||||
expect(said).toContain(ENTITY_TREE_STAMP_PATH)
|
||||
expect(said).toContain('DEMOTED')
|
||||
expect(said).toMatch(/repairIndex\(\)/)
|
||||
// Terminal, not a wait: the demotion is O(1) straight-line work, so a tear
|
||||
// cannot turn an open into the 8-minute spin this class was reported as.
|
||||
expect(openMs).toBeLessThan(30_000)
|
||||
|
||||
// The store SERVES — a tear in a stamp never locks an owner out of the
|
||||
// canonical tree the stamp merely describes.
|
||||
expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3)
|
||||
|
||||
// The demotion CONVERGED: the stamp now names committed truth, and the
|
||||
// next open is quiet. A verdict that re-narrates every open is a wait
|
||||
// wearing a different hat.
|
||||
const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
|
||||
expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
||||
await brain.close()
|
||||
const warn2 = vi.spyOn(prodLog, 'warn')
|
||||
brain = await open()
|
||||
expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([])
|
||||
})
|
||||
|
||||
it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => {
|
||||
await brain.add({ data: 'ro', type: 'document', metadata: {} })
|
||||
await brain.close()
|
||||
const torn = fabricateTear(3)
|
||||
|
||||
const warn = vi.spyOn(prodLog, 'warn')
|
||||
const reader: any = await Brainy.openReadOnly({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
||||
expect(tearLines.length).toBe(1)
|
||||
const said = String(tearLines[0][0])
|
||||
expect(said).toContain('READ-ONLY')
|
||||
expect(said).toContain('UNVERIFIED')
|
||||
expect(said).toMatch(/repairIndex\(\)/)
|
||||
await reader.close()
|
||||
|
||||
// A reader never rewrites the store: read the bytes back off disk (not
|
||||
// through a writer open, which would demote them) — the torn stamp is
|
||||
// exactly as it was found.
|
||||
const onDisk = JSON.parse(
|
||||
require('node:zlib')
|
||||
.gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
|
||||
.toString('utf-8')
|
||||
) as FamilyStamp
|
||||
expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration)
|
||||
expect(onDisk.generation).toBe(torn.generation)
|
||||
|
||||
brain = await open()
|
||||
})
|
||||
|
||||
it('the one verifier handles both member modes', () => {
|
||||
const rollup: FamilyStamp = {
|
||||
family: 'x',
|
||||
|
|
@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => {
|
|||
stampSource: 5,
|
||||
head: 9
|
||||
})
|
||||
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head
|
||||
// AHEAD is its own class — a torn generation-log tail, never folded in
|
||||
// with `incoherent`: the two have opposite cures (recount vs demote).
|
||||
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({
|
||||
state: 'torn',
|
||||
stampSource: 5,
|
||||
head: 3
|
||||
})
|
||||
expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
|
||||
|
||||
const enumerated: FamilyStamp = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue