298cb6daca
Two halves of one defect, found by a seeded-SIGKILL crash lane.
THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation()
— the ALLOCATED counter, a number a write in flight has claimed and may
never commit — while the JSDoc beside it already said the source is the
committed generation. Every crash inside a write window therefore produced
a spurious verdict at the next open: either 'sourceGeneration N is ahead of
the log head N-1' (the allocated generation died with the process) or
'rollup invariant nounCount: stamped X, observed Y' (the recovery fold
folded facts the stamp's counts predate). Both told the operator to run
repairIndex() — a whole-store recount — for a store that was coherent.
Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store
raised one of the two. The stamp and the open now both read
committedGeneration(), which is what every other open-time watermark in the
class already reasons about.
THE TERMINAL VERDICT. A stamp still ahead of committed truth after the
recovery fold witnesses a generation that is not in the log — the stamp's
fsync outlived the tail's, and there is nothing to arrive. That is its own
verdict state now ('torn'), never folded in with 'incoherent': the two have
opposite cures. A writer open demotes it — the unusable stamped surface is
re-derived at the committed generation from the live counters, O(1),
straight-line, no loop and no await on external progress, narrated with
both count sets, the stamp's path and its committedAt. A read-only open
cannot re-stamp, so it says so and names the cure instead of guessing, and
still serves. Neither branch waits, and neither locks an owner out of a
canonical tree the stamp only describes.
Pins: the verifier returns the torn verdict with both generations; a
fabricated head-behind-source store narrates precisely, demotes inside a
bounded open, serves its rows, and is quiet at the next open (the demotion
converges); a read-only open narrates the same verdict and leaves the bytes
untouched.
263 lines
11 KiB
TypeScript
263 lines
11 KiB
TypeScript
/**
|
|
* @module tests/integration/entity-tree-stamp
|
|
* @description The entity tree's FAMILY STAMP: written at flush/close with
|
|
* `sourceGeneration` (the committed generation the canonical tree reflects)
|
|
* plus rollup invariants (entity/relationship counts); verified at open by
|
|
* comparison — coherent on a clean close, LOUD on genuine incoherence
|
|
* (tampered counters), healed by repairIndex()'s recount + re-stamp. One
|
|
* verifier reads both member modes.
|
|
*/
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
import * as fs from 'node:fs'
|
|
import * as os from 'node:os'
|
|
import * as path from 'node:path'
|
|
import {
|
|
Brainy,
|
|
readFamilyStamp,
|
|
verifyFamilyStamp,
|
|
ENTITY_TREE_STAMP_PATH,
|
|
type FamilyStamp
|
|
} from '../../src/index.js'
|
|
import { prodLog } from '../../src/utils/logger.js'
|
|
|
|
describe('entity-tree family stamp', () => {
|
|
let dir: string
|
|
let brain: any
|
|
|
|
const open = async () => {
|
|
const b: any = new Brainy({
|
|
requireSubtype: false,
|
|
storage: { type: 'filesystem', path: dir },
|
|
silent: true,
|
|
dimensions: 384
|
|
})
|
|
await b.init()
|
|
return b
|
|
}
|
|
|
|
beforeEach(async () => {
|
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-'))
|
|
brain = await open()
|
|
})
|
|
afterEach(async () => {
|
|
vi.restoreAllMocks()
|
|
await brain.close?.().catch(() => {})
|
|
fs.rmSync(dir, { recursive: true, force: true })
|
|
})
|
|
|
|
it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => {
|
|
for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } })
|
|
await brain.flush()
|
|
|
|
const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
|
|
expect(stamp).not.toBeNull()
|
|
expect(stamp.family).toBe('entity-tree')
|
|
expect(stamp.members.mode).toBe('rollup')
|
|
const invariants = (stamp.members as any).invariants
|
|
expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
|
|
expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
|
|
// THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the
|
|
// counter labelled the stamp with a generation a write in flight had merely
|
|
// claimed, so every crash inside a write window produced a spurious verdict
|
|
// at the next open (see the torn-tail pins below).
|
|
expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
|
expect(stamp.generation).toBeGreaterThanOrEqual(1)
|
|
})
|
|
|
|
it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => {
|
|
await brain.add({ data: 'clean', type: 'document', metadata: {} })
|
|
await brain.close()
|
|
|
|
const warn = vi.spyOn(prodLog, 'warn')
|
|
brain = await open()
|
|
const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp'))
|
|
expect(stampWarnings).toEqual([])
|
|
})
|
|
|
|
it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => {
|
|
for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
|
|
await brain.close()
|
|
|
|
// Simulate counter drift AFTER the stamp was written: inflate the
|
|
// persisted scalar the way the historical decrement-skip did.
|
|
brain = await open()
|
|
;(brain.storage as any).totalNounCount += 52
|
|
await (brain.storage as any).persistCounts()
|
|
await brain.close()
|
|
// The close boundary re-stamps with the inflated counter — so tamper the
|
|
// STAMP instead for a deterministic mismatch: stamped counts differ from
|
|
// the (inflated) live ones at the NEXT open only if the stamp is older.
|
|
// Rewrite the stamp with the honest counts + current sourceGeneration.
|
|
const raw = JSON.parse(
|
|
require('node:zlib')
|
|
.gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
|
|
.toString('utf-8')
|
|
) as FamilyStamp
|
|
const honest = { ...raw }
|
|
;(honest.members as any).invariants.nounCount -= 52
|
|
fs.writeFileSync(
|
|
path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`),
|
|
require('node:zlib').gzipSync(JSON.stringify(honest))
|
|
)
|
|
|
|
const warn = vi.spyOn(prodLog, 'warn')
|
|
brain = await open()
|
|
const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
|
|
expect(incoherent.length).toBeGreaterThanOrEqual(1)
|
|
expect(String(incoherent[0][0])).toMatch(/nounCount/)
|
|
|
|
// The heal: recount from canonical + re-stamp → next open is quiet.
|
|
await brain.repairIndex()
|
|
await brain.close()
|
|
const warn2 = vi.spyOn(prodLog, 'warn')
|
|
brain = await open()
|
|
const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
|
|
expect(stillIncoherent).toEqual([])
|
|
})
|
|
|
|
/**
|
|
* Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's
|
|
* committed watermark — the durable shape a torn generation-log tail leaves
|
|
* behind (the stamp's fsync outlived the tail's). Fabricated rather than
|
|
* crash-produced so the pin is deterministic; the seeded-SIGKILL lane
|
|
* (`scripts/crash-consistency.mjs` in the engine repo) produces the same
|
|
* shape from a real abrupt termination.
|
|
*/
|
|
const fabricateTear = (ahead: number): FamilyStamp => {
|
|
const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)
|
|
const zlib = require('node:zlib')
|
|
const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp
|
|
const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead }
|
|
fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn)))
|
|
return torn
|
|
}
|
|
|
|
it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => {
|
|
for (let i = 0; i < 3; i++)
|
|
await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } })
|
|
await brain.close()
|
|
const torn = fabricateTear(5)
|
|
|
|
const warn = vi.spyOn(prodLog, 'warn')
|
|
const startedAt = Date.now()
|
|
brain = await open()
|
|
const openMs = Date.now() - startedAt
|
|
|
|
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
|
expect(tearLines.length).toBe(1)
|
|
const said = String(tearLines[0][0])
|
|
// Narrated PRECISELY: both generations, the file, and the named cure.
|
|
expect(said).toContain(`source generation ${torn.sourceGeneration}`)
|
|
expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`)
|
|
expect(said).toContain(ENTITY_TREE_STAMP_PATH)
|
|
expect(said).toContain('DEMOTED')
|
|
expect(said).toMatch(/repairIndex\(\)/)
|
|
// Terminal, not a wait: the demotion is O(1) straight-line work, so a tear
|
|
// cannot turn an open into the 8-minute spin this class was reported as.
|
|
expect(openMs).toBeLessThan(30_000)
|
|
|
|
// The store SERVES — a tear in a stamp never locks an owner out of the
|
|
// canonical tree the stamp merely describes.
|
|
expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3)
|
|
|
|
// The demotion CONVERGED: the stamp now names committed truth, and the
|
|
// next open is quiet. A verdict that re-narrates every open is a wait
|
|
// wearing a different hat.
|
|
const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
|
|
expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
|
await brain.close()
|
|
const warn2 = vi.spyOn(prodLog, 'warn')
|
|
brain = await open()
|
|
expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([])
|
|
})
|
|
|
|
it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => {
|
|
await brain.add({ data: 'ro', type: 'document', metadata: {} })
|
|
await brain.close()
|
|
const torn = fabricateTear(3)
|
|
|
|
const warn = vi.spyOn(prodLog, 'warn')
|
|
const reader: any = await Brainy.openReadOnly({
|
|
requireSubtype: false,
|
|
storage: { type: 'filesystem', path: dir },
|
|
silent: true,
|
|
dimensions: 384
|
|
})
|
|
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
|
expect(tearLines.length).toBe(1)
|
|
const said = String(tearLines[0][0])
|
|
expect(said).toContain('READ-ONLY')
|
|
expect(said).toContain('UNVERIFIED')
|
|
expect(said).toMatch(/repairIndex\(\)/)
|
|
await reader.close()
|
|
|
|
// A reader never rewrites the store: read the bytes back off disk (not
|
|
// through a writer open, which would demote them) — the torn stamp is
|
|
// exactly as it was found.
|
|
const onDisk = JSON.parse(
|
|
require('node:zlib')
|
|
.gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
|
|
.toString('utf-8')
|
|
) as FamilyStamp
|
|
expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration)
|
|
expect(onDisk.generation).toBe(torn.generation)
|
|
|
|
brain = await open()
|
|
})
|
|
|
|
it('the one verifier handles both member modes', () => {
|
|
const rollup: FamilyStamp = {
|
|
family: 'x',
|
|
generation: 1,
|
|
committedAt: new Date().toISOString(),
|
|
sourceGeneration: 5,
|
|
members: { mode: 'rollup', invariants: { nounCount: 10 } }
|
|
}
|
|
expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' })
|
|
expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent')
|
|
expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({
|
|
state: 'behind',
|
|
stampSource: 5,
|
|
head: 9
|
|
})
|
|
// AHEAD is its own class — a torn generation-log tail, never folded in
|
|
// with `incoherent`: the two have opposite cures (recount vs demote).
|
|
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({
|
|
state: 'torn',
|
|
stampSource: 5,
|
|
head: 3
|
|
})
|
|
expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
|
|
|
|
const enumerated: FamilyStamp = {
|
|
family: 'y',
|
|
generation: 1,
|
|
committedAt: new Date().toISOString(),
|
|
sourceGeneration: 2,
|
|
members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] }
|
|
}
|
|
expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' })
|
|
expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent')
|
|
expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member
|
|
|
|
// Rollup invariants may be STRING fingerprints (e.g. a per-tree SHA-256):
|
|
// strict equality either way; a type mismatch reads as incoherence.
|
|
const fingerprinted: FamilyStamp = {
|
|
family: 'z',
|
|
generation: 1,
|
|
committedAt: new Date().toISOString(),
|
|
sourceGeneration: 3,
|
|
members: { mode: 'rollup', invariants: { treeDigest: 'abc123', rows: 42 } }
|
|
}
|
|
expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: 42 })).toEqual({
|
|
state: 'coherent'
|
|
})
|
|
expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'deadbeef', rows: 42 }).state).toBe(
|
|
'incoherent'
|
|
)
|
|
expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: '42' }).state).toBe(
|
|
'incoherent' // type mismatch never passes
|
|
)
|
|
})
|
|
})
|