feat: entity-tree family stamp — sourceGeneration + rollup coherence at open

The canonical entity tree now carries a FAMILY STAMP
(_system/family-stamps/entity-tree.json, JSON — forensics stay
terminal-readable): which committed generation the tree reflects
(sourceGeneration) plus the rollup invariants (entity/relationship
counts) that verify a millions-of-files projection whole where per-file
checks cannot scale. Written at flush/close boundaries; open-time
coherence is a COMPARISON, not a walk:

- coherent / absent (legacy store) → silent
- behind → benign for the tree (it is written BY the commit; only the
  stamp is stale after a crash between commit and flush) — refreshes at
  the next flush
- incoherent (counts diverged at equal generation, or a stamp AHEAD of
  the log head) → loud, names the failing invariant; repairIndex()'s
  unconditional recount heals and RE-STAMPS so repair leaves a coherent
  stamp behind
- a fault reading the stamp is UNVERIFIABLE — never conflated with
  absence

One verifier (verifyFamilyStamp, exported) reads both member modes:
enumerated (exact byte size per member, bounded families) and rollup
(invariants, unbounded families). New exports: readFamilyStamp,
verifyFamilyStamp, ENTITY_TREE_STAMP_PATH, FamilyStamp, StampMembers,
StampVerdict.
This commit is contained in:
David Snelling 2026-07-15 10:54:20 -07:00
parent 38b0041464
commit 2888ae6b40
4 changed files with 404 additions and 0 deletions

View file

@ -0,0 +1,144 @@
/**
* @module tests/integration/entity-tree-stamp
* @description The entity tree's FAMILY STAMP: written at flush/close with
* `sourceGeneration` (the committed generation the canonical tree reflects)
* plus rollup invariants (entity/relationship counts); verified at open by
* comparison coherent on a clean close, LOUD on genuine incoherence
* (tampered counters), healed by repairIndex()'s recount + re-stamp. One
* verifier reads both member modes.
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import {
Brainy,
readFamilyStamp,
verifyFamilyStamp,
ENTITY_TREE_STAMP_PATH,
type FamilyStamp
} from '../../src/index.js'
import { prodLog } from '../../src/utils/logger.js'
describe('entity-tree family stamp', () => {
let dir: string
let brain: any
const open = async () => {
const b: any = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true,
dimensions: 384
})
await b.init()
return b
}
beforeEach(async () => {
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-'))
brain = await open()
})
afterEach(async () => {
vi.restoreAllMocks()
await brain.close?.().catch(() => {})
fs.rmSync(dir, { recursive: true, force: true })
})
it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => {
for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } })
await brain.flush()
const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
expect(stamp).not.toBeNull()
expect(stamp.family).toBe('entity-tree')
expect(stamp.members.mode).toBe('rollup')
const invariants = (stamp.members as any).invariants
expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
expect(stamp.sourceGeneration).toBe(brain.generation())
expect(stamp.generation).toBeGreaterThanOrEqual(1)
})
it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => {
await brain.add({ data: 'clean', type: 'document', metadata: {} })
await brain.close()
const warn = vi.spyOn(prodLog, 'warn')
brain = await open()
const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp'))
expect(stampWarnings).toEqual([])
})
it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => {
for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
await brain.close()
// Simulate counter drift AFTER the stamp was written: inflate the
// persisted scalar the way the historical decrement-skip did.
brain = await open()
;(brain.storage as any).totalNounCount += 52
await (brain.storage as any).persistCounts()
await brain.close()
// The close boundary re-stamps with the inflated counter — so tamper the
// STAMP instead for a deterministic mismatch: stamped counts differ from
// the (inflated) live ones at the NEXT open only if the stamp is older.
// Rewrite the stamp with the honest counts + current sourceGeneration.
const raw = JSON.parse(
require('node:zlib')
.gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
.toString('utf-8')
) as FamilyStamp
const honest = { ...raw }
;(honest.members as any).invariants.nounCount -= 52
fs.writeFileSync(
path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`),
require('node:zlib').gzipSync(JSON.stringify(honest))
)
const warn = vi.spyOn(prodLog, 'warn')
brain = await open()
const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
expect(incoherent.length).toBeGreaterThanOrEqual(1)
expect(String(incoherent[0][0])).toMatch(/nounCount/)
// The heal: recount from canonical + re-stamp → next open is quiet.
await brain.repairIndex()
await brain.close()
const warn2 = vi.spyOn(prodLog, 'warn')
brain = await open()
const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
expect(stillIncoherent).toEqual([])
})
it('the one verifier handles both member modes', () => {
const rollup: FamilyStamp = {
family: 'x',
generation: 1,
committedAt: new Date().toISOString(),
sourceGeneration: 5,
members: { mode: 'rollup', invariants: { nounCount: 10 } }
}
expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' })
expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent')
expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({
state: 'behind',
stampSource: 5,
head: 9
})
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head
expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
const enumerated: FamilyStamp = {
family: 'y',
generation: 1,
committedAt: new Date().toISOString(),
sourceGeneration: 2,
members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] }
}
expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' })
expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent')
expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member
})
})