feat: entity-tree family stamp — sourceGeneration + rollup coherence at open

The canonical entity tree now carries a FAMILY STAMP
(_system/family-stamps/entity-tree.json, JSON — forensics stay
terminal-readable): which committed generation the tree reflects
(sourceGeneration) plus the rollup invariants (entity/relationship
counts) that verify a millions-of-files projection whole where per-file
checks cannot scale. Written at flush/close boundaries; open-time
coherence is a COMPARISON, not a walk:

- coherent / absent (legacy store) → silent
- behind → benign for the tree (it is written BY the commit; only the
  stamp is stale after a crash between commit and flush) — refreshes at
  the next flush
- incoherent (counts diverged at equal generation, or a stamp AHEAD of
  the log head) → loud, names the failing invariant; repairIndex()'s
  unconditional recount heals and RE-STAMPS so repair leaves a coherent
  stamp behind
- a fault reading the stamp is UNVERIFIABLE — never conflated with
  absence

One verifier (verifyFamilyStamp, exported) reads both member modes:
enumerated (exact byte size per member, bounded families) and rollup
(invariants, unbounded families). New exports: readFamilyStamp,
verifyFamilyStamp, ENTITY_TREE_STAMP_PATH, FamilyStamp, StampMembers,
StampVerdict.
This commit is contained in:
David Snelling 2026-07-15 10:54:20 -07:00
parent 38b0041464
commit 2888ae6b40
4 changed files with 404 additions and 0 deletions

147
src/db/familyStamp.ts Normal file
View file

@ -0,0 +1,147 @@
/**
* @module db/familyStamp
* @description The generalized FAMILY STAMP one JSON shape that declares,
* for any derived projection, WHICH source state it reflects and HOW to verify
* it is whole. The entity tree (canonical current-state files) carries the
* first brainy-side stamp; native index families carry the same shape. One
* verifier reads both member modes:
*
* - `enumerated` bounded families: exact byte size per member file,
* verified at open.
* - `rollup` unbounded families (the entity tree: millions of files):
* the verified surface is a small set of rollup invariants (entity/
* relationship counts) plus `sourceGeneration`.
*
* `sourceGeneration` is the generation of the source-of-truth log this
* projection reflects open-time coherence becomes a COMPARISON (stamp vs
* log head), not a walk:
*
* - equal + invariants hold coherent, serve.
* - behind the projection missed the tail (crash between commit and stamp);
* for the Stage-1 tree this is benign by construction (the tree is written
* BY the commit), so the stamp refreshes; a DERIVED projection would replay
* the gap instead.
* - invariants FAIL at equal generation genuine incoherence: loud, and the
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
* canonical walk) heals it.
*
* Stamps are JSON on purpose every incident gets debugged by reading a
* stamp in a terminal.
*/
/** Storage-root-relative directory holding family stamps. */
export const FAMILY_STAMPS_PREFIX = '_system/family-stamps'
/** The entity tree's stamp path. */
export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json`
/** One enumerated member: a file and its exact expected byte size. */
export interface EnumeratedMember {
path: string
bytes: number
}
/** The stamp's verified surface, in one of the two member modes. */
export type StampMembers =
| { mode: 'enumerated'; files: EnumeratedMember[] }
| { mode: 'rollup'; invariants: Record<string, number> }
/** The generalized family stamp (one shape, one verifier, both engines). */
export interface FamilyStamp {
/** Which projection this stamps (e.g. `'entity-tree'`). */
family: string
/** Monotonic per-family stamp generation — bumps on every committed stamp. */
generation: number
/** ISO timestamp of the stamp write. */
committedAt: string
/** The source-of-truth generation this projection reflects. */
sourceGeneration: number
/** The verified surface. */
members: StampMembers
}
/** The verdict of an open-time stamp verification. */
export type StampVerdict =
| { state: 'coherent' }
| { state: 'absent' } // legacy store — first stamp writes at the next flush
| { state: 'behind'; stampSource: number; head: number }
| { state: 'incoherent'; failures: string[] }
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
/** The narrow storage surface stamps ride (JSON objects + fsync). */
export interface StampStorage {
readRawObject(path: string): Promise<any | null>
writeRawObject(path: string, data: any): Promise<void>
syncRawObjects(paths: string[]): Promise<void>
}
/** Read a family's stamp; `null` when none was ever written. */
export async function readFamilyStamp(
storage: StampStorage,
path: string
): Promise<FamilyStamp | null> {
const stored = (await storage.readRawObject(path)) as FamilyStamp | null
if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null
return stored
}
/** Write a family's stamp durably (atomic object write + fsync). */
export async function writeFamilyStamp(
storage: StampStorage,
path: string,
stamp: Omit<FamilyStamp, 'generation' | 'committedAt'> & { generation?: number }
): Promise<void> {
const prior = await readFamilyStamp(storage, path)
const full: FamilyStamp = {
...stamp,
generation: (prior?.generation ?? 0) + 1,
committedAt: new Date().toISOString()
}
await storage.writeRawObject(path, full)
await storage.syncRawObjects([path])
}
/**
* The ONE verifier, both member modes. `actual` supplies the observed rollup
* values (rollup mode) or file sizes (enumerated mode, keyed by path);
* `head` is the source-of-truth generation now.
*/
export function verifyFamilyStamp(
stamp: FamilyStamp | null,
head: number,
actual: Record<string, number>
): StampVerdict {
if (stamp === null) return { state: 'absent' }
if (stamp.sourceGeneration > head) {
// A stamp AHEAD of the log claims state that never committed — the
// projection was stamped against truth that a crash rolled back.
return {
state: 'incoherent',
failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
}
}
if (stamp.sourceGeneration < head) {
return { state: 'behind', stampSource: stamp.sourceGeneration, head }
}
const failures: string[] = []
if (stamp.members.mode === 'rollup') {
for (const [name, expected] of Object.entries(stamp.members.invariants)) {
const observed = actual[name]
if (observed === undefined) {
failures.push(`rollup invariant '${name}' has no observed value`)
} else if (observed !== expected) {
failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`)
}
}
} else {
for (const member of stamp.members.files) {
const observed = actual[member.path]
if (observed === undefined) {
failures.push(`member '${member.path}' is missing`)
} else if (observed !== member.bytes) {
failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`)
}
}
}
return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' }
}