From 2888ae6b40124b8c1adbdf08770a2365594d5319 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 15 Jul 2026 10:54:20 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20entity-tree=20family=20stamp=20?= =?UTF-8?q?=E2=80=94=20sourceGeneration=20+=20rollup=20coherence=20at=20op?= =?UTF-8?q?en?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The canonical entity tree now carries a FAMILY STAMP (_system/family-stamps/entity-tree.json, JSON — forensics stay terminal-readable): which committed generation the tree reflects (sourceGeneration) plus the rollup invariants (entity/relationship counts) that verify a millions-of-files projection whole where per-file checks cannot scale. Written at flush/close boundaries; open-time coherence is a COMPARISON, not a walk: - coherent / absent (legacy store) → silent - behind → benign for the tree (it is written BY the commit; only the stamp is stale after a crash between commit and flush) — refreshes at the next flush - incoherent (counts diverged at equal generation, or a stamp AHEAD of the log head) → loud, names the failing invariant; repairIndex()'s unconditional recount heals and RE-STAMPS so repair leaves a coherent stamp behind - a fault reading the stamp is UNVERIFIABLE — never conflated with absence One verifier (verifyFamilyStamp, exported) reads both member modes: enumerated (exact byte size per member, bounded families) and rollup (invariants, unbounded families). New exports: readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH, FamilyStamp, StampMembers, StampVerdict. --- src/brainy.ts | 108 ++++++++++++++ src/db/familyStamp.ts | 147 ++++++++++++++++++++ src/index.ts | 5 + tests/integration/entity-tree-stamp.test.ts | 144 +++++++++++++++++++ 4 files changed, 404 insertions(+) create mode 100644 src/db/familyStamp.ts create mode 100644 tests/integration/entity-tree-stamp.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index f79ca562..b18834d8 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -168,6 +168,13 @@ import { } from './db/portableGraph.js' import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js' import type { FactScanHandle } from './db/factLog.js' +import { + ENTITY_TREE_STAMP_PATH, + readFamilyStamp, + verifyFamilyStamp, + writeFamilyStamp, + type FamilyStamp +} from './db/familyStamp.js' import { ChangeFeed, type BrainyChangeEvent, @@ -992,6 +999,12 @@ export class Brainy implements BrainyInterface { }) } + // Entity-tree stamp coherence: compare the stamped sourceGeneration + + // rollup invariants against the log head + live counters. Loud on + // genuine incoherence (repairIndex heals), silent on absent/coherent, + // benign-behind refreshes at the next flush. Never blocks open. + await this.verifyEntityTreeStamp() + // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — // after the store-open phase, but BEFORE any derived index or native @@ -10354,11 +10367,94 @@ export class Brainy implements BrainyInterface { // Db pins and an explicit autoCompact: false. await this.autoCompactHistory() + // 7. Stamp the entity tree: which source generation the canonical tree + // reflects + the rollup invariants that verify it whole (the counters + // persisted in step 1). Written at flush boundaries — the tree tracks + // every commit by construction, so the stamp is a durable checkpoint, + // not a per-commit cost. Open compares stamp vs log head + rollups. + await this.stampEntityTree() + const elapsed = Date.now() - startTime console.log(`All indexes flushed to disk in ${elapsed}ms`) } + /** + * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the + * committed generation the canonical tree reflects — equal by construction, + * since the tree is written by the commit itself) plus the rollup invariants + * (entity/relationship counts) that verify the tree whole where per-file + * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp}; + * healed by `repairIndex()`, whose unconditional recount rebuilds the + * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write + * fault warns loudly but never fails the flush that carried real data. + */ + private async stampEntityTree(): Promise { + if (this.isReadOnly) return + try { + const [nounCount, verbCount] = await Promise.all([ + this.storage.getNounCount(), + this.storage.getVerbCount() + ]) + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: this.generationStore.generation(), + members: { mode: 'rollup', invariants: { nounCount, verbCount } } + }) + } catch (error) { + prodLog.warn( + `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` + + `next successful flush): ${(error as Error).message}` + ) + } + } + + /** + * @description Open-time coherence check for the entity tree's family stamp: + * compare `sourceGeneration` against the log head and the stamped rollup + * invariants against the live counters. Verdicts: + * - `coherent` / `absent` (legacy store; first flush stamps) → silent. + * - `behind` → benign for the tree (it is written BY the commit; only the + * stamp is stale — a crash landed between commit and flush). Refreshed at + * the next flush. + * - `incoherent` → LOUD: the tree or its counters diverged from what was + * stamped — `repairIndex()` recounts from canonical and re-stamps. + * Never blocks open; a fault reading the stamp is surfaced as unverifiable, + * never conflated with absence. + */ + private async verifyEntityTreeStamp(): Promise { + let stamp: FamilyStamp | null + try { + stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH) + } catch (error) { + prodLog.warn( + `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` + + `${(error as Error).message}` + ) + return + } + const [nounCount, verbCount] = await Promise.all([ + this.storage.getNounCount(), + this.storage.getVerbCount() + ]) + const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + nounCount, + verbCount + }) + if (verdict.state === 'incoherent') { + prodLog.warn( + `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + + `The canonical tree or its counters diverged from the stamped state — run ` + + `brain.repairIndex() to recount from canonical and re-stamp.` + ) + } else if (verdict.state === 'behind') { + prodLog.debug( + `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` + + `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. @@ -15373,6 +15469,11 @@ export class Brainy implements BrainyInterface { await pruner.rebuildTypeCounts?.() await pruner.rebuildSubtypeCounts?.() + // The recount changed the rollup truth — re-stamp the entity tree so the + // stamp's invariants match the healed counters (repair leaves a coherent + // stamp, not a stale one that warns on the next open). + await this.stampEntityTree() + // VFS containment reconciliation: heal "cosmetic ghost" edges left by // pre-fix renames (an entity Contains-linked from BOTH its old and new // directory — readdir listed it in two places) and duplicate edges from @@ -15775,6 +15876,13 @@ export class Brainy implements BrainyInterface { })() ]) + // Stamp the entity tree at the close boundary (counters + counter now + // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT + // instead of benign-behind. Best-effort, never blocks the close. + if (this.generationStore && !this.isReadOnly) { + await this.stampEntityTree() + } + // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 await Promise.all([ diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts new file mode 100644 index 00000000..728ad5bb --- /dev/null +++ b/src/db/familyStamp.ts @@ -0,0 +1,147 @@ +/** + * @module db/familyStamp + * @description The generalized FAMILY STAMP — one JSON shape that declares, + * for any derived projection, WHICH source state it reflects and HOW to verify + * it is whole. The entity tree (canonical current-state files) carries the + * first brainy-side stamp; native index families carry the same shape. One + * verifier reads both member modes: + * + * - `enumerated` — bounded families: exact byte size per member file, + * verified at open. + * - `rollup` — unbounded families (the entity tree: millions of files): + * the verified surface is a small set of rollup invariants (entity/ + * relationship counts) plus `sourceGeneration`. + * + * `sourceGeneration` is the generation of the source-of-truth log this + * projection reflects — open-time coherence becomes a COMPARISON (stamp vs + * log head), not a walk: + * + * - equal + invariants hold → coherent, serve. + * - behind → the projection missed the tail (crash between commit and stamp); + * for the Stage-1 tree this is benign by construction (the tree is written + * BY the commit), so the stamp refreshes; a DERIVED projection would replay + * the gap instead. + * - invariants FAIL at equal generation → genuine incoherence: loud, and the + * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a + * canonical walk) heals it. + * + * Stamps are JSON on purpose — every incident gets debugged by reading a + * stamp in a terminal. + */ + +/** Storage-root-relative directory holding family stamps. */ +export const FAMILY_STAMPS_PREFIX = '_system/family-stamps' + +/** The entity tree's stamp path. */ +export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json` + +/** One enumerated member: a file and its exact expected byte size. */ +export interface EnumeratedMember { + path: string + bytes: number +} + +/** The stamp's verified surface, in one of the two member modes. */ +export type StampMembers = + | { mode: 'enumerated'; files: EnumeratedMember[] } + | { mode: 'rollup'; invariants: Record } + +/** The generalized family stamp (one shape, one verifier, both engines). */ +export interface FamilyStamp { + /** Which projection this stamps (e.g. `'entity-tree'`). */ + family: string + /** Monotonic per-family stamp generation — bumps on every committed stamp. */ + generation: number + /** ISO timestamp of the stamp write. */ + committedAt: string + /** The source-of-truth generation this projection reflects. */ + sourceGeneration: number + /** The verified surface. */ + members: StampMembers +} + +/** The verdict of an open-time stamp verification. */ +export type StampVerdict = + | { state: 'coherent' } + | { state: 'absent' } // legacy store — first stamp writes at the next flush + | { state: 'behind'; stampSource: number; head: number } + | { state: 'incoherent'; failures: string[] } + | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence + +/** The narrow storage surface stamps ride (JSON objects + fsync). */ +export interface StampStorage { + readRawObject(path: string): Promise + writeRawObject(path: string, data: any): Promise + syncRawObjects(paths: string[]): Promise +} + +/** Read a family's stamp; `null` when none was ever written. */ +export async function readFamilyStamp( + storage: StampStorage, + path: string +): Promise { + const stored = (await storage.readRawObject(path)) as FamilyStamp | null + if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null + return stored +} + +/** Write a family's stamp durably (atomic object write + fsync). */ +export async function writeFamilyStamp( + storage: StampStorage, + path: string, + stamp: Omit & { generation?: number } +): Promise { + const prior = await readFamilyStamp(storage, path) + const full: FamilyStamp = { + ...stamp, + generation: (prior?.generation ?? 0) + 1, + committedAt: new Date().toISOString() + } + await storage.writeRawObject(path, full) + await storage.syncRawObjects([path]) +} + +/** + * The ONE verifier, both member modes. `actual` supplies the observed rollup + * values (rollup mode) or file sizes (enumerated mode, keyed by path); + * `head` is the source-of-truth generation now. + */ +export function verifyFamilyStamp( + stamp: FamilyStamp | null, + head: number, + actual: Record +): StampVerdict { + if (stamp === null) return { state: 'absent' } + if (stamp.sourceGeneration > head) { + // A stamp AHEAD of the log claims state that never committed — the + // projection was stamped against truth that a crash rolled back. + return { + state: 'incoherent', + failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] + } + } + if (stamp.sourceGeneration < head) { + return { state: 'behind', stampSource: stamp.sourceGeneration, head } + } + const failures: string[] = [] + if (stamp.members.mode === 'rollup') { + for (const [name, expected] of Object.entries(stamp.members.invariants)) { + const observed = actual[name] + if (observed === undefined) { + failures.push(`rollup invariant '${name}' has no observed value`) + } else if (observed !== expected) { + failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`) + } + } + } else { + for (const member of stamp.members.files) { + const observed = actual[member.path] + if (observed === undefined) { + failures.push(`member '${member.path}' is missing`) + } else if (observed !== member.bytes) { + failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`) + } + } + } + return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' } +} diff --git a/src/index.ts b/src/index.ts index b9135600..bdb5ff73 100644 --- a/src/index.ts +++ b/src/index.ts @@ -204,6 +204,11 @@ export type { FactScanBatch, FactScanHandle } from './db/factLog.js' +// The generalized family stamp — which source generation a projection +// reflects + the surface that verifies it whole; one verifier, both member +// modes (enumerated byte-exact / rollup invariants). +export { readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH } from './db/familyStamp.js' +export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.js' // Optional provider capability for generation-aware native indexes export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts new file mode 100644 index 00000000..f76c5094 --- /dev/null +++ b/tests/integration/entity-tree-stamp.test.ts @@ -0,0 +1,144 @@ +/** + * @module tests/integration/entity-tree-stamp + * @description The entity tree's FAMILY STAMP: written at flush/close with + * `sourceGeneration` (the committed generation the canonical tree reflects) + * plus rollup invariants (entity/relationship counts); verified at open by + * comparison — coherent on a clean close, LOUD on genuine incoherence + * (tampered counters), healed by repairIndex()'s recount + re-stamp. One + * verifier reads both member modes. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { + Brainy, + readFamilyStamp, + verifyFamilyStamp, + ENTITY_TREE_STAMP_PATH, + type FamilyStamp +} from '../../src/index.js' +import { prodLog } from '../../src/utils/logger.js' + +describe('entity-tree family stamp', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-')) + brain = await open() + }) + afterEach(async () => { + vi.restoreAllMocks() + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => { + for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } }) + await brain.flush() + + const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(stamp).not.toBeNull() + expect(stamp.family).toBe('entity-tree') + expect(stamp.members.mode).toBe('rollup') + const invariants = (stamp.members as any).invariants + expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) + expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) + expect(stamp.sourceGeneration).toBe(brain.generation()) + expect(stamp.generation).toBeGreaterThanOrEqual(1) + }) + + it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => { + await brain.add({ data: 'clean', type: 'document', metadata: {} }) + await brain.close() + + const warn = vi.spyOn(prodLog, 'warn') + brain = await open() + const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp')) + expect(stampWarnings).toEqual([]) + }) + + it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => { + for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } }) + await brain.close() + + // Simulate counter drift AFTER the stamp was written: inflate the + // persisted scalar the way the historical decrement-skip did. + brain = await open() + ;(brain.storage as any).totalNounCount += 52 + await (brain.storage as any).persistCounts() + await brain.close() + // The close boundary re-stamps with the inflated counter — so tamper the + // STAMP instead for a deterministic mismatch: stamped counts differ from + // the (inflated) live ones at the NEXT open only if the stamp is older. + // Rewrite the stamp with the honest counts + current sourceGeneration. + const raw = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + const honest = { ...raw } + ;(honest.members as any).invariants.nounCount -= 52 + fs.writeFileSync( + path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`), + require('node:zlib').gzipSync(JSON.stringify(honest)) + ) + + const warn = vi.spyOn(prodLog, 'warn') + brain = await open() + const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) + expect(incoherent.length).toBeGreaterThanOrEqual(1) + expect(String(incoherent[0][0])).toMatch(/nounCount/) + + // The heal: recount from canonical + re-stamp → next open is quiet. + await brain.repairIndex() + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT')) + expect(stillIncoherent).toEqual([]) + }) + + it('the one verifier handles both member modes', () => { + const rollup: FamilyStamp = { + family: 'x', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 5, + members: { mode: 'rollup', invariants: { nounCount: 10 } } + } + expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' }) + expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent') + expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({ + state: 'behind', + stampSource: 5, + head: 9 + }) + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) + + const enumerated: FamilyStamp = { + family: 'y', + generation: 1, + committedAt: new Date().toISOString(), + sourceGeneration: 2, + members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] } + } + expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' }) + expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent') + expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member + }) +})