feat: entity-tree family stamp — sourceGeneration + rollup coherence at open
The canonical entity tree now carries a FAMILY STAMP (_system/family-stamps/entity-tree.json, JSON — forensics stay terminal-readable): which committed generation the tree reflects (sourceGeneration) plus the rollup invariants (entity/relationship counts) that verify a millions-of-files projection whole where per-file checks cannot scale. Written at flush/close boundaries; open-time coherence is a COMPARISON, not a walk: - coherent / absent (legacy store) → silent - behind → benign for the tree (it is written BY the commit; only the stamp is stale after a crash between commit and flush) — refreshes at the next flush - incoherent (counts diverged at equal generation, or a stamp AHEAD of the log head) → loud, names the failing invariant; repairIndex()'s unconditional recount heals and RE-STAMPS so repair leaves a coherent stamp behind - a fault reading the stamp is UNVERIFIABLE — never conflated with absence One verifier (verifyFamilyStamp, exported) reads both member modes: enumerated (exact byte size per member, bounded families) and rollup (invariants, unbounded families). New exports: readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH, FamilyStamp, StampMembers, StampVerdict.
This commit is contained in:
parent
38b0041464
commit
2888ae6b40
4 changed files with 404 additions and 0 deletions
108
src/brainy.ts
108
src/brainy.ts
|
|
@ -168,6 +168,13 @@ import {
|
|||
} from './db/portableGraph.js'
|
||||
import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js'
|
||||
import type { FactScanHandle } from './db/factLog.js'
|
||||
import {
|
||||
ENTITY_TREE_STAMP_PATH,
|
||||
readFamilyStamp,
|
||||
verifyFamilyStamp,
|
||||
writeFamilyStamp,
|
||||
type FamilyStamp
|
||||
} from './db/familyStamp.js'
|
||||
import {
|
||||
ChangeFeed,
|
||||
type BrainyChangeEvent,
|
||||
|
|
@ -992,6 +999,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
})
|
||||
}
|
||||
|
||||
// Entity-tree stamp coherence: compare the stamped sourceGeneration +
|
||||
// rollup invariants against the log head + live counters. Loud on
|
||||
// genuine incoherence (repairIndex heals), silent on absent/coherent,
|
||||
// benign-behind refreshes at the next flush. Never blocks open.
|
||||
await this.verifyEntityTreeStamp()
|
||||
|
||||
// 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
|
||||
// marker (`_system/brain-format.json`) into an in-memory field NOW —
|
||||
// after the store-open phase, but BEFORE any derived index or native
|
||||
|
|
@ -10354,11 +10367,94 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Db pins and an explicit autoCompact: false.
|
||||
await this.autoCompactHistory()
|
||||
|
||||
// 7. Stamp the entity tree: which source generation the canonical tree
|
||||
// reflects + the rollup invariants that verify it whole (the counters
|
||||
// persisted in step 1). Written at flush boundaries — the tree tracks
|
||||
// every commit by construction, so the stamp is a durable checkpoint,
|
||||
// not a per-commit cost. Open compares stamp vs log head + rollups.
|
||||
await this.stampEntityTree()
|
||||
|
||||
const elapsed = Date.now() - startTime
|
||||
|
||||
console.log(`All indexes flushed to disk in ${elapsed}ms`)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the
|
||||
* committed generation the canonical tree reflects — equal by construction,
|
||||
* since the tree is written by the commit itself) plus the rollup invariants
|
||||
* (entity/relationship counts) that verify the tree whole where per-file
|
||||
* checks cannot scale. Verified at open by {@link verifyEntityTreeStamp};
|
||||
* healed by `repairIndex()`, whose unconditional recount rebuilds the
|
||||
* rollups from a canonical walk and re-stamps. Best-effort: a stamp-write
|
||||
* fault warns loudly but never fails the flush that carried real data.
|
||||
*/
|
||||
private async stampEntityTree(): Promise<void> {
|
||||
if (this.isReadOnly) return
|
||||
try {
|
||||
const [nounCount, verbCount] = await Promise.all([
|
||||
this.storage.getNounCount(),
|
||||
this.storage.getVerbCount()
|
||||
])
|
||||
await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
|
||||
family: 'entity-tree',
|
||||
sourceGeneration: this.generationStore.generation(),
|
||||
members: { mode: 'rollup', invariants: { nounCount, verbCount } }
|
||||
})
|
||||
} catch (error) {
|
||||
prodLog.warn(
|
||||
`[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` +
|
||||
`next successful flush): ${(error as Error).message}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Open-time coherence check for the entity tree's family stamp:
|
||||
* compare `sourceGeneration` against the log head and the stamped rollup
|
||||
* invariants against the live counters. Verdicts:
|
||||
* - `coherent` / `absent` (legacy store; first flush stamps) → silent.
|
||||
* - `behind` → benign for the tree (it is written BY the commit; only the
|
||||
* stamp is stale — a crash landed between commit and flush). Refreshed at
|
||||
* the next flush.
|
||||
* - `incoherent` → LOUD: the tree or its counters diverged from what was
|
||||
* stamped — `repairIndex()` recounts from canonical and re-stamps.
|
||||
* Never blocks open; a fault reading the stamp is surfaced as unverifiable,
|
||||
* never conflated with absence.
|
||||
*/
|
||||
private async verifyEntityTreeStamp(): Promise<void> {
|
||||
let stamp: FamilyStamp | null
|
||||
try {
|
||||
stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH)
|
||||
} catch (error) {
|
||||
prodLog.warn(
|
||||
`[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` +
|
||||
`${(error as Error).message}`
|
||||
)
|
||||
return
|
||||
}
|
||||
const [nounCount, verbCount] = await Promise.all([
|
||||
this.storage.getNounCount(),
|
||||
this.storage.getVerbCount()
|
||||
])
|
||||
const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), {
|
||||
nounCount,
|
||||
verbCount
|
||||
})
|
||||
if (verdict.state === 'incoherent') {
|
||||
prodLog.warn(
|
||||
`[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` +
|
||||
`The canonical tree or its counters diverged from the stamped state — run ` +
|
||||
`brain.repairIndex() to recount from canonical and re-stamp.`
|
||||
)
|
||||
} else if (verdict.state === 'behind') {
|
||||
prodLog.debug(
|
||||
`[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` +
|
||||
`${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the writer process serving this data directory to flush its in-memory
|
||||
* indexes to disk, so a read-only inspector can observe fresh state.
|
||||
|
|
@ -15373,6 +15469,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await pruner.rebuildTypeCounts?.()
|
||||
await pruner.rebuildSubtypeCounts?.()
|
||||
|
||||
// The recount changed the rollup truth — re-stamp the entity tree so the
|
||||
// stamp's invariants match the healed counters (repair leaves a coherent
|
||||
// stamp, not a stale one that warns on the next open).
|
||||
await this.stampEntityTree()
|
||||
|
||||
// VFS containment reconciliation: heal "cosmetic ghost" edges left by
|
||||
// pre-fix renames (an entity Contains-linked from BOTH its old and new
|
||||
// directory — readdir listed it in two places) and duplicate edges from
|
||||
|
|
@ -15775,6 +15876,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
})()
|
||||
])
|
||||
|
||||
// Stamp the entity tree at the close boundary (counters + counter now
|
||||
// durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT
|
||||
// instead of benign-behind. Best-effort, never blocks the close.
|
||||
if (this.generationStore && !this.isReadOnly) {
|
||||
await this.stampEntityTree()
|
||||
}
|
||||
|
||||
// Phase 2: Close components to release resources (timers, file handles)
|
||||
// Data is already safe on disk from Phase 1
|
||||
await Promise.all([
|
||||
|
|
|
|||
147
src/db/familyStamp.ts
Normal file
147
src/db/familyStamp.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
/**
|
||||
* @module db/familyStamp
|
||||
* @description The generalized FAMILY STAMP — one JSON shape that declares,
|
||||
* for any derived projection, WHICH source state it reflects and HOW to verify
|
||||
* it is whole. The entity tree (canonical current-state files) carries the
|
||||
* first brainy-side stamp; native index families carry the same shape. One
|
||||
* verifier reads both member modes:
|
||||
*
|
||||
* - `enumerated` — bounded families: exact byte size per member file,
|
||||
* verified at open.
|
||||
* - `rollup` — unbounded families (the entity tree: millions of files):
|
||||
* the verified surface is a small set of rollup invariants (entity/
|
||||
* relationship counts) plus `sourceGeneration`.
|
||||
*
|
||||
* `sourceGeneration` is the generation of the source-of-truth log this
|
||||
* projection reflects — open-time coherence becomes a COMPARISON (stamp vs
|
||||
* log head), not a walk:
|
||||
*
|
||||
* - equal + invariants hold → coherent, serve.
|
||||
* - behind → the projection missed the tail (crash between commit and stamp);
|
||||
* for the Stage-1 tree this is benign by construction (the tree is written
|
||||
* BY the commit), so the stamp refreshes; a DERIVED projection would replay
|
||||
* the gap instead.
|
||||
* - invariants FAIL at equal generation → genuine incoherence: loud, and the
|
||||
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
|
||||
* canonical walk) heals it.
|
||||
*
|
||||
* Stamps are JSON on purpose — every incident gets debugged by reading a
|
||||
* stamp in a terminal.
|
||||
*/
|
||||
|
||||
/** Storage-root-relative directory holding family stamps. */
|
||||
export const FAMILY_STAMPS_PREFIX = '_system/family-stamps'
|
||||
|
||||
/** The entity tree's stamp path. */
|
||||
export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json`
|
||||
|
||||
/** One enumerated member: a file and its exact expected byte size. */
|
||||
export interface EnumeratedMember {
|
||||
path: string
|
||||
bytes: number
|
||||
}
|
||||
|
||||
/** The stamp's verified surface, in one of the two member modes. */
|
||||
export type StampMembers =
|
||||
| { mode: 'enumerated'; files: EnumeratedMember[] }
|
||||
| { mode: 'rollup'; invariants: Record<string, number> }
|
||||
|
||||
/** The generalized family stamp (one shape, one verifier, both engines). */
|
||||
export interface FamilyStamp {
|
||||
/** Which projection this stamps (e.g. `'entity-tree'`). */
|
||||
family: string
|
||||
/** Monotonic per-family stamp generation — bumps on every committed stamp. */
|
||||
generation: number
|
||||
/** ISO timestamp of the stamp write. */
|
||||
committedAt: string
|
||||
/** The source-of-truth generation this projection reflects. */
|
||||
sourceGeneration: number
|
||||
/** The verified surface. */
|
||||
members: StampMembers
|
||||
}
|
||||
|
||||
/** The verdict of an open-time stamp verification. */
|
||||
export type StampVerdict =
|
||||
| { state: 'coherent' }
|
||||
| { state: 'absent' } // legacy store — first stamp writes at the next flush
|
||||
| { state: 'behind'; stampSource: number; head: number }
|
||||
| { state: 'incoherent'; failures: string[] }
|
||||
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
|
||||
|
||||
/** The narrow storage surface stamps ride (JSON objects + fsync). */
|
||||
export interface StampStorage {
|
||||
readRawObject(path: string): Promise<any | null>
|
||||
writeRawObject(path: string, data: any): Promise<void>
|
||||
syncRawObjects(paths: string[]): Promise<void>
|
||||
}
|
||||
|
||||
/** Read a family's stamp; `null` when none was ever written. */
|
||||
export async function readFamilyStamp(
|
||||
storage: StampStorage,
|
||||
path: string
|
||||
): Promise<FamilyStamp | null> {
|
||||
const stored = (await storage.readRawObject(path)) as FamilyStamp | null
|
||||
if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null
|
||||
return stored
|
||||
}
|
||||
|
||||
/** Write a family's stamp durably (atomic object write + fsync). */
|
||||
export async function writeFamilyStamp(
|
||||
storage: StampStorage,
|
||||
path: string,
|
||||
stamp: Omit<FamilyStamp, 'generation' | 'committedAt'> & { generation?: number }
|
||||
): Promise<void> {
|
||||
const prior = await readFamilyStamp(storage, path)
|
||||
const full: FamilyStamp = {
|
||||
...stamp,
|
||||
generation: (prior?.generation ?? 0) + 1,
|
||||
committedAt: new Date().toISOString()
|
||||
}
|
||||
await storage.writeRawObject(path, full)
|
||||
await storage.syncRawObjects([path])
|
||||
}
|
||||
|
||||
/**
|
||||
* The ONE verifier, both member modes. `actual` supplies the observed rollup
|
||||
* values (rollup mode) or file sizes (enumerated mode, keyed by path);
|
||||
* `head` is the source-of-truth generation now.
|
||||
*/
|
||||
export function verifyFamilyStamp(
|
||||
stamp: FamilyStamp | null,
|
||||
head: number,
|
||||
actual: Record<string, number>
|
||||
): StampVerdict {
|
||||
if (stamp === null) return { state: 'absent' }
|
||||
if (stamp.sourceGeneration > head) {
|
||||
// A stamp AHEAD of the log claims state that never committed — the
|
||||
// projection was stamped against truth that a crash rolled back.
|
||||
return {
|
||||
state: 'incoherent',
|
||||
failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
|
||||
}
|
||||
}
|
||||
if (stamp.sourceGeneration < head) {
|
||||
return { state: 'behind', stampSource: stamp.sourceGeneration, head }
|
||||
}
|
||||
const failures: string[] = []
|
||||
if (stamp.members.mode === 'rollup') {
|
||||
for (const [name, expected] of Object.entries(stamp.members.invariants)) {
|
||||
const observed = actual[name]
|
||||
if (observed === undefined) {
|
||||
failures.push(`rollup invariant '${name}' has no observed value`)
|
||||
} else if (observed !== expected) {
|
||||
failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const member of stamp.members.files) {
|
||||
const observed = actual[member.path]
|
||||
if (observed === undefined) {
|
||||
failures.push(`member '${member.path}' is missing`)
|
||||
} else if (observed !== member.bytes) {
|
||||
failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' }
|
||||
}
|
||||
|
|
@ -204,6 +204,11 @@ export type {
|
|||
FactScanBatch,
|
||||
FactScanHandle
|
||||
} from './db/factLog.js'
|
||||
// The generalized family stamp — which source generation a projection
|
||||
// reflects + the surface that verifies it whole; one verifier, both member
|
||||
// modes (enumerated byte-exact / rollup invariants).
|
||||
export { readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH } from './db/familyStamp.js'
|
||||
export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.js'
|
||||
// Optional provider capability for generation-aware native indexes
|
||||
export { isVersionedIndexProvider } from './plugin.js'
|
||||
export type { VersionedIndexProvider } from './plugin.js'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue