feat: entity-tree family stamp — sourceGeneration + rollup coherence at open

The canonical entity tree now carries a FAMILY STAMP
(_system/family-stamps/entity-tree.json, JSON — forensics stay
terminal-readable): which committed generation the tree reflects
(sourceGeneration) plus the rollup invariants (entity/relationship
counts) that verify a millions-of-files projection whole where per-file
checks cannot scale. Written at flush/close boundaries; open-time
coherence is a COMPARISON, not a walk:

- coherent / absent (legacy store) → silent
- behind → benign for the tree (it is written BY the commit; only the
  stamp is stale after a crash between commit and flush) — refreshes at
  the next flush
- incoherent (counts diverged at equal generation, or a stamp AHEAD of
  the log head) → loud, names the failing invariant; repairIndex()'s
  unconditional recount heals and RE-STAMPS so repair leaves a coherent
  stamp behind
- a fault reading the stamp is UNVERIFIABLE — never conflated with
  absence

One verifier (verifyFamilyStamp, exported) reads both member modes:
enumerated (exact byte size per member, bounded families) and rollup
(invariants, unbounded families). New exports: readFamilyStamp,
verifyFamilyStamp, ENTITY_TREE_STAMP_PATH, FamilyStamp, StampMembers,
StampVerdict.

src/brainy.ts

@@ -168,6 +168,13 @@ import {
 } from './db/portableGraph.js'
 import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js'
 import type { FactScanHandle } from './db/factLog.js'
+import {
+  ENTITY_TREE_STAMP_PATH,
+  readFamilyStamp,
+  verifyFamilyStamp,
+  writeFamilyStamp,
+  type FamilyStamp
+} from './db/familyStamp.js'
 import {
   ChangeFeed,
   type BrainyChangeEvent,
@@ -992,6 +999,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         })
       }
 
+      // Entity-tree stamp coherence: compare the stamped sourceGeneration +
+      // rollup invariants against the log head + live counters. Loud on
+      // genuine incoherence (repairIndex heals), silent on absent/coherent,
+      // benign-behind refreshes at the next flush. Never blocks open.
+      await this.verifyEntityTreeStamp()
+
       // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
       // marker (`_system/brain-format.json`) into an in-memory field NOW —
       // after the store-open phase, but BEFORE any derived index or native
@@ -10354,11 +10367,94 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     //    Db pins and an explicit autoCompact: false.
     await this.autoCompactHistory()
 
+    // 7. Stamp the entity tree: which source generation the canonical tree
+    //    reflects + the rollup invariants that verify it whole (the counters
+    //    persisted in step 1). Written at flush boundaries — the tree tracks
+    //    every commit by construction, so the stamp is a durable checkpoint,
+    //    not a per-commit cost. Open compares stamp vs log head + rollups.
+    await this.stampEntityTree()
+
     const elapsed = Date.now() - startTime
 
     console.log(`All indexes flushed to disk in ${elapsed}ms`)
   }
 
+  /**
+   * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the
+   * committed generation the canonical tree reflects — equal by construction,
+   * since the tree is written by the commit itself) plus the rollup invariants
+   * (entity/relationship counts) that verify the tree whole where per-file
+   * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp};
+   * healed by `repairIndex()`, whose unconditional recount rebuilds the
+   * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write
+   * fault warns loudly but never fails the flush that carried real data.
+   */
+  private async stampEntityTree(): Promise<void> {
+    if (this.isReadOnly) return
+    try {
+      const [nounCount, verbCount] = await Promise.all([
+        this.storage.getNounCount(),
+        this.storage.getVerbCount()
+      ])
+      await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
+        family: 'entity-tree',
+        sourceGeneration: this.generationStore.generation(),
+        members: { mode: 'rollup', invariants: { nounCount, verbCount } }
+      })
+    } catch (error) {
+      prodLog.warn(
+        `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` +
+          `next successful flush): ${(error as Error).message}`
+      )
+    }
+  }
+
+  /**
+   * @description Open-time coherence check for the entity tree's family stamp:
+   * compare `sourceGeneration` against the log head and the stamped rollup
+   * invariants against the live counters. Verdicts:
+   * - `coherent` / `absent` (legacy store; first flush stamps) → silent.
+   * - `behind` → benign for the tree (it is written BY the commit; only the
+   *   stamp is stale — a crash landed between commit and flush). Refreshed at
+   *   the next flush.
+   * - `incoherent` → LOUD: the tree or its counters diverged from what was
+   *   stamped — `repairIndex()` recounts from canonical and re-stamps.
+   * Never blocks open; a fault reading the stamp is surfaced as unverifiable,
+   * never conflated with absence.
+   */
+  private async verifyEntityTreeStamp(): Promise<void> {
+    let stamp: FamilyStamp | null
+    try {
+      stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH)
+    } catch (error) {
+      prodLog.warn(
+        `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` +
+          `${(error as Error).message}`
+      )
+      return
+    }
+    const [nounCount, verbCount] = await Promise.all([
+      this.storage.getNounCount(),
+      this.storage.getVerbCount()
+    ])
+    const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), {
+      nounCount,
+      verbCount
+    })
+    if (verdict.state === 'incoherent') {
+      prodLog.warn(
+        `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` +
+          `The canonical tree or its counters diverged from the stamped state — run ` +
+          `brain.repairIndex() to recount from canonical and re-stamp.`
+      )
+    } else if (verdict.state === 'behind') {
+      prodLog.debug(
+        `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` +
+          `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.`
+      )
+    }
+  }
+
   /**
    * Ask the writer process serving this data directory to flush its in-memory
    * indexes to disk, so a read-only inspector can observe fresh state.
@@ -15373,6 +15469,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     await pruner.rebuildTypeCounts?.()
     await pruner.rebuildSubtypeCounts?.()
 
+    // The recount changed the rollup truth — re-stamp the entity tree so the
+    // stamp's invariants match the healed counters (repair leaves a coherent
+    // stamp, not a stale one that warns on the next open).
+    await this.stampEntityTree()
+
     // VFS containment reconciliation: heal "cosmetic ghost" edges left by
     // pre-fix renames (an entity Contains-linked from BOTH its old and new
     // directory — readdir listed it in two places) and duplicate edges from
@@ -15775,6 +15876,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       })()
     ])
 
+    // Stamp the entity tree at the close boundary (counters + counter now
+    // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT
+    // instead of benign-behind. Best-effort, never blocks the close.
+    if (this.generationStore && !this.isReadOnly) {
+      await this.stampEntityTree()
+    }
+
     // Phase 2: Close components to release resources (timers, file handles)
     // Data is already safe on disk from Phase 1
     await Promise.all([

src/db/familyStamp.ts

@@ -0,0 +1,147 @@
+/**
+ * @module db/familyStamp
+ * @description The generalized FAMILY STAMP — one JSON shape that declares,
+ * for any derived projection, WHICH source state it reflects and HOW to verify
+ * it is whole. The entity tree (canonical current-state files) carries the
+ * first brainy-side stamp; native index families carry the same shape. One
+ * verifier reads both member modes:
+ *
+ * - `enumerated` — bounded families: exact byte size per member file,
+ *   verified at open.
+ * - `rollup`     — unbounded families (the entity tree: millions of files):
+ *   the verified surface is a small set of rollup invariants (entity/
+ *   relationship counts) plus `sourceGeneration`.
+ *
+ * `sourceGeneration` is the generation of the source-of-truth log this
+ * projection reflects — open-time coherence becomes a COMPARISON (stamp vs
+ * log head), not a walk:
+ *
+ * - equal + invariants hold → coherent, serve.
+ * - behind → the projection missed the tail (crash between commit and stamp);
+ *   for the Stage-1 tree this is benign by construction (the tree is written
+ *   BY the commit), so the stamp refreshes; a DERIVED projection would replay
+ *   the gap instead.
+ * - invariants FAIL at equal generation → genuine incoherence: loud, and the
+ *   repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
+ *   canonical walk) heals it.
+ *
+ * Stamps are JSON on purpose — every incident gets debugged by reading a
+ * stamp in a terminal.
+ */
+
+/** Storage-root-relative directory holding family stamps. */
+export const FAMILY_STAMPS_PREFIX = '_system/family-stamps'
+
+/** The entity tree's stamp path. */
+export const ENTITY_TREE_STAMP_PATH = `${FAMILY_STAMPS_PREFIX}/entity-tree.json`
+
+/** One enumerated member: a file and its exact expected byte size. */
+export interface EnumeratedMember {
+  path: string
+  bytes: number
+}
+
+/** The stamp's verified surface, in one of the two member modes. */
+export type StampMembers =
+  | { mode: 'enumerated'; files: EnumeratedMember[] }
+  | { mode: 'rollup'; invariants: Record<string, number> }
+
+/** The generalized family stamp (one shape, one verifier, both engines). */
+export interface FamilyStamp {
+  /** Which projection this stamps (e.g. `'entity-tree'`). */
+  family: string
+  /** Monotonic per-family stamp generation — bumps on every committed stamp. */
+  generation: number
+  /** ISO timestamp of the stamp write. */
+  committedAt: string
+  /** The source-of-truth generation this projection reflects. */
+  sourceGeneration: number
+  /** The verified surface. */
+  members: StampMembers
+}
+
+/** The verdict of an open-time stamp verification. */
+export type StampVerdict =
+  | { state: 'coherent' }
+  | { state: 'absent' } // legacy store — first stamp writes at the next flush
+  | { state: 'behind'; stampSource: number; head: number }
+  | { state: 'incoherent'; failures: string[] }
+  | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
+
+/** The narrow storage surface stamps ride (JSON objects + fsync). */
+export interface StampStorage {
+  readRawObject(path: string): Promise<any | null>
+  writeRawObject(path: string, data: any): Promise<void>
+  syncRawObjects(paths: string[]): Promise<void>
+}
+
+/** Read a family's stamp; `null` when none was ever written. */
+export async function readFamilyStamp(
+  storage: StampStorage,
+  path: string
+): Promise<FamilyStamp | null> {
+  const stored = (await storage.readRawObject(path)) as FamilyStamp | null
+  if (!stored || typeof stored !== 'object' || typeof stored.family !== 'string') return null
+  return stored
+}
+
+/** Write a family's stamp durably (atomic object write + fsync). */
+export async function writeFamilyStamp(
+  storage: StampStorage,
+  path: string,
+  stamp: Omit<FamilyStamp, 'generation' | 'committedAt'> & { generation?: number }
+): Promise<void> {
+  const prior = await readFamilyStamp(storage, path)
+  const full: FamilyStamp = {
+    ...stamp,
+    generation: (prior?.generation ?? 0) + 1,
+    committedAt: new Date().toISOString()
+  }
+  await storage.writeRawObject(path, full)
+  await storage.syncRawObjects([path])
+}
+
+/**
+ * The ONE verifier, both member modes. `actual` supplies the observed rollup
+ * values (rollup mode) or file sizes (enumerated mode, keyed by path);
+ * `head` is the source-of-truth generation now.
+ */
+export function verifyFamilyStamp(
+  stamp: FamilyStamp | null,
+  head: number,
+  actual: Record<string, number>
+): StampVerdict {
+  if (stamp === null) return { state: 'absent' }
+  if (stamp.sourceGeneration > head) {
+    // A stamp AHEAD of the log claims state that never committed — the
+    // projection was stamped against truth that a crash rolled back.
+    return {
+      state: 'incoherent',
+      failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
+    }
+  }
+  if (stamp.sourceGeneration < head) {
+    return { state: 'behind', stampSource: stamp.sourceGeneration, head }
+  }
+  const failures: string[] = []
+  if (stamp.members.mode === 'rollup') {
+    for (const [name, expected] of Object.entries(stamp.members.invariants)) {
+      const observed = actual[name]
+      if (observed === undefined) {
+        failures.push(`rollup invariant '${name}' has no observed value`)
+      } else if (observed !== expected) {
+        failures.push(`rollup invariant '${name}': stamped ${expected}, observed ${observed}`)
+      }
+    }
+  } else {
+    for (const member of stamp.members.files) {
+      const observed = actual[member.path]
+      if (observed === undefined) {
+        failures.push(`member '${member.path}' is missing`)
+      } else if (observed !== member.bytes) {
+        failures.push(`member '${member.path}': stamped ${member.bytes} bytes, observed ${observed}`)
+      }
+    }
+  }
+  return failures.length > 0 ? { state: 'incoherent', failures } : { state: 'coherent' }
+}

src/index.ts

@@ -204,6 +204,11 @@ export type {
   FactScanBatch,
   FactScanHandle
 } from './db/factLog.js'
+// The generalized family stamp — which source generation a projection
+// reflects + the surface that verifies it whole; one verifier, both member
+// modes (enumerated byte-exact / rollup invariants).
+export { readFamilyStamp, verifyFamilyStamp, ENTITY_TREE_STAMP_PATH } from './db/familyStamp.js'
+export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.js'
 // Optional provider capability for generation-aware native indexes
 export { isVersionedIndexProvider } from './plugin.js'
 export type { VersionedIndexProvider } from './plugin.js'

tests/integration/entity-tree-stamp.test.ts

@@ -0,0 +1,144 @@
+/**
+ * @module tests/integration/entity-tree-stamp
+ * @description The entity tree's FAMILY STAMP: written at flush/close with
+ * `sourceGeneration` (the committed generation the canonical tree reflects)
+ * plus rollup invariants (entity/relationship counts); verified at open by
+ * comparison — coherent on a clean close, LOUD on genuine incoherence
+ * (tampered counters), healed by repairIndex()'s recount + re-stamp. One
+ * verifier reads both member modes.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import {
+  Brainy,
+  readFamilyStamp,
+  verifyFamilyStamp,
+  ENTITY_TREE_STAMP_PATH,
+  type FamilyStamp
+} from '../../src/index.js'
+import { prodLog } from '../../src/utils/logger.js'
+
+describe('entity-tree family stamp', () => {
+  let dir: string
+  let brain: any
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-'))
+    brain = await open()
+  })
+  afterEach(async () => {
+    vi.restoreAllMocks()
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => {
+    for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } })
+    await brain.flush()
+
+    const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
+    expect(stamp).not.toBeNull()
+    expect(stamp.family).toBe('entity-tree')
+    expect(stamp.members.mode).toBe('rollup')
+    const invariants = (stamp.members as any).invariants
+    expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
+    expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
+    expect(stamp.sourceGeneration).toBe(brain.generation())
+    expect(stamp.generation).toBeGreaterThanOrEqual(1)
+  })
+
+  it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => {
+    await brain.add({ data: 'clean', type: 'document', metadata: {} })
+    await brain.close()
+
+    const warn = vi.spyOn(prodLog, 'warn')
+    brain = await open()
+    const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp'))
+    expect(stampWarnings).toEqual([])
+  })
+
+  it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => {
+    for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
+    await brain.close()
+
+    // Simulate counter drift AFTER the stamp was written: inflate the
+    // persisted scalar the way the historical decrement-skip did.
+    brain = await open()
+    ;(brain.storage as any).totalNounCount += 52
+    await (brain.storage as any).persistCounts()
+    await brain.close()
+    // The close boundary re-stamps with the inflated counter — so tamper the
+    // STAMP instead for a deterministic mismatch: stamped counts differ from
+    // the (inflated) live ones at the NEXT open only if the stamp is older.
+    // Rewrite the stamp with the honest counts + current sourceGeneration.
+    const raw = JSON.parse(
+      require('node:zlib')
+        .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
+        .toString('utf-8')
+    ) as FamilyStamp
+    const honest = { ...raw }
+    ;(honest.members as any).invariants.nounCount -= 52
+    fs.writeFileSync(
+      path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`),
+      require('node:zlib').gzipSync(JSON.stringify(honest))
+    )
+
+    const warn = vi.spyOn(prodLog, 'warn')
+    brain = await open()
+    const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
+    expect(incoherent.length).toBeGreaterThanOrEqual(1)
+    expect(String(incoherent[0][0])).toMatch(/nounCount/)
+
+    // The heal: recount from canonical + re-stamp → next open is quiet.
+    await brain.repairIndex()
+    await brain.close()
+    const warn2 = vi.spyOn(prodLog, 'warn')
+    brain = await open()
+    const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
+    expect(stillIncoherent).toEqual([])
+  })
+
+  it('the one verifier handles both member modes', () => {
+    const rollup: FamilyStamp = {
+      family: 'x',
+      generation: 1,
+      committedAt: new Date().toISOString(),
+      sourceGeneration: 5,
+      members: { mode: 'rollup', invariants: { nounCount: 10 } }
+    }
+    expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' })
+    expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent')
+    expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({
+      state: 'behind',
+      stampSource: 5,
+      head: 9
+    })
+    expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head
+    expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
+
+    const enumerated: FamilyStamp = {
+      family: 'y',
+      generation: 1,
+      committedAt: new Date().toISOString(),
+      sourceGeneration: 2,
+      members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] }
+    }
+    expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' })
+    expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent')
+    expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member
+  })
+})