open-brainy/tests/integration/entity-tree-stamp.test.ts

/**
 * @module tests/integration/entity-tree-stamp
 * @description The entity tree's FAMILY STAMP: written at flush/close with
 * `sourceGeneration` (the committed generation the canonical tree reflects)
 * plus rollup invariants (entity/relationship counts); verified at open by
 * comparison — coherent on a clean close, LOUD on genuine incoherence
 * (tampered counters), healed by repairIndex()'s recount + re-stamp. One
 * verifier reads both member modes.
 */
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import {
  Brainy,
  readFamilyStamp,
  verifyFamilyStamp,
  ENTITY_TREE_STAMP_PATH,
  type FamilyStamp
} from '../../src/index.js'
import { prodLog } from '../../src/utils/logger.js'

describe('entity-tree family stamp', () => {
  let dir: string
  let brain: any

  const open = async () => {
    const b: any = new Brainy({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir },
      silent: true,
      dimensions: 384
    })
    await b.init()
    return b
  }

  beforeEach(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-stamp-'))
    brain = await open()
  })
  afterEach(async () => {
    vi.restoreAllMocks()
    await brain.close?.().catch(() => {})
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('flush() writes the stamp: rollup mode, counts + sourceGeneration match live state', async () => {
    for (let i = 0; i < 3; i++) await brain.add({ data: `s${i}`, type: 'document', metadata: { i } })
    await brain.flush()

    const stamp = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
    expect(stamp).not.toBeNull()
    expect(stamp.family).toBe('entity-tree')
    expect(stamp.members.mode).toBe('rollup')
    const invariants = (stamp.members as any).invariants
    expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
    expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
    // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the
    // counter labelled the stamp with a generation a write in flight had merely
    // claimed, so every crash inside a write window produced a spurious verdict
    // at the next open (see the torn-tail pins below).
    expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration())
    expect(stamp.generation).toBeGreaterThanOrEqual(1)
  })

  it('a cleanly-closed store reopens COHERENT (no incoherence warning)', async () => {
    await brain.add({ data: 'clean', type: 'document', metadata: {} })
    await brain.close()

    const warn = vi.spyOn(prodLog, 'warn')
    brain = await open()
    const stampWarnings = warn.mock.calls.filter((c) => String(c[0]).includes('entity-tree stamp'))
    expect(stampWarnings).toEqual([])
  })

  it('tampered counters surface as INCOHERENT at open; repairIndex() heals + re-stamps', async () => {
    for (let i = 0; i < 3; i++) await brain.add({ data: `t${i}`, type: 'document', metadata: { i } })
    await brain.close()

    // Simulate counter drift AFTER the stamp was written: inflate the
    // persisted scalar the way the historical decrement-skip did.
    brain = await open()
    ;(brain.storage as any).totalNounCount += 52
    await (brain.storage as any).persistCounts()
    await brain.close()
    // The close boundary re-stamps with the inflated counter — so tamper the
    // STAMP instead for a deterministic mismatch: stamped counts differ from
    // the (inflated) live ones at the NEXT open only if the stamp is older.
    // Rewrite the stamp with the honest counts + current sourceGeneration.
    const raw = JSON.parse(
      require('node:zlib')
        .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
        .toString('utf-8')
    ) as FamilyStamp
    const honest = { ...raw }
    ;(honest.members as any).invariants.nounCount -= 52
    fs.writeFileSync(
      path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`),
      require('node:zlib').gzipSync(JSON.stringify(honest))
    )

    const warn = vi.spyOn(prodLog, 'warn')
    brain = await open()
    const incoherent = warn.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
    expect(incoherent.length).toBeGreaterThanOrEqual(1)
    expect(String(incoherent[0][0])).toMatch(/nounCount/)

    // The heal: recount from canonical + re-stamp → next open is quiet.
    await brain.repairIndex()
    await brain.close()
    const warn2 = vi.spyOn(prodLog, 'warn')
    brain = await open()
    const stillIncoherent = warn2.mock.calls.filter((c) => String(c[0]).includes('INCOHERENT'))
    expect(stillIncoherent).toEqual([])
  })

  /**
   * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's
   * committed watermark — the durable shape a torn generation-log tail leaves
   * behind (the stamp's fsync outlived the tail's). Fabricated rather than
   * crash-produced so the pin is deterministic; the seeded-SIGKILL lane
   * (`scripts/crash-consistency.mjs` in the engine repo) produces the same
   * shape from a real abrupt termination.
   */
  const fabricateTear = (ahead: number): FamilyStamp => {
    const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)
    const zlib = require('node:zlib')
    const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp
    const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead }
    fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn)))
    return torn
  }

  it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => {
    for (let i = 0; i < 3; i++)
      await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } })
    await brain.close()
    const torn = fabricateTear(5)

    const warn = vi.spyOn(prodLog, 'warn')
    const startedAt = Date.now()
    brain = await open()
    const openMs = Date.now() - startedAt

    const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
    expect(tearLines.length).toBe(1)
    const said = String(tearLines[0][0])
    // Narrated PRECISELY: both generations, the file, and the named cure.
    expect(said).toContain(`source generation ${torn.sourceGeneration}`)
    expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`)
    expect(said).toContain(ENTITY_TREE_STAMP_PATH)
    expect(said).toContain('DEMOTED')
    expect(said).toMatch(/repairIndex\(\)/)
    // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear
    // cannot turn an open into the 8-minute spin this class was reported as.
    expect(openMs).toBeLessThan(30_000)

    // The store SERVES — a tear in a stamp never locks an owner out of the
    // canonical tree the stamp merely describes.
    expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3)

    // The demotion CONVERGED: the stamp now names committed truth, and the
    // next open is quiet. A verdict that re-narrates every open is a wait
    // wearing a different hat.
    const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
    expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration())
    await brain.close()
    const warn2 = vi.spyOn(prodLog, 'warn')
    brain = await open()
    expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([])
  })

  it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => {
    await brain.add({ data: 'ro', type: 'document', metadata: {} })
    await brain.close()
    const torn = fabricateTear(3)

    const warn = vi.spyOn(prodLog, 'warn')
    const reader: any = await Brainy.openReadOnly({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir },
      silent: true,
      dimensions: 384
    })
    const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
    expect(tearLines.length).toBe(1)
    const said = String(tearLines[0][0])
    expect(said).toContain('READ-ONLY')
    expect(said).toContain('UNVERIFIED')
    expect(said).toMatch(/repairIndex\(\)/)
    await reader.close()

    // A reader never rewrites the store: read the bytes back off disk (not
    // through a writer open, which would demote them) — the torn stamp is
    // exactly as it was found.
    const onDisk = JSON.parse(
      require('node:zlib')
        .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
        .toString('utf-8')
    ) as FamilyStamp
    expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration)
    expect(onDisk.generation).toBe(torn.generation)

    brain = await open()
  })

  it('the one verifier handles both member modes', () => {
    const rollup: FamilyStamp = {
      family: 'x',
      generation: 1,
      committedAt: new Date().toISOString(),
      sourceGeneration: 5,
      members: { mode: 'rollup', invariants: { nounCount: 10 } }
    }
    expect(verifyFamilyStamp(rollup, 5, { nounCount: 10 })).toEqual({ state: 'coherent' })
    expect(verifyFamilyStamp(rollup, 5, { nounCount: 11 }).state).toBe('incoherent')
    expect(verifyFamilyStamp(rollup, 9, { nounCount: 10 })).toEqual({
      state: 'behind',
      stampSource: 5,
      head: 9
    })
    // AHEAD is its own class — a torn generation-log tail, never folded in
    // with `incoherent`: the two have opposite cures (recount vs demote).
    expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({
      state: 'torn',
      stampSource: 5,
      head: 3
    })
    expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })

    const enumerated: FamilyStamp = {
      family: 'y',
      generation: 1,
      committedAt: new Date().toISOString(),
      sourceGeneration: 2,
      members: { mode: 'enumerated', files: [{ path: 'a.bin', bytes: 128 }] }
    }
    expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 128 })).toEqual({ state: 'coherent' })
    expect(verifyFamilyStamp(enumerated, 2, { 'a.bin': 64 }).state).toBe('incoherent')
    expect(verifyFamilyStamp(enumerated, 2, {}).state).toBe('incoherent') // missing member

    // Rollup invariants may be STRING fingerprints (e.g. a per-tree SHA-256):
    // strict equality either way; a type mismatch reads as incoherence.
    const fingerprinted: FamilyStamp = {
      family: 'z',
      generation: 1,
      committedAt: new Date().toISOString(),
      sourceGeneration: 3,
      members: { mode: 'rollup', invariants: { treeDigest: 'abc123', rows: 42 } }
    }
    expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: 42 })).toEqual({
      state: 'coherent'
    })
    expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'deadbeef', rows: 42 }).state).toBe(
      'incoherent'
    )
    expect(verifyFamilyStamp(fingerprinted, 3, { treeDigest: 'abc123', rows: '42' }).state).toBe(
      'incoherent' // type mismatch never passes
    )
  })
})