open-brainy/tests/integration/log-authority.test.ts

/**
 * @module tests/integration/log-authority
 * @description The guarded log-authority core, end-to-end: the per-brain
 * authority switch (default 'tree', stored artifact, checked at open only),
 * the verification oracle (replay the fact log, diff latest per-id state
 * against the canonical tree, NAME every divergence by class), the guarded
 * flip (refuses on red with the cure in the message; lands on green and
 * engages durable-at-ack immediately), and the switch surviving reopen.
 *
 * KNOWN GAPS PINNED WITH `.fails` (real findings, not test bugs — see the
 * comments on each): a fresh brain is NOT log-complete by construction
 * today, because the VFS root is written at init as a baseline
 * (generation-less) write that never gets a fact, so the oracle reports it
 * as a `pre-log-record` and no fresh brain can flip without a manual
 * baseline backfill. The tests that need a green oracle perform that
 * backfill explicitly (an identity update of the root as the FINAL write —
 * final, because derived-index maintenance rewrites canonical noun records
 * outside generations, so an earlier fact's after-image goes stale; see the
 * module tail comment on `backfillBaseline`).
 */
import { describe, it, expect, afterEach } from 'vitest'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Brainy } from '../../src/index.js'
import type { OracleReport } from '../../src/db/logAuthority.js'

/** The VFS root — created at init by a baseline (generation-less) write. */
const VFS_ROOT = '00000000-0000-0000-0000-000000000000'
const AUTHORITY_ARTIFACT = '_system/log-authority.json'

/** White-box view of the internals this suite instruments (read-only spies
 *  plus the sanctioned direct-storage writes for aging/drifting a brain). */
type BrainInternals = {
  generationStore: {
    getFactLog(): { ensureSynced(): Promise<void> } | null
    logDurability: 'deferred' | 'at-ack'
  }
  storage: {
    readRawObject(path: string): Promise<unknown | null>
    saveNoun(n: unknown): Promise<void>
    saveNounMetadata(id: string, m: Record<string, unknown>): Promise<void>
    getNounMetadata(id: string): Promise<Record<string, unknown> | null>
    writeNounRaw(id: string, r: { metadata: null; vector: null }): Promise<void>
  }
}

const internals = (brain: Brainy): BrainInternals =>
  brain as unknown as BrainInternals

/** Count calls to the fact log's ensureSynced without changing behavior. */
function spyEnsureSynced(brain: Brainy): { calls: () => number } {
  const factLog = internals(brain).generationStore.getFactLog()
  expect(factLog, 'filesystem storage hosts a fact log').not.toBeNull()
  let calls = 0
  const original = factLog!.ensureSynced.bind(factLog)
  factLog!.ensureSynced = async () => {
    calls++
    return original()
  }
  return { calls: () => calls }
}

/**
 * The minimal baseline backfill: an identity update of the VFS root, so the
 * one canonical record the log never saw (the init-time baseline write) gets
 * a fact carrying its current state. MUST be the final write of the setup —
 * derived-index maintenance (HNSW/enumeration denormalization) rewrites the
 * root's canonical noun record outside any generation, so a root fact taken
 * before later writes digests stale and reports `state-differs`.
 */
async function backfillBaseline(brain: Brainy): Promise<void> {
  const root = await brain.get(VFS_ROOT)
  expect(root, 'the VFS root exists on a fresh brain').toBeTruthy()
  await brain.update({ id: VFS_ROOT, metadata: root!.metadata })
}

/** Seed a brain with the standard write mix: 2 adds, an update, a remove. */
async function seedWrites(brain: Brainy): Promise<{ kept: string; removed: string }> {
  const kept = await brain.add({ data: 'alpha document', type: 'document', metadata: { n: 1 } })
  const removed = await brain.add({ data: 'beta document', type: 'document', metadata: { n: 2 } })
  await brain.update({ id: kept, metadata: { n: 10 } })
  await brain.remove(removed)
  return { kept, removed }
}

describe('log authority — the switch, the oracle, the guarded flip', () => {
  const dirs: string[] = []
  const brains: Brainy[] = []

  const openBrain = async (dir?: string): Promise<{ brain: Brainy; dir: string }> => {
    const d = dir ?? mkdtempSync(join(tmpdir(), 'brainy-log-authority-'))
    if (!dir) dirs.push(d)
    const brain = new Brainy({
      storage: { type: 'filesystem', path: d },
      requireSubtype: false,
      silent: true,
      dimensions: 384
    })
    brains.push(brain)
    await brain.init()
    return { brain, dir: d }
  }

  afterEach(async () => {
    for (const b of brains.splice(0)) {
      await (b as unknown as { close?: () => Promise<void> }).close?.().catch(() => {})
    }
    for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
  })

  it('DEFAULT IS TREE: a fresh brain reports tree authority, stores no artifact, and plain acks never await a log fsync', async () => {
    const { brain } = await openBrain()

    expect(brain.logAuthority().authority).toBe('tree')
    expect(brain.logAuthority().flippedAt).toBeUndefined()

    const artifact = await internals(brain)
      .storage.readRawObject(AUTHORITY_ARTIFACT)
      .catch(() => null)
    expect(artifact, 'no switch artifact exists before any flip').toBeNull()

    // The MODE assertion (not a timing one): in tree authority a single-op
    // ack must never call the log's covering-fsync path.
    const spy = spyEnsureSynced(brain)
    await brain.add({ data: 'tree mode write', type: 'document', metadata: { n: 1 } })
    expect(spy.calls(), 'tree mode: add() does not call ensureSynced').toBe(0)
    expect(internals(brain).generationStore.logDurability).toBe('deferred')
  })

  // KNOWN GAP (marked .fails — remove the marker when fixed in src): the
  // intended contract is that a fresh brain is log-complete by construction,
  // because every write dual-writes a fact. Today the VFS root
  // (00000000-0000-0000-0000-000000000000) is created at init by a baseline
  // write with NO generation and NO fact, yet it is enumerated by the
  // canonical walk — so the oracle on a fresh brain is red with exactly one
  // `pre-log-record` mismatch on the root, and adoptLogAuthority() refuses
  // on every fresh brain. Verified empirically on this branch.
  it.fails('ORACLE INTENT: a fresh brain is log-complete by construction — verdict green with zero mismatches', async () => {
    const { brain } = await openBrain()
    await seedWrites(brain)
    await brain.flush()

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('green')
    expect(report.mismatches).toEqual([])
  })

  it('a fresh, un-backfilled brain diverges ONLY on the init-time baseline record — every user write is exactly reproduced', async () => {
    const { brain } = await openBrain()
    await seedWrites(brain)
    await brain.flush()

    const report = await brain.verifyLogAuthority()
    // Tolerant pin (stays true after the baseline gap is fixed in src):
    // whatever the verdict, no USER record may ever diverge — the only
    // admissible mismatch is the init-time baseline root, as pre-log-record.
    expect(
      report.mismatches.every(
        (m) => m.id === VFS_ROOT && m.reason === 'pre-log-record' && m.kind === 'noun'
      ),
      'the only divergence on a fresh brain is the baseline root record'
    ).toBe(true)
    expect(report.matched).toBe(report.nounsChecked - report.mismatches.length)
    expect(report.mismatchListTruncated).toBe(false)
  })

  it('THE ORACLE GOES GREEN on a log-complete brain: adds + update + remove, every canonical row exactly reproduced', async () => {
    const { brain } = await openBrain()
    await seedWrites(brain)
    await backfillBaseline(brain) // final write — see the helper's contract
    await brain.flush()

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('green')
    expect(report.mismatches).toEqual([])
    expect(report.mismatchListTruncated).toBe(false)
    // Live count: the kept document + the VFS root (the removed one is a
    // tombstone in the log and absent from canonical — checked, not counted).
    expect(report.nounsChecked).toBe(2)
    expect(report.matched).toBe(2)
    // 5 committed generations: add, add, update, remove, root backfill.
    expect(report.generationsScanned).toBe(5)
  })

  it('THE ORACLE NAMES pre-log records: a canonical row no fact ever recorded reports pre-log-record, by id', async () => {
    const { brain } = await openBrain()
    await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()
    expect((await brain.verifyLogAuthority()).verdict, 'sanity: green before aging').toBe('green')

    // Simulate an aged brain: write one canonical record DIRECTLY at the
    // storage layer (the write path never sees it, so no fact exists) —
    // the pre-log shape: flat metadata, no _fmt stamp, 384-dim vector.
    const legacyId = '00000000-0000-4000-8000-00000000a6ed'
    const storage = internals(brain).storage
    await storage.saveNoun({
      id: legacyId,
      vector: new Array(384).fill(0.01),
      connections: new Map(),
      level: 0
    })
    await storage.saveNounMetadata(legacyId, {
      noun: 'document',
      confidence: 0.75,
      createdAt: 1700000000000,
      updatedAt: 1700000000000,
      _rev: 1,
      legacyField: 'legacy-value'
    })

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('red')
    expect(report.mismatches).toHaveLength(1)
    expect(report.mismatches[0]).toEqual({
      id: legacyId,
      kind: 'noun',
      reason: 'pre-log-record'
    })
  })

  it('THE FLIP REFUSES ON A LOG-AHEAD DIVERGENCE: the witness denies what the log claims — nothing written, nothing changed', async () => {
    // Contract update (adoptLogAuthority's baseline backfill): curable
    // divergences — pre-log records and witness drift — are re-committed
    // and the flip proceeds; ONLY log-AHEAD divergences (the log claims
    // state canonical denies) refuse, because no backfill can make the log
    // un-claim a live row. This test stages exactly that incurable shape.
    const { brain } = await openBrain()
    const { kept } = await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()

    // The log says `kept` is live; its canonical record vanishes behind the
    // write path's back (log-live-canonical-absent — the witness wins).
    const storage = internals(brain).storage
    await storage.writeNounRaw(kept, { metadata: null, vector: null })

    let error: Error | null = null
    try {
      await brain.adoptLogAuthority()
    } catch (err) {
      error = err as Error
    }
    expect(error, 'the flip rejects on a log-ahead divergence').not.toBeNull()
    expect(error!.message).toMatch(/witness denies/)
    expect(error!.message).toMatch(/log-live-canonical-absent/)

    // Nothing changed: authority still tree, no artifact, deferred durability.
    expect(brain.logAuthority().authority).toBe('tree')
    const artifact = await storage.readRawObject(AUTHORITY_ARTIFACT).catch(() => null)
    expect(artifact, 'a refused flip writes no artifact').toBeNull()
    expect(internals(brain).generationStore.logDurability).toBe('deferred')
  })

  it('THE FLIP LANDS ON GREEN: the report is the receipt, the artifact is on disk, and durable-at-ack engages immediately', async () => {
    const { brain } = await openBrain()
    await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()

    const report: OracleReport = await brain.adoptLogAuthority()
    expect(report.verdict).toBe('green')

    const authority = brain.logAuthority()
    expect(authority.authority).toBe('log')
    expect(typeof authority.flippedAt).toBe('number')
    expect(authority.oracle).toBeDefined()
    expect(authority.oracle!.nounsChecked).toBe(report.nounsChecked)
    expect(authority.oracle!.generationsScanned).toBe(report.generationsScanned)

    const artifact = (await internals(brain)
      .storage.readRawObject(AUTHORITY_ARTIFACT)
      .catch(() => null)) as { authority?: string } | null
    expect(artifact, 'the switch artifact exists on disk').not.toBeNull()
    expect(artifact!.authority).toBe('log')

    // Durable-at-ack engaged in THIS session: the next single-op ack awaits
    // a covering log fsync.
    expect(internals(brain).generationStore.logDurability).toBe('at-ack')
    const spy = spyEnsureSynced(brain)
    await brain.add({ data: 'post-flip write', type: 'document', metadata: { n: 3 } })
    expect(spy.calls(), 'log mode: add() awaits the covering fsync').toBeGreaterThanOrEqual(1)
  })

  it('THE SWITCH SURVIVES REOPEN: authority restored at open with no re-verification, durable-at-ack active in the new session', async () => {
    const { brain, dir } = await openBrain()
    await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()
    await brain.adoptLogAuthority()
    const flipReceipt = brain.logAuthority()
    await (brain as unknown as { close: () => Promise<void> }).close()

    const { brain: reopened } = await openBrain(dir)
    const restored = reopened.logAuthority()
    expect(restored.authority).toBe('log')
    // No re-verification happened at open: the restored record IS the stored
    // flip receipt, oracle summary and timestamp intact.
    expect(restored.flippedAt).toBe(flipReceipt.flippedAt)
    expect(restored.oracle).toEqual(flipReceipt.oracle)

    // Mode restored at open: an ack in the new session awaits the log fsync.
    expect(internals(reopened).generationStore.logDurability).toBe('at-ack')
    const spy = spyEnsureSynced(reopened)
    await reopened.add({ data: 'new session write', type: 'document', metadata: { n: 4 } })
    expect(spy.calls(), 'reopened log mode: add() awaits the covering fsync').toBeGreaterThanOrEqual(1)
  })

  it('STATE-DIFFERS: canonical drift the write path never saw is named, by id', async () => {
    const { brain } = await openBrain()
    const { kept } = await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()
    expect((await brain.verifyLogAuthority()).verdict, 'sanity: green before drift').toBe('green')

    // Drift one canonical metadata record DIRECTLY at the storage layer —
    // the log never hears about it. This is the witness-drift case the
    // oracle exists to catch.
    const storage = internals(brain).storage
    const current = await storage.getNounMetadata(kept)
    expect(current, 'the seeded record has stored metadata').toBeTruthy()
    await storage.saveNounMetadata(kept, { ...current!, driftedByTest: true })

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('red')
    expect(report.mismatches).toHaveLength(1)
    expect(report.mismatches[0]).toEqual({
      id: kept,
      kind: 'noun',
      reason: 'state-differs'
    })
  })
})