open-brainy/tests/integration/log-authority.test.ts

/**
 * @module tests/integration/log-authority
 * @description The guarded log-authority core, end-to-end: the per-brain
 * authority switch (stored artifact, checked at open only), the
 * verification oracle (replay the fact log, diff latest per-id state
 * against the canonical tree, NAME every divergence by class), the guarded
 * flip (refuses on red with the cure in the message; lands on green and
 * engages durable-at-ack immediately), and the switch surviving reopen.
 *
 * THE 10.0.0 FLEET DEFAULT is ADOPT-AT-OPEN (`logAuthority: 'adopt'`): a
 * fresh brain with no stored artifact runs the oracle at open, backfills
 * curable divergences, and flips to log authority on green — so a
 * default-config brain opens ALREADY log-authoritative and durable-at-ack.
 * The first two pins hold that default and its explicit opt-out
 * (`logAuthority: 'defer'`, the pre-10 tree behavior). Every test below
 * them that exercises the ORACLE or the EXPLICIT flip opens its brain with
 * `'defer'` — otherwise the open-time adoption would have pre-flipped the
 * brain and pre-cured the very divergences under test.
 *
 * KNOWN GAPS PINNED WITH `.fails` (real findings, not test bugs — see the
 * comments on each): a fresh brain is NOT log-complete by construction
 * today, because the VFS root is written at init as a baseline
 * (generation-less) write that never gets a fact, so the oracle reports it
 * as a `pre-log-record`. The open-time adoption (and adoptLogAuthority())
 * CURES this by baseline backfill — a re-commit, not construction — so the
 * by-construction pin stays `.fails` on a deferred brain. Tests that need
 * a green oracle on a deferred brain perform that backfill explicitly (an
 * identity update of the root as the FINAL write — final, because
 * derived-index maintenance rewrites canonical noun records outside
 * generations, so an earlier fact's after-image goes stale; see the module
 * tail comment on `backfillBaseline`).
 */
import { describe, it, expect, afterEach } from 'vitest'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Brainy } from '../../src/index.js'
import type { OracleReport } from '../../src/db/logAuthority.js'

/** The VFS root — created at init by a baseline (generation-less) write. */
const VFS_ROOT = '00000000-0000-0000-0000-000000000000'
const AUTHORITY_ARTIFACT = '_system/log-authority.json'

/** White-box view of the internals this suite instruments (read-only spies
 *  plus the sanctioned direct-storage writes for aging/drifting a brain). */
type BrainInternals = {
  generationStore: {
    getFactLog(): { ensureSynced(): Promise<void> } | null
    logDurability: 'deferred' | 'at-ack'
  }
  storage: {
    readRawObject(path: string): Promise<unknown | null>
    saveNoun(n: unknown): Promise<void>
    saveNounMetadata(id: string, m: Record<string, unknown>): Promise<void>
    getNounMetadata(id: string): Promise<Record<string, unknown> | null>
    writeNounRaw(id: string, r: { metadata: null; vector: null }): Promise<void>
  }
}

const internals = (brain: Brainy): BrainInternals =>
  brain as unknown as BrainInternals

/** Count calls to the fact log's ensureSynced without changing behavior. */
function spyEnsureSynced(brain: Brainy): { calls: () => number } {
  const factLog = internals(brain).generationStore.getFactLog()
  expect(factLog, 'filesystem storage hosts a fact log').not.toBeNull()
  let calls = 0
  const original = factLog!.ensureSynced.bind(factLog)
  factLog!.ensureSynced = async () => {
    calls++
    return original()
  }
  return { calls: () => calls }
}

/**
 * The minimal baseline backfill: an identity update of the VFS root, so the
 * one canonical record the log never saw (the init-time baseline write) gets
 * a fact carrying its current state. MUST be the final write of the setup —
 * derived-index maintenance (HNSW/enumeration denormalization) rewrites the
 * root's canonical noun record outside any generation, so a root fact taken
 * before later writes digests stale and reports `state-differs`.
 */
async function backfillBaseline(brain: Brainy): Promise<void> {
  const root = await brain.get(VFS_ROOT)
  expect(root, 'the VFS root exists on a fresh brain').toBeTruthy()
  await brain.update({ id: VFS_ROOT, metadata: root!.metadata })
}

/** Seed a brain with the standard write mix: 2 adds, an update, a remove. */
async function seedWrites(brain: Brainy): Promise<{ kept: string; removed: string }> {
  const kept = await brain.add({ data: 'alpha document', type: 'document', metadata: { n: 1 } })
  const removed = await brain.add({ data: 'beta document', type: 'document', metadata: { n: 2 } })
  await brain.update({ id: kept, metadata: { n: 10 } })
  await brain.remove(removed)
  return { kept, removed }
}

describe('log authority — the switch, the oracle, the guarded flip', () => {
  const dirs: string[] = []
  const brains: Brainy[] = []

  /**
   * Open a brain over `dir`. Omit `logAuthority` to exercise the FLEET
   * DEFAULT (adopt-at-open); pass `'defer'` for the tests that need a
   * tree-authoritative brain so the oracle/explicit-flip path is actually
   * the thing under test (the default would pre-flip and pre-backfill).
   */
  const openBrain = async (
    dir?: string,
    logAuthority?: 'adopt' | 'defer'
  ): Promise<{ brain: Brainy; dir: string }> => {
    const d = dir ?? mkdtempSync(join(tmpdir(), 'brainy-log-authority-'))
    if (!dir) dirs.push(d)
    const brain = new Brainy({
      storage: { type: 'filesystem', path: d },
      requireSubtype: false,
      silent: true,
      dimensions: 384,
      ...(logAuthority ? { logAuthority } : {})
    })
    brains.push(brain)
    await brain.init()
    return { brain, dir: d }
  }

  afterEach(async () => {
    for (const b of brains.splice(0)) {
      await (b as unknown as { close?: () => Promise<void> }).close?.().catch(() => {})
    }
    for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
  })

  // THE RULED DEFAULT (10.0.0): with no config and no stored artifact, a
  // fresh brain ADOPTS log authority at open — oracle green (the open-time
  // baseline backfill cures the generation-0 VFS root), artifact on disk,
  // durable-at-ack live from the first write.
  it('DEFAULT IS ADOPT-AT-OPEN: a fresh brain opens already log-authoritative — artifact stored, plain acks await the covering log fsync', async () => {
    const { brain } = await openBrain() // no logAuthority config = the fleet default

    const authority = brain.logAuthority()
    expect(authority.authority).toBe('log')
    expect(typeof authority.flippedAt).toBe('number')
    expect(authority.oracle, 'the open-time flip records its green oracle summary').toBeDefined()

    const artifact = (await internals(brain)
      .storage.readRawObject(AUTHORITY_ARTIFACT)
      .catch(() => null)) as { authority?: string } | null
    expect(artifact, 'the adoption wrote the switch artifact').not.toBeNull()
    expect(artifact!.authority).toBe('log')

    // The MODE assertion (not a timing one): in log authority a single-op
    // ack awaits the log's covering-fsync path.
    expect(internals(brain).generationStore.logDurability).toBe('at-ack')
    const spy = spyEnsureSynced(brain)
    await brain.add({ data: 'log mode write', type: 'document', metadata: { n: 1 } })
    expect(spy.calls(), 'adopted default: add() awaits the covering fsync').toBeGreaterThanOrEqual(1)
  })

  // THE EXPLICIT OPT-OUT: `logAuthority: 'defer'` is the pre-10 behavior —
  // tree authority, NO artifact written (a deferred posture is config, not
  // stored state), and single-op acks never await a log fsync.
  it("OPT-OUT ('defer'): the brain stays tree-authoritative, stores no artifact, and plain acks never await a log fsync", async () => {
    const { brain } = await openBrain(undefined, 'defer')

    expect(brain.logAuthority().authority).toBe('tree')
    expect(brain.logAuthority().flippedAt).toBeUndefined()

    const artifact = await internals(brain)
      .storage.readRawObject(AUTHORITY_ARTIFACT)
      .catch(() => null)
    expect(artifact, "'defer' writes no switch artifact").toBeNull()

    // The MODE assertion (not a timing one): in tree authority a single-op
    // ack must never call the log's covering-fsync path.
    const spy = spyEnsureSynced(brain)
    await brain.add({ data: 'tree mode write', type: 'document', metadata: { n: 1 } })
    expect(spy.calls(), 'tree mode: add() does not call ensureSynced').toBe(0)
    expect(internals(brain).generationStore.logDurability).toBe('deferred')
  })

  // KNOWN GAP (marked .fails — remove the marker when fixed in src): the
  // intended contract is that a fresh brain is log-complete by construction,
  // because every write dual-writes a fact. Today the VFS root
  // (00000000-0000-0000-0000-000000000000) is created at init by a baseline
  // write with NO generation and NO fact, yet it is enumerated by the
  // canonical walk — so the oracle on a fresh brain is red with exactly one
  // `pre-log-record` mismatch on the root. The adopt-at-open default (and
  // adoptLogAuthority()) CURES this by baseline backfill — a re-commit,
  // which is why this pin opens with 'defer': it holds the BY-CONSTRUCTION
  // intent, which the backfill masks but does not deliver.
  it.fails('ORACLE INTENT: a fresh brain is log-complete by construction — verdict green with zero mismatches', async () => {
    const { brain } = await openBrain(undefined, 'defer')
    await seedWrites(brain)
    await brain.flush()

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('green')
    expect(report.mismatches).toEqual([])
  })

  it('a fresh, un-backfilled brain diverges ONLY on the init-time baseline record — every user write is exactly reproduced', async () => {
    // 'defer': the adopt-at-open default would have backfilled the baseline
    // already — this pin needs the brain genuinely un-backfilled.
    const { brain } = await openBrain(undefined, 'defer')
    await seedWrites(brain)
    await brain.flush()

    const report = await brain.verifyLogAuthority()
    // Tolerant pin (stays true after the baseline gap is fixed in src):
    // whatever the verdict, no USER record may ever diverge — the only
    // admissible mismatch is the init-time baseline root, as pre-log-record.
    expect(
      report.mismatches.every(
        (m) => m.id === VFS_ROOT && m.reason === 'pre-log-record' && m.kind === 'noun'
      ),
      'the only divergence on a fresh brain is the baseline root record'
    ).toBe(true)
    expect(report.matched).toBe(report.nounsChecked - report.mismatches.length)
    expect(report.mismatchListTruncated).toBe(false)
  })

  it('THE ORACLE GOES GREEN on a log-complete brain: adds + update + remove, every canonical row exactly reproduced', async () => {
    // 'defer' + manual backfill: the exact-count pins below (5 generations)
    // depend on the log holding ONLY this test's writes — the adopt-at-open
    // default would inject its own backfill generation at init.
    const { brain } = await openBrain(undefined, 'defer')
    await seedWrites(brain)
    await backfillBaseline(brain) // final write — see the helper's contract
    await brain.flush()

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('green')
    expect(report.mismatches).toEqual([])
    expect(report.mismatchListTruncated).toBe(false)
    // Live count: the kept document + the VFS root (the removed one is a
    // tombstone in the log and absent from canonical — checked, not counted).
    expect(report.nounsChecked).toBe(2)
    expect(report.matched).toBe(2)
    // 5 committed generations: add, add, update, remove, root backfill.
    expect(report.generationsScanned).toBe(5)
  })

  it('THE ORACLE NAMES pre-log records: a canonical row no fact ever recorded reports pre-log-record, by id', async () => {
    const { brain } = await openBrain(undefined, 'defer')
    await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()
    expect((await brain.verifyLogAuthority()).verdict, 'sanity: green before aging').toBe('green')

    // Simulate an aged brain: write one canonical record DIRECTLY at the
    // storage layer (the write path never sees it, so no fact exists) —
    // the pre-log shape: flat metadata, no _fmt stamp, 384-dim vector.
    const legacyId = '00000000-0000-4000-8000-00000000a6ed'
    const storage = internals(brain).storage
    await storage.saveNoun({
      id: legacyId,
      vector: new Array(384).fill(0.01),
      connections: new Map(),
      level: 0
    })
    await storage.saveNounMetadata(legacyId, {
      noun: 'document',
      confidence: 0.75,
      createdAt: 1700000000000,
      updatedAt: 1700000000000,
      _rev: 1,
      legacyField: 'legacy-value'
    })

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('red')
    expect(report.mismatches).toHaveLength(1)
    expect(report.mismatches[0]).toEqual({
      id: legacyId,
      kind: 'noun',
      reason: 'pre-log-record'
    })
  })

  it('THE FLIP REFUSES ON A LOG-AHEAD DIVERGENCE: the witness denies what the log claims — nothing written, nothing changed', async () => {
    // Contract update (adoptLogAuthority's baseline backfill): curable
    // divergences — pre-log records and witness drift — are re-committed
    // and the flip proceeds; ONLY log-AHEAD divergences (the log claims
    // state canonical denies) refuse, because no backfill can make the log
    // un-claim a live row. This test stages exactly that incurable shape.
    // 'defer': the brain must still be tree-authoritative (no artifact) so
    // the refusal's nothing-written pins below have meaning.
    const { brain } = await openBrain(undefined, 'defer')
    const { kept } = await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()

    // The log says `kept` is live; its canonical record vanishes behind the
    // write path's back (log-live-canonical-absent — the witness wins).
    const storage = internals(brain).storage
    await storage.writeNounRaw(kept, { metadata: null, vector: null })

    let error: Error | null = null
    try {
      await brain.adoptLogAuthority()
    } catch (err) {
      error = err as Error
    }
    expect(error, 'the flip rejects on a log-ahead divergence').not.toBeNull()
    expect(error!.message).toMatch(/witness denies/)
    expect(error!.message).toMatch(/log-live-canonical-absent/)

    // Nothing changed: authority still tree, no artifact, deferred durability.
    expect(brain.logAuthority().authority).toBe('tree')
    const artifact = await storage.readRawObject(AUTHORITY_ARTIFACT).catch(() => null)
    expect(artifact, 'a refused flip writes no artifact').toBeNull()
    expect(internals(brain).generationStore.logDurability).toBe('deferred')
  })

  it('THE FLIP LANDS ON GREEN: the report is the receipt, the artifact is on disk, and durable-at-ack engages immediately', async () => {
    // 'defer': this pin exercises the EXPLICIT flip — the adopt-at-open
    // default would have landed it before the test began.
    const { brain } = await openBrain(undefined, 'defer')
    await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()

    const report: OracleReport = await brain.adoptLogAuthority()
    expect(report.verdict).toBe('green')

    const authority = brain.logAuthority()
    expect(authority.authority).toBe('log')
    expect(typeof authority.flippedAt).toBe('number')
    expect(authority.oracle).toBeDefined()
    expect(authority.oracle!.nounsChecked).toBe(report.nounsChecked)
    expect(authority.oracle!.generationsScanned).toBe(report.generationsScanned)

    const artifact = (await internals(brain)
      .storage.readRawObject(AUTHORITY_ARTIFACT)
      .catch(() => null)) as { authority?: string } | null
    expect(artifact, 'the switch artifact exists on disk').not.toBeNull()
    expect(artifact!.authority).toBe('log')

    // Durable-at-ack engaged in THIS session: the next single-op ack awaits
    // a covering log fsync.
    expect(internals(brain).generationStore.logDurability).toBe('at-ack')
    const spy = spyEnsureSynced(brain)
    await brain.add({ data: 'post-flip write', type: 'document', metadata: { n: 3 } })
    expect(spy.calls(), 'log mode: add() awaits the covering fsync').toBeGreaterThanOrEqual(1)
  })

  it('THE SWITCH SURVIVES REOPEN: authority restored at open with no re-verification, durable-at-ack active in the new session', async () => {
    const { brain, dir } = await openBrain(undefined, 'defer')
    await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()
    await brain.adoptLogAuthority()
    const flipReceipt = brain.logAuthority()
    await (brain as unknown as { close: () => Promise<void> }).close()

    // Reopen with 'defer' too: the restored authority below can then ONLY
    // come from the stored artifact (a stored artifact always wins; had the
    // default re-adopted, flippedAt/oracle would differ from the receipt).
    const { brain: reopened } = await openBrain(dir, 'defer')
    const restored = reopened.logAuthority()
    expect(restored.authority).toBe('log')
    // No re-verification happened at open: the restored record IS the stored
    // flip receipt, oracle summary and timestamp intact.
    expect(restored.flippedAt).toBe(flipReceipt.flippedAt)
    expect(restored.oracle).toEqual(flipReceipt.oracle)

    // Mode restored at open: an ack in the new session awaits the log fsync.
    expect(internals(reopened).generationStore.logDurability).toBe('at-ack')
    const spy = spyEnsureSynced(reopened)
    await reopened.add({ data: 'new session write', type: 'document', metadata: { n: 4 } })
    expect(spy.calls(), 'reopened log mode: add() awaits the covering fsync').toBeGreaterThanOrEqual(1)
  })

  it('STATE-DIFFERS: canonical drift the write path never saw is named, by id', async () => {
    const { brain } = await openBrain(undefined, 'defer')
    const { kept } = await seedWrites(brain)
    await backfillBaseline(brain)
    await brain.flush()
    expect((await brain.verifyLogAuthority()).verdict, 'sanity: green before drift').toBe('green')

    // Drift one canonical metadata record DIRECTLY at the storage layer —
    // the log never hears about it. This is the witness-drift case the
    // oracle exists to catch.
    const storage = internals(brain).storage
    const current = await storage.getNounMetadata(kept)
    expect(current, 'the seeded record has stored metadata').toBeTruthy()
    await storage.saveNounMetadata(kept, { ...current!, driftedByTest: true })

    const report = await brain.verifyLogAuthority()
    expect(report.verdict).toBe('red')
    expect(report.mismatches).toHaveLength(1)
    expect(report.mismatches[0]).toEqual({
      id: kept,
      kind: 'noun',
      reason: 'state-differs'
    })
  })
})