brainy/tests/unit/db/fact-log.test.ts

/**
 * @module tests/unit/db/fact-log
 * @description The generation fact log in isolation: wire-format round-trip
 * (positional msgpack facts, bin16 uuids, body-less tombstones), crc32c
 * framing with torn-tail detection, open-time truncation to committed truth
 * (the log can only ever be AHEAD after a crash; open cuts it back), rotation
 * with a manifest-first flip, exactly-once scans with the frozen telemetry
 * shape, and the mmap segment handoff excluding the mutable tail.
 */
import { describe, it, expect, beforeEach } from 'vitest'
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
import {
  FactLog,
  FACTS_PREFIX,
  type CommitFact,
  type FactLogStorage,
  storageSupportsFactLog
} from '../../../src/db/factLog.js'
import { crc32c } from '../../../src/utils/crc32c.js'

const UUID = (n: number): string =>
  `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`

const fact = (generation: number, overrides?: Partial<CommitFact>): CommitFact => ({
  generation,
  timestamp: 1_700_000_000_000 + generation,
  ops: [
    {
      kind: 'noun',
      id: UUID(generation),
      record: { metadata: { noun: 'document', title: `doc ${generation}` }, vector: { v: [1, 2] } }
    }
  ],
  ...overrides
})

describe('crc32c known-answer vectors', () => {
  it('matches the RFC 3720 test vectors', () => {
    expect(crc32c(new TextEncoder().encode('123456789'))).toBe(0xe3069283)
    expect(crc32c(new Uint8Array(32))).toBe(0x8a9136aa)
  })
})

describe('fact log — round-trip, framing, reconcile, rotation, scan', () => {
  let storage: FactLogStorage
  let log: FactLog

  beforeEach(async () => {
    const mem: any = new MemoryStorage()
    await mem.init()
    expect(storageSupportsFactLog(mem)).toBe(true)
    storage = mem
    log = new FactLog(storage)
    await log.open(0)
  })

  it('facts round-trip byte-exactly: ops, tombstones, meta, blobHashes', async () => {
    await log.append(fact(1))
    await log.append(
      fact(2, {
        ops: [
          { kind: 'verb', id: UUID(21), record: { metadata: { verb: 'contains' }, vector: null } },
          { kind: 'noun', id: UUID(22), record: null } // TOMBSTONE
        ],
        meta: { source: 'test' },
        blobHashes: ['abc123', 'abc123'] // multiset — duplicates preserved
      })
    )
    await log.sync()

    const scan = log.scanFacts()
    expect(scan.headGeneration).toBe(2)
    const all: CommitFact[] = []
    for await (const batch of scan.batches()) all.push(...batch.facts)

    expect(all).toHaveLength(2)
    expect(all[0].generation).toBe(1)
    expect(all[0].ops[0].id).toBe(UUID(1))
    expect(all[0].ops[0].record?.metadata).toEqual({ noun: 'document', title: 'doc 1' })
    expect(all[1].ops[0].kind).toBe('verb')
    expect(all[1].ops[1].record).toBeNull() // the tombstone is body-less
    expect(all[1].meta).toEqual({ source: 'test' })
    expect(all[1].blobHashes).toEqual(['abc123', 'abc123'])
    expect(scan.summary().factsYielded).toBe(2)
  })

  it('appends are monotonic — a replayed/duplicate generation throws', async () => {
    await log.append(fact(5))
    await expect(log.append(fact(5))).rejects.toThrow(/non-monotonic/)
    await expect(log.append(fact(3))).rejects.toThrow(/non-monotonic/)
    await expect(log.append(fact(6))).resolves.toBeUndefined() // gaps are fine (aborted reservations)
  })

  it('a torn tail (partial frame) is detected and ignored — intact prefix survives', async () => {
    await log.append(fact(1))
    await log.append(fact(2))
    await log.sync()

    // Simulate a crash mid-append: chop bytes off the tail file.
    const tailPath = `${FACTS_PREFIX}/seg-${'1'.padStart(20, '0')}.bfl`
    const bytes = (await storage.readRawBytes(tailPath))!
    await storage.writeRawBytes(tailPath, bytes.subarray(0, bytes.length - 7))

    const reopened = new FactLog(storage)
    await reopened.open(2)
    expect(reopened.headGeneration()).toBe(1) // fact 2's frame was torn → gone

    const all: CommitFact[] = []
    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
    expect(all.map((f) => f.generation)).toEqual([1])
  })

  it('open() truncates facts beyond committed truth (the crash-ahead shape)', async () => {
    await log.append(fact(1))
    await log.append(fact(2))
    await log.append(fact(3))
    await log.sync()

    // The store's committed generation is 1 — facts 2..3 never committed.
    const reopened = new FactLog(storage)
    await reopened.open(1)
    expect(reopened.headGeneration()).toBe(1)

    const all: CommitFact[] = []
    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
    expect(all.map((f) => f.generation)).toEqual([1])

    // And appends continue cleanly from the truncated head.
    await reopened.append(fact(2))
    expect(reopened.headGeneration()).toBe(2)
  })

  it('a cleared store (committed=0) truncates everything', async () => {
    await log.append(fact(1))
    await log.append(fact(2))
    await log.sync()
    const reopened = new FactLog(storage)
    await reopened.open(0)
    expect(reopened.headGeneration()).toBe(0)
  })

  it('scan honors fromGeneration/toGeneration inclusively and filters kinds', async () => {
    for (let g = 1; g <= 6; g++) await log.append(fact(g))
    await log.sync()

    const scan = log.scanFacts({ fromGeneration: 2, toGeneration: 4 })
    const all: CommitFact[] = []
    for await (const b of scan.batches()) all.push(...b.facts)
    expect(all.map((f) => f.generation)).toEqual([2, 3, 4])

    const verbsOnly = log.scanFacts({ kinds: ['verb'] })
    for await (const b of verbsOnly.batches()) {
      for (const f of b.facts) expect(f.ops.every((op) => op.kind === 'verb')).toBe(true)
    }
  })

  it('batch telemetry carries the frozen shape', async () => {
    for (let g = 1; g <= 5; g++) await log.append(fact(g))
    await log.sync()

    const scan = log.scanFacts({ batchSize: 2 })
    expect(scan.approxFactCount).toBe(5)
    const batches = []
    for await (const b of scan.batches()) batches.push(b)
    expect(batches.length).toBe(3)
    expect(batches[0]).toMatchObject({ firstGeneration: 1, lastGeneration: 2, factCount: 2 })
    expect(batches[0].byteSize).toBeGreaterThan(0)
    expect(typeof batches[0].segmentId).toBe('string')
    expect(scan.summary()).toEqual({ factsYielded: 5, segmentsRead: 1 })
  })

  it('survives reopen: head and content come back from disk', async () => {
    for (let g = 1; g <= 3; g++) await log.append(fact(g))
    await log.sync()

    const reopened = new FactLog(storage)
    await reopened.open(3)
    expect(reopened.headGeneration()).toBe(3)
    const all: CommitFact[] = []
    for await (const b of reopened.scanFacts().batches()) all.push(...b.facts)
    expect(all.map((f) => f.generation)).toEqual([1, 2, 3])
  })

  it('segmentPaths excludes the mutable tail (mmap handoff = sealed only)', async () => {
    await log.append(fact(1))
    await log.sync()
    expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed
  })

  describe('scanFacts liveness contract (Stage-2 D1)', () => {
    it('a wedged store fails LOUDLY within the first-batch bound — never a silent hang', async () => {
      // Force a sealed segment (tiny rotateBytes) so the scan must READ from
      // storage, then wedge that read: the exact production shape (a
      // backlogged brain whose segment read never returned).
      const mem: any = new MemoryStorage()
      await mem.init()
      const wedgeable = new FactLog(mem, { rotateBytes: 1 })
      await wedgeable.open(0)
      await wedgeable.append(fact(1))
      await wedgeable.append(fact(2)) // second append rotates → seg 1 sealed
      await wedgeable.sync()

      const realRead = mem.readRawBytes.bind(mem)
      mem.readRawBytes = (p: string) =>
        p.includes('facts/seg-') ? new Promise(() => {}) : realRead(p) // hangs forever

      const scan = wedgeable.scanFacts({ firstBatchTimeoutMs: 200 })
      const started = Date.now()
      await expect(scan.batches().next()).rejects.toThrow(/no first batch within 200ms/)
      expect(Date.now() - started).toBeLessThan(5_000) // bound held, not a hang
    })

    it('a healthy scan is unaffected — first batch well inside the bound, all facts delivered', async () => {
      for (let g = 1; g <= 5; g++) await log.append(fact(g))
      await log.sync()
      const scan = log.scanFacts({ batchSize: 2 })
      const all: CommitFact[] = []
      for await (const b of scan.batches()) all.push(...b.facts)
      expect(all.map((f) => f.generation)).toEqual([1, 2, 3, 4, 5])
      expect(scan.summary().factsYielded).toBe(5)
    })

    it('consumer think-time between pulls never counts against the producer', async () => {
      for (let g = 1; g <= 4; g++) await log.append(fact(g))
      await log.sync()
      // Bound tighter than the consumer's pause: only the FIRST pull is
      // raced, so a slow consumer after batch 1 must not trip the deadline.
      const gen = log.scanFacts({ batchSize: 2, firstBatchTimeoutMs: 150 }).batches()
      const first = await gen.next()
      expect(first.done).toBe(false)
      await new Promise((r) => setTimeout(r, 400)) // dawdle past the bound
      const second = await gen.next()
      expect(second.done).toBe(false)
      expect((await gen.next()).done).toBe(true)
    })
  })
})