open-brainy/tests/unit/db/generation-segments.test.ts

/**
 * @module tests/unit/db/generation-segments
 * @description The generation-segment store (Stage-2 D1+D3 file format).
 * Laws: (1) fold → read round-trips deltas and records byte-faithfully via
 * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen
 * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from
 * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong
 * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow;
 * (5) the packed digest is deterministic across reopen; (6) immutability —
 * fold refuses overlap with sealed ranges.
 */
import { describe, it, expect, beforeEach } from 'vitest'
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
import {
  GenerationSegmentStore,
  SEGMENTS_PREFIX,
  type FoldGeneration
} from '../../../src/db/generationSegments.js'

const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`

const gen = (g: number, recordCount = 2): FoldGeneration => ({
  generation: g,
  timestamp: 1_700_000_000_000 + g,
  delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g },
  records: Array.from({ length: recordCount }, (_, i) => ({
    kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb',
    id: UUID(g * 100 + i),
    record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } }
  }))
})

describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => {
  let storage: MemoryStorage
  let store: GenerationSegmentStore

  beforeEach(async () => {
    storage = new MemoryStorage()
    await storage.init()
    store = new GenerationSegmentStore(storage as any)
    await store.open()
  })

  it('fold → read round-trips deltas and records via sidecar point-reads', async () => {
    const meta = await store.fold([gen(1), gen(2), gen(3)])
    expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 })
    expect(meta.checksum).toBeGreaterThan(0)

    expect(store.hasGeneration(2)).toBe(true)
    expect(store.hasGeneration(4)).toBe(false)

    const d2 = await store.readDelta(2)
    expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 })
    expect(d2?.timestamp).toBe(1_700_000_000_002)

    const records = await store.readRecords(3)
    expect(records).toHaveLength(2)
    expect(records![0]).toEqual({
      kind: 'noun',
      id: UUID(300),
      record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } }
    })
    // Point read by id, both kinds.
    expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({
      metadata: { noun: 'document', v: 3 },
      vector: { v: [3, 1] }
    })
    expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull()
  })

  it('reopen discovers everything from the manifest alone — no listing', async () => {
    await store.fold([gen(1), gen(2)])
    await store.fold([gen(3), gen(4)])

    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    expect(reopened.segments()).toHaveLength(2)
    expect(reopened.hasGeneration(4)).toBe(true)
    expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001)
  })

  it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => {
    const meta = await store.fold([gen(1), gen(2)])
    const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx`
    await storage.deleteRawObject(idxPath)

    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    // Rebuild path: still serves correct data.
    expect((await reopened.readRecords(2))!).toHaveLength(2)

    // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud.
    const segPath = `${SEGMENTS_PREFIX}/${meta.file}`
    const bytes = (await storage.readRawBytes(segPath))!
    bytes[bytes.length - 3] ^= 0xff
    await storage.writeRawBytes(segPath, bytes)
    const damaged = new GenerationSegmentStore(storage as any)
    await damaged.open()
    ;(damaged as any).sidecars.clear()
    await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes
    await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/)
  })

  it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => {
    await store.fold([gen(1), gen(2)])
    await store.fold([gen(3), gen(4)])
    await store.fold([gen(5), gen(6)])

    // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops.
    const r1 = await store.dropSegmentsBelow(4)
    expect(r1).toEqual({ dropped: 1, compactedBelow: 3 })
    expect(store.hasGeneration(1)).toBe(false)
    expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole

    // Bytes actually gone.
    expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull()

    // Horizon past everything: the rest drop; compactedBelow is durable.
    const r2 = await store.dropSegmentsBelow(7)
    expect(r2.dropped).toBe(2)
    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    expect(reopened.compactedBelow()).toBe(7)
    expect(reopened.segments()).toHaveLength(0)
  })

  it('the packed digest is deterministic across reopen and changes with history', async () => {
    await store.fold([gen(1), gen(2), gen(3)])
    const atSeal = await store.digestThroughPacked(3)
    const midSegment = await store.digestThroughPacked(2)
    expect(atSeal).not.toBeNull()
    expect(midSegment).not.toBeNull()
    expect(midSegment).not.toBe(atSeal)

    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    expect(await reopened.digestThroughPacked(3)).toBe(atSeal)
    expect(await reopened.digestThroughPacked(2)).toBe(midSegment)

    await reopened.fold([gen(4)])
    expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal)
  })

  it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => {
    await store.fold([gen(1), gen(2)])
    await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/)
    await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/)
    await expect(store.fold([])).rejects.toThrow(/at least one generation/)
  })

  // ==========================================================================
  // THE DENSITY LAW
  // ==========================================================================
  //
  // A sealed segment declares a CONTIGUOUS range and every reader treats that
  // range as containment. Folding a sparse batch therefore makes the segment
  // claim generations it does not hold — and because `open()` merges declared
  // ranges back into committedRanges, the hole is re-admitted as committed
  // history and every later maintenance pass fails asking for a frame that was
  // never written. That is the "generation N is inside sealed segment
  // seg-....bgs's declared range but has no frame — packed history is damaged"
  // narration seen on every run of the affected stores.

  it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => {
    await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow(
      /not contiguous: 2 → 4 skips 1 generation/
    )
    // The refusal loses nothing: no segment was sealed, so the generations
    // stay in the live tier and the next pass folds them correctly.
    expect(store.segments()).toHaveLength(0)
    expect(store.hasGeneration(1)).toBe(false)
  })

  it('a wider gap names how many generations it would have swallowed', async () => {
    await expect(store.fold([gen(10), gen(20)])).rejects.toThrow(
      /not contiguous: 10 → 20 skips 9 generation\(s\)/
    )
  })

  it('two contiguous runs folded separately declare honest ranges', async () => {
    // What the caller now does instead of folding across the gap.
    const a = await store.fold([gen(1), gen(2), gen(3)])
    const b = await store.fold([gen(7), gen(8)])
    expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 })
    expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 })
    // The gap is honestly outside the packed tier.
    for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false)
    for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true)
    expect(await store.actualRanges()).toEqual([
      [1, 3],
      [7, 8]
    ])
  })

  it('actualRanges() is exact and I/O-free for dense segments', async () => {
    await store.fold([gen(1), gen(2)])
    await store.fold([gen(3), gen(4)])
    // Adjacent dense segments each contribute their declared range.
    expect(await store.actualRanges()).toEqual([
      [1, 2],
      [3, 4]
    ])
  })

  // ---- pre-existing damage: a store sealed by the old writer ----------------

  /**
   * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and
   * sidecar for a contiguous run, then rewrite the manifest so the segment
   * declares a wider range than the frames it holds. This reproduces on disk
   * exactly what the affected stores carry, without needing the old code.
   */
  const sealSparseSegment = async (): Promise<void> => {
    await store.fold([gen(1), gen(2), gen(3)])
    const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any
    // Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become
    // holes inside a sealed range.
    manifest.segments[0].lastGeneration = 5
    await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest)
  }

  it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => {
    await sealSparseSegment()
    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()

    // The frames it really holds still serve, byte-faithfully.
    expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002)
    expect(await reopened.readRecords(3)).toHaveLength(2)

    // The holes answer "not packed" instead of throwing. This is the fix for
    // the wedge: the old reader threw here on EVERY maintenance pass.
    expect(await reopened.readDelta(4)).toBeNull()
    expect(await reopened.readRecords(5)).toBeNull()
  })

  it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => {
    await sealSparseSegment()
    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    // Declared 1..5; actually holds 1..3. The store seeds committedRanges from
    // THIS, so generations 4 and 5 never become committed history again.
    expect(await reopened.actualRanges()).toEqual([[1, 3]])
  })

  it('a DENSE segment missing a frame is still loud damage', async () => {
    // The other side of the branch: when the manifest claims a complete span,
    // a missing frame means the manifest and sidecar disagree — real damage,
    // and it must not be quietly downgraded to "unpacked".
    await store.fold([gen(1), gen(2), gen(3)])
    const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx`
    const raw = (await storage.readRawBytes(idxPath))!
    const { decode, encode } = await import('@msgpack/msgpack')
    const idx = decode(raw) as any
    // Drop generation 2's entry while the manifest still declares 3 frames.
    idx.generations = idx.generations.filter(([g]: [number]) => g !== 2)
    await storage.writeRawBytes(idxPath, encode(idx))

    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    await expect(reopened.readDelta(2)).rejects.toThrow(
      /manifest and the sidecar disagree; packed history is damaged/
    )
  })
})