brainy/tests/unit/db/generation-segments.test.ts

/**
 * @module tests/unit/db/generation-segments
 * @description The generation-segment store (Stage-2 D1+D3 file format).
 * Laws: (1) fold → read round-trips deltas and records byte-faithfully via
 * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen
 * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from
 * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong
 * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow;
 * (5) the packed digest is deterministic across reopen; (6) immutability —
 * fold refuses overlap with sealed ranges.
 */
import { describe, it, expect, beforeEach } from 'vitest'
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
import {
  GenerationSegmentStore,
  SEGMENTS_PREFIX,
  type FoldGeneration
} from '../../../src/db/generationSegments.js'

const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`

const gen = (g: number, recordCount = 2): FoldGeneration => ({
  generation: g,
  timestamp: 1_700_000_000_000 + g,
  delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g },
  records: Array.from({ length: recordCount }, (_, i) => ({
    kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb',
    id: UUID(g * 100 + i),
    record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } }
  }))
})

describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => {
  let storage: MemoryStorage
  let store: GenerationSegmentStore

  beforeEach(async () => {
    storage = new MemoryStorage()
    await storage.init()
    store = new GenerationSegmentStore(storage as any)
    await store.open()
  })

  it('fold → read round-trips deltas and records via sidecar point-reads', async () => {
    const meta = await store.fold([gen(1), gen(2), gen(3)])
    expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 })
    expect(meta.checksum).toBeGreaterThan(0)

    expect(store.hasGeneration(2)).toBe(true)
    expect(store.hasGeneration(4)).toBe(false)

    const d2 = await store.readDelta(2)
    expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 })
    expect(d2?.timestamp).toBe(1_700_000_000_002)

    const records = await store.readRecords(3)
    expect(records).toHaveLength(2)
    expect(records![0]).toEqual({
      kind: 'noun',
      id: UUID(300),
      record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } }
    })
    // Point read by id, both kinds.
    expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({
      metadata: { noun: 'document', v: 3 },
      vector: { v: [3, 1] }
    })
    expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull()
  })

  it('reopen discovers everything from the manifest alone — no listing', async () => {
    await store.fold([gen(1), gen(2)])
    await store.fold([gen(3), gen(4)])

    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    expect(reopened.segments()).toHaveLength(2)
    expect(reopened.hasGeneration(4)).toBe(true)
    expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001)
  })

  it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => {
    const meta = await store.fold([gen(1), gen(2)])
    const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx`
    await storage.deleteRawObject(idxPath)

    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    // Rebuild path: still serves correct data.
    expect((await reopened.readRecords(2))!).toHaveLength(2)

    // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud.
    const segPath = `${SEGMENTS_PREFIX}/${meta.file}`
    const bytes = (await storage.readRawBytes(segPath))!
    bytes[bytes.length - 3] ^= 0xff
    await storage.writeRawBytes(segPath, bytes)
    const damaged = new GenerationSegmentStore(storage as any)
    await damaged.open()
    ;(damaged as any).sidecars.clear()
    await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes
    await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/)
  })

  it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => {
    await store.fold([gen(1), gen(2)])
    await store.fold([gen(3), gen(4)])
    await store.fold([gen(5), gen(6)])

    // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops.
    const r1 = await store.dropSegmentsBelow(4)
    expect(r1).toEqual({ dropped: 1, compactedBelow: 3 })
    expect(store.hasGeneration(1)).toBe(false)
    expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole

    // Bytes actually gone.
    expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull()

    // Horizon past everything: the rest drop; compactedBelow is durable.
    const r2 = await store.dropSegmentsBelow(7)
    expect(r2.dropped).toBe(2)
    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    expect(reopened.compactedBelow()).toBe(7)
    expect(reopened.segments()).toHaveLength(0)
  })

  it('the packed digest is deterministic across reopen and changes with history', async () => {
    await store.fold([gen(1), gen(2), gen(3)])
    const atSeal = await store.digestThroughPacked(3)
    const midSegment = await store.digestThroughPacked(2)
    expect(atSeal).not.toBeNull()
    expect(midSegment).not.toBeNull()
    expect(midSegment).not.toBe(atSeal)

    const reopened = new GenerationSegmentStore(storage as any)
    await reopened.open()
    expect(await reopened.digestThroughPacked(3)).toBe(atSeal)
    expect(await reopened.digestThroughPacked(2)).toBe(midSegment)

    await reopened.fold([gen(4)])
    expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal)
  })

  it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => {
    await store.fold([gen(1), gen(2)])
    await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/)
    await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/)
    await expect(store.fold([])).rejects.toThrow(/at least one generation/)
  })
})