/** * @module tests/unit/db/generation-segments * @description The generation-segment store (Stage-2 D1+D3 file format). * Laws: (1) fold → read round-trips deltas and records byte-faithfully via * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow; * (5) the packed digest is deterministic across reopen; (6) immutability — * fold refuses overlap with sealed ranges. */ import { describe, it, expect, beforeEach } from 'vitest' import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' import { GenerationSegmentStore, SEGMENTS_PREFIX, type FoldGeneration } from '../../../src/db/generationSegments.js' const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` const gen = (g: number, recordCount = 2): FoldGeneration => ({ generation: g, timestamp: 1_700_000_000_000 + g, delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g }, records: Array.from({ length: recordCount }, (_, i) => ({ kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb', id: UUID(g * 100 + i), record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } } })) }) describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { let storage: MemoryStorage let store: GenerationSegmentStore beforeEach(async () => { storage = new MemoryStorage() await storage.init() store = new GenerationSegmentStore(storage as any) await store.open() }) it('fold → read round-trips deltas and records via sidecar point-reads', async () => { const meta = await store.fold([gen(1), gen(2), gen(3)]) expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) expect(meta.checksum).toBeGreaterThan(0) expect(store.hasGeneration(2)).toBe(true) expect(store.hasGeneration(4)).toBe(false) const d2 = await store.readDelta(2) expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 }) expect(d2?.timestamp).toBe(1_700_000_000_002) const records = await store.readRecords(3) expect(records).toHaveLength(2) expect(records![0]).toEqual({ kind: 'noun', id: UUID(300), record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } } }) // Point read by id, both kinds. expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({ metadata: { noun: 'document', v: 3 }, vector: { v: [3, 1] } }) expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull() }) it('reopen discovers everything from the manifest alone — no listing', async () => { await store.fold([gen(1), gen(2)]) await store.fold([gen(3), gen(4)]) const reopened = new GenerationSegmentStore(storage as any) await reopened.open() expect(reopened.segments()).toHaveLength(2) expect(reopened.hasGeneration(4)).toBe(true) expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001) }) it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => { const meta = await store.fold([gen(1), gen(2)]) const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` await storage.deleteRawObject(idxPath) const reopened = new GenerationSegmentStore(storage as any) await reopened.open() // Rebuild path: still serves correct data. expect((await reopened.readRecords(2))!).toHaveLength(2) // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud. const segPath = `${SEGMENTS_PREFIX}/${meta.file}` const bytes = (await storage.readRawBytes(segPath))! bytes[bytes.length - 3] ^= 0xff await storage.writeRawBytes(segPath, bytes) const damaged = new GenerationSegmentStore(storage as any) await damaged.open() ;(damaged as any).sidecars.clear() await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/) }) it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => { await store.fold([gen(1), gen(2)]) await store.fold([gen(3), gen(4)]) await store.fold([gen(5), gen(6)]) // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops. const r1 = await store.dropSegmentsBelow(4) expect(r1).toEqual({ dropped: 1, compactedBelow: 3 }) expect(store.hasGeneration(1)).toBe(false) expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole // Bytes actually gone. expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull() // Horizon past everything: the rest drop; compactedBelow is durable. const r2 = await store.dropSegmentsBelow(7) expect(r2.dropped).toBe(2) const reopened = new GenerationSegmentStore(storage as any) await reopened.open() expect(reopened.compactedBelow()).toBe(7) expect(reopened.segments()).toHaveLength(0) }) it('the packed digest is deterministic across reopen and changes with history', async () => { await store.fold([gen(1), gen(2), gen(3)]) const atSeal = await store.digestThroughPacked(3) const midSegment = await store.digestThroughPacked(2) expect(atSeal).not.toBeNull() expect(midSegment).not.toBeNull() expect(midSegment).not.toBe(atSeal) const reopened = new GenerationSegmentStore(storage as any) await reopened.open() expect(await reopened.digestThroughPacked(3)).toBe(atSeal) expect(await reopened.digestThroughPacked(2)).toBe(midSegment) await reopened.fold([gen(4)]) expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal) }) it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => { await store.fold([gen(1), gen(2)]) await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/) await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) })