feat: generation-segment store — the D1+D3 packed-tier file format
First stage of the co-frozen D1+D3+repacking unit: the format core, self-contained under _generations/segments/. - seg-<firstGen,20pad>.bgs: append-once packs of consecutive generations (magic BGS1; frame = u32 len + u32 crc32c + msgpack [generation, timestamp, delta, records, flags]; flags reserves compressed-payload evolution without a format break). Sealed segments are immutable — fold refuses overlap with sealed ranges. - seg-<firstGen>.idx: DERIVED sidecar (per-generation frame offsets + per-id generation postings + checksums); lost/corrupt sidecars rebuild from their segment loudly; a damaged segment (frame CRC mismatch) fails loudly, never serves wrong bytes. - manifest.json: the one discovery path — open() reads it and never lists the packed backlog (the scan-wedge class's cure); refuses a newer manifest version rather than serving partial history. - D3 semantics: dropSegmentsBelow reclaims WHOLE segments at boundaries only and bumps compactedBelow durably; archival-profile enforcement stays with the caller per the co-freeze. - D8 rider: digestThroughPacked(g) — deterministic crc32c chain over sealed-segment checksums (+ frame-level prefix mid-segment), O(segments), reopen-stable. Also fixes a cross-adapter contract bug the suite caught: memory storage's deleteObjectFromPath ignored the raw-bytes store, so deleteRawObject on a raw-bytes path (fact-log or segment files) silently no-op'd — deletes now match filesystem unlink semantics. Six pins. Wiring into GenerationStore (two-tier reads, the repacker, cold-open manifest path) lands with the rest of the unit before its release; cortex's fact-record/stamp shapes reconcile the sidecar keying when they post.
This commit is contained in:
parent
f8e6da2b66
commit
d8acb3776b
3 changed files with 614 additions and 0 deletions
150
tests/unit/db/generation-segments.test.ts
Normal file
150
tests/unit/db/generation-segments.test.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
/**
|
||||
* @module tests/unit/db/generation-segments
|
||||
* @description The generation-segment store (Stage-2 D1+D3 file format).
|
||||
* Laws: (1) fold → read round-trips deltas and records byte-faithfully via
|
||||
* sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen
|
||||
* reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from
|
||||
* its segment loudly, a damaged SEGMENT fails loudly (never silent wrong
|
||||
* data); (4) D3 reclaim drops whole segments only and bumps compactedBelow;
|
||||
* (5) the packed digest is deterministic across reopen; (6) immutability —
|
||||
* fold refuses overlap with sealed ranges.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import {
|
||||
GenerationSegmentStore,
|
||||
SEGMENTS_PREFIX,
|
||||
type FoldGeneration
|
||||
} from '../../../src/db/generationSegments.js'
|
||||
|
||||
const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
|
||||
|
||||
const gen = (g: number, recordCount = 2): FoldGeneration => ({
|
||||
generation: g,
|
||||
timestamp: 1_700_000_000_000 + g,
|
||||
delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g },
|
||||
records: Array.from({ length: recordCount }, (_, i) => ({
|
||||
kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb',
|
||||
id: UUID(g * 100 + i),
|
||||
record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } }
|
||||
}))
|
||||
})
|
||||
|
||||
describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => {
|
||||
let storage: MemoryStorage
|
||||
let store: GenerationSegmentStore
|
||||
|
||||
beforeEach(async () => {
|
||||
storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
store = new GenerationSegmentStore(storage as any)
|
||||
await store.open()
|
||||
})
|
||||
|
||||
it('fold → read round-trips deltas and records via sidecar point-reads', async () => {
|
||||
const meta = await store.fold([gen(1), gen(2), gen(3)])
|
||||
expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 })
|
||||
expect(meta.checksum).toBeGreaterThan(0)
|
||||
|
||||
expect(store.hasGeneration(2)).toBe(true)
|
||||
expect(store.hasGeneration(4)).toBe(false)
|
||||
|
||||
const d2 = await store.readDelta(2)
|
||||
expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 })
|
||||
expect(d2?.timestamp).toBe(1_700_000_000_002)
|
||||
|
||||
const records = await store.readRecords(3)
|
||||
expect(records).toHaveLength(2)
|
||||
expect(records![0]).toEqual({
|
||||
kind: 'noun',
|
||||
id: UUID(300),
|
||||
record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } }
|
||||
})
|
||||
// Point read by id, both kinds.
|
||||
expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({
|
||||
metadata: { noun: 'document', v: 3 },
|
||||
vector: { v: [3, 1] }
|
||||
})
|
||||
expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull()
|
||||
})
|
||||
|
||||
it('reopen discovers everything from the manifest alone — no listing', async () => {
|
||||
await store.fold([gen(1), gen(2)])
|
||||
await store.fold([gen(3), gen(4)])
|
||||
|
||||
const reopened = new GenerationSegmentStore(storage as any)
|
||||
await reopened.open()
|
||||
expect(reopened.segments()).toHaveLength(2)
|
||||
expect(reopened.hasGeneration(4)).toBe(true)
|
||||
expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001)
|
||||
})
|
||||
|
||||
it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => {
|
||||
const meta = await store.fold([gen(1), gen(2)])
|
||||
const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx`
|
||||
await storage.deleteRawObject(idxPath)
|
||||
|
||||
const reopened = new GenerationSegmentStore(storage as any)
|
||||
await reopened.open()
|
||||
// Rebuild path: still serves correct data.
|
||||
expect((await reopened.readRecords(2))!).toHaveLength(2)
|
||||
|
||||
// Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud.
|
||||
const segPath = `${SEGMENTS_PREFIX}/${meta.file}`
|
||||
const bytes = (await storage.readRawBytes(segPath))!
|
||||
bytes[bytes.length - 3] ^= 0xff
|
||||
await storage.writeRawBytes(segPath, bytes)
|
||||
const damaged = new GenerationSegmentStore(storage as any)
|
||||
await damaged.open()
|
||||
;(damaged as any).sidecars.clear()
|
||||
await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes
|
||||
await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/)
|
||||
})
|
||||
|
||||
it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => {
|
||||
await store.fold([gen(1), gen(2)])
|
||||
await store.fold([gen(3), gen(4)])
|
||||
await store.fold([gen(5), gen(6)])
|
||||
|
||||
// Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops.
|
||||
const r1 = await store.dropSegmentsBelow(4)
|
||||
expect(r1).toEqual({ dropped: 1, compactedBelow: 3 })
|
||||
expect(store.hasGeneration(1)).toBe(false)
|
||||
expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole
|
||||
|
||||
// Bytes actually gone.
|
||||
expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull()
|
||||
|
||||
// Horizon past everything: the rest drop; compactedBelow is durable.
|
||||
const r2 = await store.dropSegmentsBelow(7)
|
||||
expect(r2.dropped).toBe(2)
|
||||
const reopened = new GenerationSegmentStore(storage as any)
|
||||
await reopened.open()
|
||||
expect(reopened.compactedBelow()).toBe(7)
|
||||
expect(reopened.segments()).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('the packed digest is deterministic across reopen and changes with history', async () => {
|
||||
await store.fold([gen(1), gen(2), gen(3)])
|
||||
const atSeal = await store.digestThroughPacked(3)
|
||||
const midSegment = await store.digestThroughPacked(2)
|
||||
expect(atSeal).not.toBeNull()
|
||||
expect(midSegment).not.toBeNull()
|
||||
expect(midSegment).not.toBe(atSeal)
|
||||
|
||||
const reopened = new GenerationSegmentStore(storage as any)
|
||||
await reopened.open()
|
||||
expect(await reopened.digestThroughPacked(3)).toBe(atSeal)
|
||||
expect(await reopened.digestThroughPacked(2)).toBe(midSegment)
|
||||
|
||||
await reopened.fold([gen(4)])
|
||||
expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal)
|
||||
})
|
||||
|
||||
it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => {
|
||||
await store.fold([gen(1), gen(2)])
|
||||
await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/)
|
||||
await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/)
|
||||
await expect(store.fold([])).rejects.toThrow(/at least one generation/)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue