feat: generation-segment store — the D1+D3 packed-tier file format
First stage of the co-frozen D1+D3+repacking unit: the format core, self-contained under _generations/segments/. - seg-<firstGen,20pad>.bgs: append-once packs of consecutive generations (magic BGS1; frame = u32 len + u32 crc32c + msgpack [generation, timestamp, delta, records, flags]; flags reserves compressed-payload evolution without a format break). Sealed segments are immutable — fold refuses overlap with sealed ranges. - seg-<firstGen>.idx: DERIVED sidecar (per-generation frame offsets + per-id generation postings + checksums); lost/corrupt sidecars rebuild from their segment loudly; a damaged segment (frame CRC mismatch) fails loudly, never serves wrong bytes. - manifest.json: the one discovery path — open() reads it and never lists the packed backlog (the scan-wedge class's cure); refuses a newer manifest version rather than serving partial history. - D3 semantics: dropSegmentsBelow reclaims WHOLE segments at boundaries only and bumps compactedBelow durably; archival-profile enforcement stays with the caller per the co-freeze. - D8 rider: digestThroughPacked(g) — deterministic crc32c chain over sealed-segment checksums (+ frame-level prefix mid-segment), O(segments), reopen-stable. Also fixes a cross-adapter contract bug the suite caught: memory storage's deleteObjectFromPath ignored the raw-bytes store, so deleteRawObject on a raw-bytes path (fact-log or segment files) silently no-op'd — deletes now match filesystem unlink semantics. Six pins. Wiring into GenerationStore (two-tier reads, the repacker, cold-open manifest path) lands with the rest of the unit before its release; cortex's fact-record/stamp shapes reconcile the sidecar keying when they post.
This commit is contained in:
parent
f8e6da2b66
commit
d8acb3776b
3 changed files with 614 additions and 0 deletions
459
src/db/generationSegments.ts
Normal file
459
src/db/generationSegments.ts
Normal file
|
|
@ -0,0 +1,459 @@
|
||||||
|
/**
|
||||||
|
* @module db/generationSegments
|
||||||
|
* @description The generation-segment store — Stage-2 D1+D3+repacking's file
|
||||||
|
* format (co-frozen 2026-07-19; design: the d1-d3-repacking spec).
|
||||||
|
*
|
||||||
|
* Packs CONSECUTIVE cold generations' record-sets (before-images + delta)
|
||||||
|
* into append-once segment files with derived sidecar indexes, so history
|
||||||
|
* scales in SEGMENTS (tens) instead of FILES-PER-GENERATION (hundreds of
|
||||||
|
* thousands), and cold-open reads ONE manifest instead of listing the
|
||||||
|
* backlog. Layout under `_generations/segments/`:
|
||||||
|
*
|
||||||
|
* - `seg-<firstGen, zero-padded 20>.bgs` — magic "BGS1", then one frame per
|
||||||
|
* generation: `u32 payloadLen | u32 crc32c | msgpack payload`. Payload is
|
||||||
|
* POSITIONAL: `[generation, timestamp, delta, records[], flags]` with
|
||||||
|
* records `[kindByte, id, record]`. `flags` reserves encoding evolution
|
||||||
|
* (bit 0 = compressed payload — v1 always 0; a future writer upgrade,
|
||||||
|
* never a format break). Sealed segments are IMMUTABLE — the fact log's
|
||||||
|
* own law, generalized.
|
||||||
|
* - `seg-<firstGen>.idx` — DERIVED sidecar (msgpack): per-generation frame
|
||||||
|
* offsets (point reads = one ranged read, never a listing) + per-id
|
||||||
|
* generation postings (per-id chain rebuilds read only what they need).
|
||||||
|
* Corrupt/missing → rebuilt from its segment in one sequential read,
|
||||||
|
* loudly.
|
||||||
|
* - `manifest.json` — the segment catalogue + `compactedBelow` (D3's
|
||||||
|
* horizon marker). Cold-open reads THIS; the packed backlog is never
|
||||||
|
* listed.
|
||||||
|
*
|
||||||
|
* D3 semantics carried here: bounded-retention reclaim drops WHOLE segments
|
||||||
|
* at boundaries (O(1) per segment, no rewrite); under the archival profile
|
||||||
|
* (`retention: 'all'`) nothing here is ever dropped — folding is the only
|
||||||
|
* transform (re-representation, never deletion).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { encode as msgpackEncode, decode as msgpackDecode } from '@msgpack/msgpack'
|
||||||
|
import { crc32c } from '../utils/crc32c.js'
|
||||||
|
import type { FactLogStorage } from './factLog.js'
|
||||||
|
import { prodLog } from '../utils/logger.js'
|
||||||
|
|
||||||
|
/** Directory for segment files + manifest, under the generations prefix. */
|
||||||
|
export const SEGMENTS_PREFIX = '_generations/segments'
|
||||||
|
|
||||||
|
/** Target sealed-segment size (co-freeze proposal; tunable on evidence). */
|
||||||
|
export const SEGMENT_TARGET_BYTES = 64 * 1024 * 1024
|
||||||
|
|
||||||
|
const MAGIC = new TextEncoder().encode('BGS1')
|
||||||
|
const FRAME_PREFIX_BYTES = 8 // u32 payloadLen + u32 crc32c
|
||||||
|
const MANIFEST_PATH = `${SEGMENTS_PREFIX}/manifest.json`
|
||||||
|
|
||||||
|
/** One generation's fold input — exactly what the live tier holds for it. */
|
||||||
|
export interface FoldGeneration {
|
||||||
|
generation: number
|
||||||
|
timestamp: number
|
||||||
|
/** The tx.json delta object, carried verbatim. */
|
||||||
|
delta: unknown
|
||||||
|
/** The before-image record-set (empty for record-less generations). */
|
||||||
|
records: Array<{ kind: 'noun' | 'verb'; id: string; record: unknown }>
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Manifest entry for one sealed segment. */
|
||||||
|
export interface SegmentMeta {
|
||||||
|
file: string
|
||||||
|
firstGeneration: number
|
||||||
|
lastGeneration: number
|
||||||
|
frames: number
|
||||||
|
bytes: number
|
||||||
|
/** crc32c of the full segment byte stream — the digest chain's link. */
|
||||||
|
checksum: number
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SegmentManifest {
|
||||||
|
version: 1
|
||||||
|
compactedBelow: number
|
||||||
|
segments: SegmentMeta[]
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SidecarIndex {
|
||||||
|
version: 1
|
||||||
|
/** [generation, frameOffset, frameLen] ascending by generation. */
|
||||||
|
generations: Array<[number, number, number]>
|
||||||
|
/** `${kindByte}:${id}` → ascending generations holding a record for it. */
|
||||||
|
ids: Record<string, number[]>
|
||||||
|
}
|
||||||
|
|
||||||
|
const segmentFileName = (firstGeneration: number): string =>
|
||||||
|
`seg-${String(firstGeneration).padStart(20, '0')}.bgs`
|
||||||
|
const sidecarFileName = (firstGeneration: number): string =>
|
||||||
|
`seg-${String(firstGeneration).padStart(20, '0')}.idx`
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The generation-segment store. Owns the packed tier ONLY — the live
|
||||||
|
* per-generation tier and the routing between tiers belong to
|
||||||
|
* `GenerationStore`. All mutating entry points here are called under the
|
||||||
|
* generation store's commit mutex.
|
||||||
|
*/
|
||||||
|
export class GenerationSegmentStore {
|
||||||
|
private readonly storage: FactLogStorage
|
||||||
|
private manifest: SegmentManifest = { version: 1, compactedBelow: 0, segments: [] }
|
||||||
|
/** Sidecar cache — segments are immutable, so entries never invalidate. */
|
||||||
|
private readonly sidecars = new Map<string, SidecarIndex>()
|
||||||
|
|
||||||
|
constructor(storage: FactLogStorage) {
|
||||||
|
this.storage = storage
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Load the manifest (ONE read — never a directory listing). */
|
||||||
|
async open(): Promise<void> {
|
||||||
|
const raw = (await this.storage.readRawObject(MANIFEST_PATH)) as SegmentManifest | null
|
||||||
|
if (raw) {
|
||||||
|
if (raw.version !== 1) {
|
||||||
|
throw new Error(
|
||||||
|
`[GenerationSegments] manifest version ${String(raw.version)} is newer than this ` +
|
||||||
|
`engine understands — refusing to serve partial history. Upgrade the engine.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
this.manifest = raw
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The packed tier's catalogue (ascending, immutable snapshot). */
|
||||||
|
segments(): readonly SegmentMeta[] {
|
||||||
|
return this.manifest.segments
|
||||||
|
}
|
||||||
|
|
||||||
|
/** D3's horizon marker: generations below this were reclaimed (bounded profiles only). */
|
||||||
|
compactedBelow(): number {
|
||||||
|
return this.manifest.compactedBelow
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The covering sealed segment for `gen`, or null if it lives outside the packed tier. */
|
||||||
|
private coveringSegment(gen: number): SegmentMeta | null {
|
||||||
|
// Manifest is ascending and ranges never overlap — binary search.
|
||||||
|
const segs = this.manifest.segments
|
||||||
|
let lo = 0
|
||||||
|
let hi = segs.length - 1
|
||||||
|
while (lo <= hi) {
|
||||||
|
const mid = (lo + hi) >> 1
|
||||||
|
const s = segs[mid]
|
||||||
|
if (gen < s.firstGeneration) hi = mid - 1
|
||||||
|
else if (gen > s.lastGeneration) lo = mid + 1
|
||||||
|
else return s
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when `gen` is packed (readable from this tier). */
|
||||||
|
hasGeneration(gen: number): boolean {
|
||||||
|
return this.coveringSegment(gen) !== null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fold consecutive generations into ONE new sealed segment + sidecar and
|
||||||
|
* append it to the manifest atomically. Caller guarantees: `gens` is
|
||||||
|
* ascending, contiguous with the packed tier (first = last packed + 1 when
|
||||||
|
* segments exist), and already durable in the live tier. Crash between the
|
||||||
|
* segment write and the caller's live-tier delete leaves a DUPLICATE
|
||||||
|
* representation — resolved live-tier-wins by the reader; never a gap.
|
||||||
|
*/
|
||||||
|
async fold(gens: FoldGeneration[]): Promise<SegmentMeta> {
|
||||||
|
if (gens.length === 0) {
|
||||||
|
throw new Error('[GenerationSegments] fold() requires at least one generation')
|
||||||
|
}
|
||||||
|
for (let i = 1; i < gens.length; i++) {
|
||||||
|
if (gens[i].generation <= gens[i - 1].generation) {
|
||||||
|
throw new Error('[GenerationSegments] fold() input must be strictly ascending')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const last = this.manifest.segments[this.manifest.segments.length - 1]
|
||||||
|
if (last && gens[0].generation <= last.lastGeneration) {
|
||||||
|
throw new Error(
|
||||||
|
`[GenerationSegments] fold() overlaps the packed tier: ${gens[0].generation} ≤ ` +
|
||||||
|
`sealed ${last.lastGeneration} — segments are immutable, never rewritten`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const first = gens[0].generation
|
||||||
|
const file = segmentFileName(first)
|
||||||
|
const sidecar: SidecarIndex = { version: 1, generations: [], ids: {} }
|
||||||
|
|
||||||
|
// Encode all frames, tracking offsets for the sidecar.
|
||||||
|
const parts: Uint8Array[] = [MAGIC]
|
||||||
|
let offset = MAGIC.length
|
||||||
|
for (const g of gens) {
|
||||||
|
const payload = msgpackEncode([
|
||||||
|
g.generation,
|
||||||
|
g.timestamp,
|
||||||
|
g.delta,
|
||||||
|
g.records.map((r) => [r.kind === 'noun' ? 0 : 1, r.id, r.record]),
|
||||||
|
0 // flags: v1 = uncompressed
|
||||||
|
])
|
||||||
|
const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length)
|
||||||
|
const view = new DataView(frame.buffer)
|
||||||
|
view.setUint32(0, payload.length, true)
|
||||||
|
view.setUint32(4, crc32c(payload), true)
|
||||||
|
frame.set(payload, FRAME_PREFIX_BYTES)
|
||||||
|
sidecar.generations.push([g.generation, offset, frame.length])
|
||||||
|
for (const r of g.records) {
|
||||||
|
const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}`
|
||||||
|
;(sidecar.ids[key] ??= []).push(g.generation)
|
||||||
|
}
|
||||||
|
parts.push(frame)
|
||||||
|
offset += frame.length
|
||||||
|
}
|
||||||
|
const total = parts.reduce((n, p) => n + p.length, 0)
|
||||||
|
const bytes = new Uint8Array(total)
|
||||||
|
let at = 0
|
||||||
|
for (const p of parts) {
|
||||||
|
bytes.set(p, at)
|
||||||
|
at += p.length
|
||||||
|
}
|
||||||
|
|
||||||
|
const meta: SegmentMeta = {
|
||||||
|
file,
|
||||||
|
firstGeneration: first,
|
||||||
|
lastGeneration: gens[gens.length - 1].generation,
|
||||||
|
frames: gens.length,
|
||||||
|
bytes: total,
|
||||||
|
checksum: crc32c(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Durability order: segment + sidecar fsync'd BEFORE the manifest names
|
||||||
|
// them (a crash before the manifest = invisible orphan files, harmless);
|
||||||
|
// manifest last, atomically.
|
||||||
|
const segPath = `${SEGMENTS_PREFIX}/${file}`
|
||||||
|
const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(first)}`
|
||||||
|
await this.storage.writeRawBytes(segPath, bytes)
|
||||||
|
await this.storage.writeRawBytes(idxPath, msgpackEncode(sidecar))
|
||||||
|
await this.storage.syncRawObjects([segPath, idxPath])
|
||||||
|
const next: SegmentManifest = {
|
||||||
|
...this.manifest,
|
||||||
|
segments: [...this.manifest.segments, meta]
|
||||||
|
}
|
||||||
|
await this.storage.writeRawObject(MANIFEST_PATH, next)
|
||||||
|
await this.storage.syncRawObjects([MANIFEST_PATH])
|
||||||
|
this.manifest = next
|
||||||
|
this.sidecars.set(file, sidecar)
|
||||||
|
return meta
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Load (or rebuild, loudly) a segment's sidecar. */
|
||||||
|
private async sidecarFor(meta: SegmentMeta): Promise<SidecarIndex> {
|
||||||
|
const cached = this.sidecars.get(meta.file)
|
||||||
|
if (cached) return cached
|
||||||
|
const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(meta.firstGeneration)}`
|
||||||
|
const raw = await this.storage.readRawBytes(idxPath)
|
||||||
|
if (raw) {
|
||||||
|
try {
|
||||||
|
const idx = msgpackDecode(raw) as SidecarIndex
|
||||||
|
if (idx.version === 1) {
|
||||||
|
this.sidecars.set(meta.file, idx)
|
||||||
|
return idx
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// fall through to rebuild
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Sidecars are DERIVED: rebuild from the segment, loudly — never serve
|
||||||
|
// wrong offsets silently.
|
||||||
|
prodLog.warn(
|
||||||
|
`[GenerationSegments] sidecar for ${meta.file} missing or unreadable — rebuilding from the segment`
|
||||||
|
)
|
||||||
|
const rebuilt = await this.rebuildSidecar(meta)
|
||||||
|
await this.storage.writeRawBytes(idxPath, msgpackEncode(rebuilt))
|
||||||
|
this.sidecars.set(meta.file, rebuilt)
|
||||||
|
return rebuilt
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One sequential read of the segment → a fresh sidecar. Verifies every frame CRC. */
|
||||||
|
private async rebuildSidecar(meta: SegmentMeta): Promise<SidecarIndex> {
|
||||||
|
const frames = await this.readAllFrames(meta)
|
||||||
|
const idx: SidecarIndex = { version: 1, generations: [], ids: {} }
|
||||||
|
for (const f of frames) {
|
||||||
|
idx.generations.push([f.generation, f.offset, f.frameLen])
|
||||||
|
for (const r of f.records) {
|
||||||
|
const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}`
|
||||||
|
;(idx.ids[key] ??= []).push(f.generation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return idx
|
||||||
|
}
|
||||||
|
|
||||||
|
private decodeFrame(
|
||||||
|
payload: Uint8Array
|
||||||
|
): { generation: number; timestamp: number; delta: unknown; records: FoldGeneration['records'] } {
|
||||||
|
const [generation, timestamp, delta, rawRecords] = msgpackDecode(payload) as [
|
||||||
|
number,
|
||||||
|
number,
|
||||||
|
unknown,
|
||||||
|
Array<[number, string, unknown]>,
|
||||||
|
number
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
generation,
|
||||||
|
timestamp,
|
||||||
|
delta,
|
||||||
|
records: rawRecords.map(([kindByte, id, record]) => ({
|
||||||
|
kind: kindByte === 0 ? ('noun' as const) : ('verb' as const),
|
||||||
|
id,
|
||||||
|
record
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async readAllFrames(meta: SegmentMeta): Promise<
|
||||||
|
Array<ReturnType<GenerationSegmentStore['decodeFrame']> & { offset: number; frameLen: number }>
|
||||||
|
> {
|
||||||
|
const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`)
|
||||||
|
if (!bytes) {
|
||||||
|
throw new Error(
|
||||||
|
`[GenerationSegments] sealed segment ${meta.file} is MISSING — packed history is damaged; ` +
|
||||||
|
`refusing to continue silently`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const out: Array<ReturnType<GenerationSegmentStore['decodeFrame']> & { offset: number; frameLen: number }> = []
|
||||||
|
let at = MAGIC.length
|
||||||
|
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
||||||
|
while (at + FRAME_PREFIX_BYTES <= bytes.length) {
|
||||||
|
const payloadLen = view.getUint32(at, true)
|
||||||
|
const crc = view.getUint32(at + 4, true)
|
||||||
|
const payload = bytes.subarray(at + FRAME_PREFIX_BYTES, at + FRAME_PREFIX_BYTES + payloadLen)
|
||||||
|
if (payload.length !== payloadLen || crc32c(payload) !== crc) {
|
||||||
|
throw new Error(
|
||||||
|
`[GenerationSegments] frame CRC mismatch in ${meta.file} at offset ${at} — ` +
|
||||||
|
`packed history is damaged; refusing to serve it`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
out.push({ ...this.decodeFrame(payload), offset: at, frameLen: FRAME_PREFIX_BYTES + payloadLen })
|
||||||
|
at += FRAME_PREFIX_BYTES + payloadLen
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read one packed generation's frame via its sidecar offset (one ranged read). */
|
||||||
|
private async readFrame(
|
||||||
|
gen: number
|
||||||
|
): Promise<ReturnType<GenerationSegmentStore['decodeFrame']> | null> {
|
||||||
|
const meta = this.coveringSegment(gen)
|
||||||
|
if (!meta) return null
|
||||||
|
const idx = await this.sidecarFor(meta)
|
||||||
|
// generations ascending → binary search.
|
||||||
|
const gens = idx.generations
|
||||||
|
let lo = 0
|
||||||
|
let hi = gens.length - 1
|
||||||
|
while (lo <= hi) {
|
||||||
|
const mid = (lo + hi) >> 1
|
||||||
|
if (gens[mid][0] < gen) lo = mid + 1
|
||||||
|
else if (gens[mid][0] > gen) hi = mid - 1
|
||||||
|
else {
|
||||||
|
const [, offset, frameLen] = gens[mid]
|
||||||
|
const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`)
|
||||||
|
if (!bytes) {
|
||||||
|
throw new Error(`[GenerationSegments] sealed segment ${meta.file} is MISSING`)
|
||||||
|
}
|
||||||
|
const frame = bytes.subarray(offset, offset + frameLen)
|
||||||
|
const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength)
|
||||||
|
const payloadLen = view.getUint32(0, true)
|
||||||
|
const crc = view.getUint32(4, true)
|
||||||
|
const payload = frame.subarray(FRAME_PREFIX_BYTES, FRAME_PREFIX_BYTES + payloadLen)
|
||||||
|
if (payload.length !== payloadLen || crc32c(payload) !== crc) {
|
||||||
|
throw new Error(
|
||||||
|
`[GenerationSegments] frame CRC mismatch for generation ${gen} in ${meta.file} — ` +
|
||||||
|
`packed history is damaged; refusing to serve it`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return this.decodeFrame(payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// In the covering range but not present: the packed tier is dense by
|
||||||
|
// construction (fold packs every generation it is handed, including
|
||||||
|
// record-less ones) — absence inside a sealed range is damage.
|
||||||
|
throw new Error(
|
||||||
|
`[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
|
||||||
|
`range but has no frame — packed history is damaged`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The packed tier's delta for `gen` (null = not packed). */
|
||||||
|
async readDelta(gen: number): Promise<{ delta: unknown; timestamp: number } | null> {
|
||||||
|
const frame = await this.readFrame(gen)
|
||||||
|
return frame ? { delta: frame.delta, timestamp: frame.timestamp } : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The packed tier's full record-set for `gen` (null = not packed). */
|
||||||
|
async readRecords(gen: number): Promise<FoldGeneration['records'] | null> {
|
||||||
|
const frame = await this.readFrame(gen)
|
||||||
|
return frame ? frame.records : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One packed before-image (null = not packed OR no record for the id in that generation). */
|
||||||
|
async readRecord(gen: number, kind: 'noun' | 'verb', id: string): Promise<unknown | null> {
|
||||||
|
const frame = await this.readFrame(gen)
|
||||||
|
if (!frame) return null
|
||||||
|
const hit = frame.records.find((r) => r.kind === kind && r.id === id)
|
||||||
|
return hit ? hit.record : null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* D3 reclaim: drop WHOLE segments whose lastGeneration < `belowGeneration`
|
||||||
|
* and bump `compactedBelow`. Partial segments are never dropped — the
|
||||||
|
* boundary waits. NEVER called under the archival profile (the caller
|
||||||
|
* enforces retention semantics; this method only executes boundary drops).
|
||||||
|
*/
|
||||||
|
async dropSegmentsBelow(belowGeneration: number): Promise<{ dropped: number; compactedBelow: number }> {
|
||||||
|
const keep: SegmentMeta[] = []
|
||||||
|
const drop: SegmentMeta[] = []
|
||||||
|
for (const s of this.manifest.segments) {
|
||||||
|
;(s.lastGeneration < belowGeneration ? drop : keep).push(s)
|
||||||
|
}
|
||||||
|
if (drop.length === 0) {
|
||||||
|
return { dropped: 0, compactedBelow: this.manifest.compactedBelow }
|
||||||
|
}
|
||||||
|
const compactedBelow = Math.max(
|
||||||
|
this.manifest.compactedBelow,
|
||||||
|
drop[drop.length - 1].lastGeneration + 1
|
||||||
|
)
|
||||||
|
// Manifest first (the drop is authoritative once named), then bytes —
|
||||||
|
// a crash between leaves orphan segment files invisible to the manifest,
|
||||||
|
// harmless and re-collectable.
|
||||||
|
const next: SegmentManifest = { ...this.manifest, compactedBelow, segments: keep }
|
||||||
|
await this.storage.writeRawObject(MANIFEST_PATH, next)
|
||||||
|
await this.storage.syncRawObjects([MANIFEST_PATH])
|
||||||
|
this.manifest = next
|
||||||
|
for (const s of drop) {
|
||||||
|
await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${s.file}`)
|
||||||
|
await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${sidecarFileName(s.firstGeneration)}`)
|
||||||
|
this.sidecars.delete(s.file)
|
||||||
|
}
|
||||||
|
return { dropped: drop.length, compactedBelow }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* D8 rider — the packed portion of `generationDigest(g)`: a deterministic
|
||||||
|
* crc32c chain over sealed-segment checksums fully below `g`, plus the
|
||||||
|
* frame CRC of `g`'s own frame when `g` is mid-segment. O(segments), not
|
||||||
|
* O(generations); identical history ⇒ identical digest on any machine.
|
||||||
|
* The live-tier portion is composed by the caller.
|
||||||
|
*/
|
||||||
|
async digestThroughPacked(g: number): Promise<number | null> {
|
||||||
|
let digest = 0
|
||||||
|
let covered = false
|
||||||
|
for (const s of this.manifest.segments) {
|
||||||
|
if (s.lastGeneration <= g) {
|
||||||
|
digest = crc32c(new TextEncoder().encode(`${digest}:${s.checksum}`))
|
||||||
|
if (s.lastGeneration === g) covered = true
|
||||||
|
} else if (s.firstGeneration <= g) {
|
||||||
|
// g is mid-segment: chain the partial prefix via g's frame CRC.
|
||||||
|
const frame = await this.readFrame(g)
|
||||||
|
if (frame === null) return null
|
||||||
|
const idx = await this.sidecarFor(s)
|
||||||
|
const upTo = idx.generations.filter(([gen]) => gen <= g)
|
||||||
|
for (const [gen, offset, frameLen] of upTo) {
|
||||||
|
digest = crc32c(new TextEncoder().encode(`${digest}:${gen}:${offset}:${frameLen}`))
|
||||||
|
}
|
||||||
|
covered = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return covered || this.manifest.segments.length > 0 ? digest : null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -133,6 +133,11 @@ export class MemoryStorage extends BaseStorage {
|
||||||
*/
|
*/
|
||||||
protected async deleteObjectFromPath(path: string): Promise<void> {
|
protected async deleteObjectFromPath(path: string): Promise<void> {
|
||||||
this.objectStore.delete(path)
|
this.objectStore.delete(path)
|
||||||
|
// Filesystem parity: on disk, objects and raw BYTE files are both just
|
||||||
|
// files — unlink removes whichever exists. Without this, deleteRawObject
|
||||||
|
// on a raw-bytes path (fact-log/generation segments) silently no-ops on
|
||||||
|
// memory storage: the delete "succeeds" and the bytes remain.
|
||||||
|
this.rawBytesStore.delete(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
150
tests/unit/db/generation-segments.test.ts
Normal file
150
tests/unit/db/generation-segments.test.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
||||||
|
/**
|
||||||
|
* @module tests/unit/db/generation-segments
|
||||||
|
* @description The generation-segment store (Stage-2 D1+D3 file format).
|
||||||
|
* Laws: (1) fold → read round-trips deltas and records byte-faithfully via
|
||||||
|
* sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen
|
||||||
|
* reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from
|
||||||
|
* its segment loudly, a damaged SEGMENT fails loudly (never silent wrong
|
||||||
|
* data); (4) D3 reclaim drops whole segments only and bumps compactedBelow;
|
||||||
|
* (5) the packed digest is deterministic across reopen; (6) immutability —
|
||||||
|
* fold refuses overlap with sealed ranges.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach } from 'vitest'
|
||||||
|
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||||
|
import {
|
||||||
|
GenerationSegmentStore,
|
||||||
|
SEGMENTS_PREFIX,
|
||||||
|
type FoldGeneration
|
||||||
|
} from '../../../src/db/generationSegments.js'
|
||||||
|
|
||||||
|
const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
|
||||||
|
|
||||||
|
const gen = (g: number, recordCount = 2): FoldGeneration => ({
|
||||||
|
generation: g,
|
||||||
|
timestamp: 1_700_000_000_000 + g,
|
||||||
|
delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g },
|
||||||
|
records: Array.from({ length: recordCount }, (_, i) => ({
|
||||||
|
kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb',
|
||||||
|
id: UUID(g * 100 + i),
|
||||||
|
record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } }
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => {
|
||||||
|
let storage: MemoryStorage
|
||||||
|
let store: GenerationSegmentStore
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
storage = new MemoryStorage()
|
||||||
|
await storage.init()
|
||||||
|
store = new GenerationSegmentStore(storage as any)
|
||||||
|
await store.open()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('fold → read round-trips deltas and records via sidecar point-reads', async () => {
|
||||||
|
const meta = await store.fold([gen(1), gen(2), gen(3)])
|
||||||
|
expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 })
|
||||||
|
expect(meta.checksum).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
expect(store.hasGeneration(2)).toBe(true)
|
||||||
|
expect(store.hasGeneration(4)).toBe(false)
|
||||||
|
|
||||||
|
const d2 = await store.readDelta(2)
|
||||||
|
expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 })
|
||||||
|
expect(d2?.timestamp).toBe(1_700_000_000_002)
|
||||||
|
|
||||||
|
const records = await store.readRecords(3)
|
||||||
|
expect(records).toHaveLength(2)
|
||||||
|
expect(records![0]).toEqual({
|
||||||
|
kind: 'noun',
|
||||||
|
id: UUID(300),
|
||||||
|
record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } }
|
||||||
|
})
|
||||||
|
// Point read by id, both kinds.
|
||||||
|
expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({
|
||||||
|
metadata: { noun: 'document', v: 3 },
|
||||||
|
vector: { v: [3, 1] }
|
||||||
|
})
|
||||||
|
expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reopen discovers everything from the manifest alone — no listing', async () => {
|
||||||
|
await store.fold([gen(1), gen(2)])
|
||||||
|
await store.fold([gen(3), gen(4)])
|
||||||
|
|
||||||
|
const reopened = new GenerationSegmentStore(storage as any)
|
||||||
|
await reopened.open()
|
||||||
|
expect(reopened.segments()).toHaveLength(2)
|
||||||
|
expect(reopened.hasGeneration(4)).toBe(true)
|
||||||
|
expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => {
|
||||||
|
const meta = await store.fold([gen(1), gen(2)])
|
||||||
|
const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx`
|
||||||
|
await storage.deleteRawObject(idxPath)
|
||||||
|
|
||||||
|
const reopened = new GenerationSegmentStore(storage as any)
|
||||||
|
await reopened.open()
|
||||||
|
// Rebuild path: still serves correct data.
|
||||||
|
expect((await reopened.readRecords(2))!).toHaveLength(2)
|
||||||
|
|
||||||
|
// Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud.
|
||||||
|
const segPath = `${SEGMENTS_PREFIX}/${meta.file}`
|
||||||
|
const bytes = (await storage.readRawBytes(segPath))!
|
||||||
|
bytes[bytes.length - 3] ^= 0xff
|
||||||
|
await storage.writeRawBytes(segPath, bytes)
|
||||||
|
const damaged = new GenerationSegmentStore(storage as any)
|
||||||
|
await damaged.open()
|
||||||
|
;(damaged as any).sidecars.clear()
|
||||||
|
await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes
|
||||||
|
await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => {
|
||||||
|
await store.fold([gen(1), gen(2)])
|
||||||
|
await store.fold([gen(3), gen(4)])
|
||||||
|
await store.fold([gen(5), gen(6)])
|
||||||
|
|
||||||
|
// Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops.
|
||||||
|
const r1 = await store.dropSegmentsBelow(4)
|
||||||
|
expect(r1).toEqual({ dropped: 1, compactedBelow: 3 })
|
||||||
|
expect(store.hasGeneration(1)).toBe(false)
|
||||||
|
expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole
|
||||||
|
|
||||||
|
// Bytes actually gone.
|
||||||
|
expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull()
|
||||||
|
|
||||||
|
// Horizon past everything: the rest drop; compactedBelow is durable.
|
||||||
|
const r2 = await store.dropSegmentsBelow(7)
|
||||||
|
expect(r2.dropped).toBe(2)
|
||||||
|
const reopened = new GenerationSegmentStore(storage as any)
|
||||||
|
await reopened.open()
|
||||||
|
expect(reopened.compactedBelow()).toBe(7)
|
||||||
|
expect(reopened.segments()).toHaveLength(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the packed digest is deterministic across reopen and changes with history', async () => {
|
||||||
|
await store.fold([gen(1), gen(2), gen(3)])
|
||||||
|
const atSeal = await store.digestThroughPacked(3)
|
||||||
|
const midSegment = await store.digestThroughPacked(2)
|
||||||
|
expect(atSeal).not.toBeNull()
|
||||||
|
expect(midSegment).not.toBeNull()
|
||||||
|
expect(midSegment).not.toBe(atSeal)
|
||||||
|
|
||||||
|
const reopened = new GenerationSegmentStore(storage as any)
|
||||||
|
await reopened.open()
|
||||||
|
expect(await reopened.digestThroughPacked(3)).toBe(atSeal)
|
||||||
|
expect(await reopened.digestThroughPacked(2)).toBe(midSegment)
|
||||||
|
|
||||||
|
await reopened.fold([gen(4)])
|
||||||
|
expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => {
|
||||||
|
await store.fold([gen(1), gen(2)])
|
||||||
|
await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/)
|
||||||
|
await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/)
|
||||||
|
await expect(store.fold([])).rejects.toThrow(/at least one generation/)
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue