soulcraftlabs/open-brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 340 Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/reclaim/packed-history-density'

Browse source
This commit is contained in:
David Snelling 2026-08-31 09:30:08 -07:00
commit 39c71ecdac
4 changed files with 389 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

119
src/db/generationSegments.ts
View file

@ -147,6 +147,60 @@ export class GenerationSegmentStore {
return this.coveringSegment(gen) !== null
}
/**
* @description True when `meta` declares more generations than it holds
* frames — a segment sealed by a writer that folded across a hole. The
* manifest records `frames` at fold time, so this is an O(1) comparison
* against the declared span and needs no I/O.
*/
private isSparse(meta: SegmentMeta): boolean {
return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames
}
/**
* @description The generations this tier ACTUALLY holds, as coalesced
* ascending intervals — not what the segments declare.
*
* Dense segments (every one a current writer produces) contribute their
* declared range with no I/O. A SPARSE segment — one sealed before the
* density law was enforced, whose declared range spans generations it has
* no frame for — has its real generation list read from its sidecar and
* contributed instead, with the discrepancy narrated once.
*
* This is what keeps a store that already carries the damage from wedging.
* `open()` seeds `committedRanges` from these intervals, so a hole is never
* re-admitted as a committed generation, and the auto-compaction pass that
* used to fail on every run with "packed history is damaged" simply never
* asks for the missing frame.
*
* @returns Ascending, non-overlapping `[first, last]` intervals.
*/
async actualRanges(): Promise<Array<[number, number]>> {
const out: Array<[number, number]> = []
for (const meta of this.manifest.segments) {
if (!this.isSparse(meta)) {
out.push([meta.firstGeneration, meta.lastGeneration])
continue
}
const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames
prodLog.warn(
`[GenerationSegments] sealed segment ${meta.file} declares generations ` +
`${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` +
`frame(s) — ${missing} generation(s) in that span were never folded into it. ` +
`Serving the frames it actually holds; the declared span is not treated as ` +
`committed history. (Written by a pre-density-law writer that folded across a ` +
`gap; the segment itself is intact and no record is lost.)`
)
const idx = await this.sidecarFor(meta)
for (const [gen] of idx.generations) {
const last = out[out.length - 1]
if (last !== undefined && gen === last[1] + 1) last[1] = gen
else out.push([gen, gen])
}
}
return out
}
/**
* Fold consecutive generations into ONE new sealed segment + sidecar and
* append it to the manifest atomically. Caller guarantees: `gens` is
@ -164,6 +218,38 @@ export class GenerationSegmentStore {
throw new Error('[GenerationSegments] fold() input must be strictly ascending')
}
}
// THE DENSITY LAW, MADE MECHANICAL.
//
// A sealed segment declares a CONTIGUOUS range [firstGeneration,
// lastGeneration] and every reader treats that range as containment:
// `coveringSegment` is an interval test, `hasGeneration` returns true for
// anything inside it, and `open()` seeds committedRanges from it. So a
// segment folded from a SPARSE input silently claims generations it does
// not hold, and the first read of one of those holes throws
// "inside sealed segment ... but has no frame — packed history is damaged".
//
// That is exactly how the damage was produced. `repackHistory` skipped
// generations mid-batch — ones absent from committedRanges, ones still in
// the pending buffer, ones whose tx.json would not read — and handed the
// survivors here, where the range was computed from the first and last of
// them. Worse, the mis-declared range was then merged back into
// committedRanges at the next open, which is what turned a quiet hole into
// a repeating auto-compaction failure on every subsequent run.
//
// Callers now split at discontinuities; this refusal is what keeps any
// future caller from reintroducing the class. A refusal here loses
// nothing — the generations stay in the live tier, readable, and the next
// pass folds them correctly.
for (let i = 1; i < gens.length; i++) {
if (gens[i].generation !== gens[i - 1].generation + 1) {
throw new Error(
`[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` +
`${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` +
`generation(s). A sealed segment declares a dense range, so folding a sparse ` +
`batch would claim generations it does not hold. Split the batch at the gap.`
)
}
}
const last = this.manifest.segments[this.manifest.segments.length - 1]
if (last && gens[0].generation <= last.lastGeneration) {
throw new Error(
@ -364,12 +450,37 @@ export class GenerationSegmentStore {
return this.decodeFrame(payload)
}
}
// In the covering range but not present: the packed tier is dense by
// construction (fold packs every generation it is handed, including
// record-less ones) — absence inside a sealed range is damage.
// Inside the covering range but with no frame. Two very different causes,
// and conflating them is what made this class wedge every maintenance pass
// on the affected stores.
//
// (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than
// the span it declares. That segment was sealed by a writer that
// folded across a hole (the class this file's density law now bars).
// The segment is INTACT and nothing is lost; it simply never held this
// generation. Answering "not packed" is the honest answer, and it lets
// the caller's two-tier read decide what a genuinely absent generation
// means, instead of every compaction pass dying on a repeating throw.
// `actualRanges()` keeps such holes out of committedRanges at open, so
// in a healed store nobody asks this question in the first place.
//
// (2) A DENSE SEGMENT missing a frame it says it has — the manifest and
// the sidecar disagree about a segment that claims to be complete.
// That IS damage, and it stays loud.
if (this.isSparse(meta)) {
prodLog.warn(
`[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` +
`declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` +
`holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` +
`-generation span — it was sealed across a gap and never held this generation. ` +
`Reporting it as unpacked rather than as damage; no record is lost.`
)
return null
}
throw new Error(
`[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` +
`range but has no frame — packed history is damaged`
`range but has no frame, and that segment declares a complete ${meta.frames}-frame ` +
`span — the manifest and the sidecar disagree; packed history is damaged`
)
}

66
src/db/generationStore.ts
View file

@ -96,6 +96,35 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json'
/** Storage-root-relative prefix of the per-generation record directories. */
export const GENERATIONS_PREFIX = '_generations'
/**
* @description Split an ascending list of fold candidates into maximal
* CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`.
*
* A sealed segment declares one dense range `[firstGeneration,
* lastGeneration]`, and every reader treats that range as containment. So a
* batch with a hole in it must never become one segment: it would claim a
* generation it does not hold, and the first read of that hole reports the
* packed history as damaged. One run, one segment — the ranges then describe
* exactly what the segments contain.
*
* @param gens - Fold candidates, strictly ascending by generation.
* @returns One array per contiguous run, in ascending order. Empty in, empty out.
*/
export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] {
const runs: FoldGeneration[][] = []
let run: FoldGeneration[] = []
for (const g of gens) {
const prev = run[run.length - 1]
if (prev !== undefined && g.generation !== prev.generation + 1) {
runs.push(run)
run = []
}
run.push(g)
}
if (run.length > 0) runs.push(run)
return runs
}
/**
* @description Phases of the {@link GenerationStore.commitTransaction} commit
* protocol at which a test-only fault injector can simulate a process crash.
@ -784,9 +813,15 @@ export class GenerationStore {
if (storageSupportsFactLog(this.storage)) {
this.segments = new GenerationSegmentStore(this.storage)
await this.segments.open()
const packedRanges = this.segments
.segments()
.map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
// ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law
// writer can declare a span wider than the frames it holds; seeding
// committedRanges from the declared span re-admits those holes as
// committed generations, and every later maintenance pass then asks for a
// frame that was never written. `actualRanges()` reads the real
// generation list from the sidecar for exactly those segments (and does
// no I/O for the dense ones, which is all of them on a healthy store).
const packedRanges = (await this.segments.actualRanges())
.map((r): [number, number] => [r[0], Math.min(r[1], this.committed)])
.filter(([lo, hi]) => lo <= hi)
if (packedRanges.length > 0) {
// Merge packed (older) + live (newer) interval sets — both ascending;
@ -3121,13 +3156,26 @@ export class GenerationStore {
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
}
if (foldInput.length === 0) continue
await segments.fold(foldInput)
segmentsCreated++
// Segment + manifest durable → the live copies retire.
for (const g of foldInput) {
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
// SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three
// filters above punch holes in it: a generation missing from
// committedRanges never appears, one still in the pending buffer is
// skipped, and one whose tx.json will not read is skipped. A sealed
// segment declares a DENSE range, so folding across such a hole makes
// the segment claim a generation it does not hold; the next open
// merges that mis-declared range into committedRanges, and every
// subsequent auto-compaction pass then asks for the missing frame and
// fails with "packed history is damaged". Fold each contiguous RUN as
// its own segment instead — same bytes, honest ranges.
for (const run of contiguousRuns(foldInput)) {
if (deadline !== undefined && Date.now() >= deadline) break
await segments.fold(run)
segmentsCreated++
// Segment + manifest durable → the live copies retire.
for (const g of run) {
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
}
folded += run.length
}
folded += foldInput.length
}
if (folded > 0) {
prodLog.info(

102
tests/integration/history-repacking.test.ts
View file

@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as path from 'node:path'
import * as os from 'node:os'
import * as zlib from 'node:zlib'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import { GenerationStore } from '../../src/db/generationStore.js'
@ -57,6 +58,107 @@ describe('history repacking — the two-tier lifecycle', () => {
}
})
/**
* THE HOLE, END TO END — the shape a real store carries.
*
* A forensic fixture was measured with generation directories 1..2503
* present except for exactly one: 1416. Its fact-log segment already showed
* the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration
* 1940 (531 generations) while recording only 530 facts.
*
* Before the fix, repacking such a store folded ACROSS that hole: the batch
* skipped 1416 (no readable delta) and the sealed segment declared a range
* spanning it anyway. The next open merged that declared range back into
* committedRanges, re-admitting 1416 as committed history, and every
* subsequent auto-compaction pass then asked the packed tier for a frame
* that was never written — producing, on EVERY run, the non-fatal narration
*
* Auto-compaction of generational history failed (non-fatal): generation
* N is inside sealed segment seg-....bgs's declared range but has no frame
* — packed history is damaged
*
* This pin removes a generation directory to make the same hole, then
* requires repack + reopen + compaction to complete cleanly.
*/
it('a missing generation directory does not poison the packed tier', async () => {
const dir = tempDir()
// `retention: 'all'` throughout: close() otherwise auto-compacts the
// history away, and this pin needs the cold generations still on disk so
// there is something to punch a hole in. The live window stays at its
// production default for the build phase, so nothing folds yet.
const archival = async (): Promise<Brainy> => {
const b = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
embeddingFunction: stub,
retention: 'all'
})
await b.init()
return b
}
const brain = await archival()
const id = await brain.add({
data: 'holed-entity',
type: NounType.Document,
metadata: { v: 0 }
})
// One flush per update: single-op writes coalesce inside a flush window,
// so a history deep enough to have a middle needs the windows separated.
for (let v = 1; v <= 12; v++) {
await brain.update({ id, metadata: { v } })
await brain.flush()
}
await brain.close()
// Punch the hole: delete ONE generation directory in the middle of the
// cold range, exactly as the real store presents it.
const genRoot = path.join(dir, '_generations')
const numeric = fs
.readdirSync(genRoot, { withFileTypes: true })
.filter((e) => e.isDirectory() && /^\d+$/.test(e.name))
.map((e) => Number(e.name))
.sort((a, b) => a - b)
expect(numeric.length).toBeGreaterThan(6)
const victim = numeric[Math.floor(numeric.length / 2)]
fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true })
// Now shrink the live window and reopen. close() repacks automatically
// (brainy.ts phase 0b), so this is the production sequence exactly: a
// store with a hole in its history gets folded by ordinary housekeeping,
// with nobody asking for it.
;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
const reopened = await archival()
const result = await reopened.repackHistory()
expect(result.foldedGenerations).toBeGreaterThan(0)
const segDir = path.join(dir, SEGMENTS_PREFIX)
const manifestPath = ['manifest.json', 'manifest.json.gz']
.map((f) => path.join(segDir, f))
.find((p) => fs.existsSync(p))!
const raw = manifestPath.endsWith('.gz')
? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8')
: fs.readFileSync(manifestPath, 'utf8')
const manifest = JSON.parse(raw) as {
segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }>
}
// THE LAW: every sealed segment declares exactly as many generations as it
// holds frames, and none of them spans the victim.
for (const s of manifest.segments) {
expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames)
expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false)
}
await reopened.close()
// And the pass that used to fail on every run now completes: reopen (which
// re-seeds committedRanges from the packed tier) then compact history.
const third = await openBrain(dir)
await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined()
await third.close()
})
it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => {
;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
const dir = tempDir()

115
tests/unit/db/generation-segments.test.ts
View file

@ -147,4 +147,119 @@ describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => {
await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/)
await expect(store.fold([])).rejects.toThrow(/at least one generation/)
})
// ==========================================================================
// THE DENSITY LAW
// ==========================================================================
//
// A sealed segment declares a CONTIGUOUS range and every reader treats that
// range as containment. Folding a sparse batch therefore makes the segment
// claim generations it does not hold — and because `open()` merges declared
// ranges back into committedRanges, the hole is re-admitted as committed
// history and every later maintenance pass fails asking for a frame that was
// never written. That is the "generation N is inside sealed segment
// seg-....bgs's declared range but has no frame — packed history is damaged"
// narration seen on every run of the affected stores.
it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => {
await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow(
/not contiguous: 2 → 4 skips 1 generation/
)
// The refusal loses nothing: no segment was sealed, so the generations
// stay in the live tier and the next pass folds them correctly.
expect(store.segments()).toHaveLength(0)
expect(store.hasGeneration(1)).toBe(false)
})
it('a wider gap names how many generations it would have swallowed', async () => {
await expect(store.fold([gen(10), gen(20)])).rejects.toThrow(
/not contiguous: 10 → 20 skips 9 generation\(s\)/
)
})
it('two contiguous runs folded separately declare honest ranges', async () => {
// What the caller now does instead of folding across the gap.
const a = await store.fold([gen(1), gen(2), gen(3)])
const b = await store.fold([gen(7), gen(8)])
expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 })
expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 })
// The gap is honestly outside the packed tier.
for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false)
for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true)
expect(await store.actualRanges()).toEqual([
[1, 3],
[7, 8]
])
})
it('actualRanges() is exact and I/O-free for dense segments', async () => {
await store.fold([gen(1), gen(2)])
await store.fold([gen(3), gen(4)])
// Adjacent dense segments each contribute their declared range.
expect(await store.actualRanges()).toEqual([
[1, 2],
[3, 4]
])
})
// ---- pre-existing damage: a store sealed by the old writer ----------------
/**
* Seal a SPARSE segment the way the pre-fix writer did: write the bytes and
* sidecar for a contiguous run, then rewrite the manifest so the segment
* declares a wider range than the frames it holds. This reproduces on disk
* exactly what the affected stores carry, without needing the old code.
*/
const sealSparseSegment = async (): Promise<void> => {
await store.fold([gen(1), gen(2), gen(3)])
const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any
// Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become
// holes inside a sealed range.
manifest.segments[0].lastGeneration = 5
await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest)
}
it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => {
await sealSparseSegment()
const reopened = new GenerationSegmentStore(storage as any)
await reopened.open()
// The frames it really holds still serve, byte-faithfully.
expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002)
expect(await reopened.readRecords(3)).toHaveLength(2)
// The holes answer "not packed" instead of throwing. This is the fix for
// the wedge: the old reader threw here on EVERY maintenance pass.
expect(await reopened.readDelta(4)).toBeNull()
expect(await reopened.readRecords(5)).toBeNull()
})
it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => {
await sealSparseSegment()
const reopened = new GenerationSegmentStore(storage as any)
await reopened.open()
// Declared 1..5; actually holds 1..3. The store seeds committedRanges from
// THIS, so generations 4 and 5 never become committed history again.
expect(await reopened.actualRanges()).toEqual([[1, 3]])
})
it('a DENSE segment missing a frame is still loud damage', async () => {
// The other side of the branch: when the manifest claims a complete span,
// a missing frame means the manifest and sidecar disagree — real damage,
// and it must not be quietly downgraded to "unpacked".
await store.fold([gen(1), gen(2), gen(3)])
const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx`
const raw = (await storage.readRawBytes(idxPath))!
const { decode, encode } = await import('@msgpack/msgpack')
const idx = decode(raw) as any
// Drop generation 2's entry while the manifest still declares 3 frames.
idx.generations = idx.generations.filter(([g]: [number]) => g !== 2)
await storage.writeRawBytes(idxPath, encode(idx))
const reopened = new GenerationSegmentStore(storage as any)
await reopened.open()
await expect(reopened.readDelta(2)).rejects.toThrow(
/manifest and the sidecar disagree; packed history is damaged/
)
})
})