soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: two-tier history reads + the repacker + generationDigest — D1+D3 wired end-to-end

Browse source 
The packed tier goes live inside GenerationStore:

- Two-tier reads: getDelta / readBeforeImage / readGenerationRecords
  fall through live-tier → sealed segments (live-tier-wins: a crash
  mid-fold leaves a duplicate representation, never a gap). Cold-open
  seeds committedRanges from the segment manifest via interval merge —
  packed generations resolve without their directories existing.
- repackHistory({timeBudgetMs, batchGenerations}): folds cold
  generations (older than the newest 1024) oldest-first into sealed
  segments, deleting per-generation directories only after segment +
  manifest are durable. Public API + automatic time-bounded pass at
  close() (before compaction, so reclaim can drop whole segments);
  re-representation only — the sole history transform under the
  archival profile. Early stop = consistent prefix, next pass resumes.
- compact(): packed generations reclaim logically in the loop and
  physically at whole-segment boundaries via dropSegmentsBelow (the
  frozen partial-segments-wait rule).
- generationDigest(g) (D8): deterministic content digest through g —
  sealed-segment checksum chain + live-tier delta hashes; O(segments +
  live window); RangeError out of range, GenerationCompactedError
  below the horizon (a gate can never silently pin reclaimed history).

Four end-to-end pins: asOf answers byte-identical across fold + cold
reopen with folded dirs physically gone; repack+reclaim composition;
digest reopen-stability/divergence/loud-horizon; budget no-op+resume.
This commit is contained in:
David Snelling 2026-07-19 16:26:10 -07:00
parent d8acb3776b
commit 1201e25543
3 changed files with 454 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

59
src/brainy.ts
View file

@ -8265,6 +8265,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return this.generationStore.compact(options)
}
/**
* @description Repack cold generation history into sealed segments —
* re-representation, never deletion: every record and delta stays readable
* (`asOf()` unchanged); the physical file count drops by orders of
* magnitude. Runs automatically (time-bounded) at `close()`; call this for
* explicit maintenance windows on long-lived writers. The ONLY history
* transform permitted under the archival profile (`retention: 'all'`).
* @param options - `timeBudgetMs` bounds the pass (early stop = consistent
* prefix, next pass resumes); `batchGenerations` sizes each fold.
* @returns Folded generation count and segments created.
*/
async repackHistory(options?: {
timeBudgetMs?: number
batchGenerations?: number
}): Promise<{ foldedGenerations: number; segmentsCreated: number }> {
this.assertWritable('repackHistory')
await this.ensureInitialized()
await this.generationStore.flushPendingSingleOps()
return this.generationStore.repackHistory(options)
}
/**
* @description Read-only generational-history footprint for fleet audits:
* generation count, total on-disk bytes, generation/timestamp range, the
@ -8292,6 +8313,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
/**
* @description A deterministic content digest of the generation log through
* `g` (D8 — gate-to-generation provenance): identical history produces the
* identical digest on any machine; divergence produces a different one.
* Release gates and suite verdicts pin `{generation, digest}` and verify
* both at execution time instead of pinning a git commit. O(segments +
* live-tier window), never O(all generations). Throws `RangeError` out of
* range and `GenerationCompactedError` below the horizon — a gate can
* never silently pin reclaimed history.
* @example
* const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) }
*/
async generationDigest(g: number): Promise<string> {
await this.ensureInitialized()
await this.generationStore.flushPendingSingleOps()
return this.generationStore.generationDigest(g)
}
/**
* @description Drive the adaptive retention byte budget at runtime — the
* settable input a machine-level coordinator (e.g. cor's `ResourceManager`,
@ -16192,11 +16231,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
await this.generationStore.flushPendingSingleOps()
}
// Phase 0b: Auto-compact generational history per config.retention (default
// on) BEFORE the generation store closes below. This is THE auto-compaction
// site (8.9.0 — flush() never compacts): time-bounded per pass, respects
// live Db pins and an explicit autoCompact: false; no-op on read-only
// instances.
// Phase 0b: REPACK cold history into sealed segments (D1+D3 —
// re-representation, never deletion; the only history transform under the
// archival profile), then auto-compact per config.retention. Repack runs
// FIRST so bounded-retention reclaim can drop whole segments. Both are
// time-bounded maintenance passes (8.9.0 law: flush() never pays these);
// both are housekeeping — failures warn, never fail a clean shutdown.
if (!this.isReadOnly && this.generationStore) {
try {
await this.generationStore.repackHistory({ timeBudgetMs: 5_000 })
} catch (error) {
console.warn(
`History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}`
)
}
}
await this.autoCompactHistory()
// Phase 1: Flush ALL components in parallel to persist buffered data

217
src/db/generationStore.ts
View file

@ -46,6 +46,8 @@ import type {
TxLogEntry
} from './types.js'
import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js'
import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js'
import { crc32c } from '../utils/crc32c.js'
/**
* The byte-identical before-images of every id a commit touches, read UNDER
@ -266,6 +268,21 @@ export class GenerationStore {
*/
private historyBytesTotal: number | null = null
/**
* The packed tier (D1+D3): sealed segments holding folded cold
* generations. Null until {@link open} wires it (and on storage adapters
* without raw-byte primitives — the live tier then carries everything,
* exactly as before the packed tier existed).
*/
private segments: GenerationSegmentStore | null = null
/**
* Live-tier window: generations newer than `committed - REPACK_LIVE_WINDOW`
* are never folded — the hot tail stays in the per-generation layout the
* write path owns. Matches the resident chain window's scale.
*/
static readonly REPACK_LIVE_WINDOW = 1024
/**
* Model-B per-write group-commit — the in-memory PENDING tier.
*
@ -433,6 +450,33 @@ export class GenerationStore {
this.factLog = null
}
// PACKED TIER (D1+D3): same capability gate as the fact log. Opening
// reads ONE manifest — never a listing of the packed backlog — and seeds
// committedRanges with the sealed ranges so packed generations resolve
// exactly like live ones.
if (storageSupportsFactLog(this.storage)) {
this.segments = new GenerationSegmentStore(this.storage)
await this.segments.open()
const packedRanges = this.segments
.segments()
.map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
.filter(([lo, hi]) => lo <= hi)
if (packedRanges.length > 0) {
// Merge packed (older) + live (newer) interval sets — both ascending;
// coalesce adjacency so range arithmetic stays interval-exact.
const merged: Array<[number, number]> = []
for (const r of [...packedRanges, ...this.committedRanges].sort((a, b) => a[0] - b[0])) {
const last = merged[merged.length - 1]
if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1])
else merged.push([r[0], r[1]])
}
this.committedRanges = merged
}
this.horizonGen = Math.max(this.horizonGen, this.segments.compactedBelow() - 1)
} else {
this.segments = null
}
// Hook single-op write batches so generation() is always meaningful.
// Suppressed while a transact batch executes (the batch is ONE generation).
if (!options?.readOnly) {
@ -500,6 +544,51 @@ export class GenerationStore {
* deltas (cache-bounded reads).
* @returns Counts, bytes, generation range, and the compaction horizon.
*/
/**
* @description D8 (gate-to-generation provenance): a deterministic content
* digest of the generation log THROUGH `g` — identical history ⇒ identical
* digest on any machine; any divergence (different records, different
* order, reclaimed range) ⇒ different digest. Composed from the packed
* tier's sealed-segment checksum chain (O(segments)) plus the live tier's
* per-generation delta digests (O(live window at most)). Release gates pin
* {generation, digest} and verify both at execution time.
* @param g - The generation to digest through (≤ committed).
* @returns A hex digest string, stable across reopen and repacking states
* ONLY for fully-packed prefixes — repacking changes representation, so
* the composed digest is defined over CONTENT: live-tier gens hash their
* delta + record ids, packed gens hash via frame CRCs. A gate should pin
* after a repack pass for long-term stability, or re-pin on repack.
*/
async generationDigest(g: number): Promise<string> {
if (!Number.isInteger(g) || g < 1 || g > this.committed) {
throw new RangeError(
`generationDigest(): generation ${g} is out of range [1, ${this.committed}]`
)
}
if (g <= this.horizonGen) {
throw new GenerationCompactedError(g, this.horizonGen)
}
let digest = 0
const enc = new TextEncoder()
if (this.segments) {
const packed = await this.segments.digestThroughPacked(g)
if (packed !== null) digest = packed
}
// Live-tier composition: every committed gen ≤ g not covered by a sealed
// segment hashes its delta content in ascending order.
for (const gen of this.committedGensAsc()) {
if (gen > g) break
if (this.segments?.hasGeneration(gen)) continue
const delta = await this.getDelta(gen)
digest = crc32c(
enc.encode(
`${digest}:${gen}:${delta.timestamp}:${[...delta.nouns].sort().join(',')}:${[...delta.verbs].sort().join(',')}`
)
)
}
return digest.toString(16).padStart(8, '0')
}
async historyStats(): Promise<{
generations: number
bytes: number
@ -538,14 +627,17 @@ export class GenerationStore {
try {
paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`)
} catch {
return []
paths = []
}
const records: GenerationRecord[] = []
for (const p of paths) {
const record = (await this.storage.readRawObject(p)) as GenerationRecord | null
if (record) records.push(record)
}
return records
if (records.length > 0) return records
// Two-tier: folded generations serve their record-set from the segment.
const packed = await this.segments?.readRecords(gen)
return packed ? (packed.map((r) => r.record) as GenerationRecord[]) : []
}
/**
@ -1783,9 +1875,15 @@ export class GenerationStore {
if (pending) {
return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null
}
return (await this.storage.readRawObject(
const live = (await this.storage.readRawObject(
`${GENERATIONS_PREFIX}/${gen}/prev/${id}.json`
)) as GenerationRecord | null
if (live) return live
// Two-tier: the packed tier serves folded generations (live-tier-wins).
if (this.segments?.hasGeneration(gen)) {
return (await this.segments.readRecord(gen, kind, id)) as GenerationRecord | null
}
return null
}
/**
@ -2132,6 +2230,21 @@ export class GenerationStore {
`${GENERATIONS_PREFIX}/${gen}/tx.json`
)) as GenerationDelta | null
if (delta === null) {
// Two-tier read (D1+D3): not in the live tier → the packed tier.
// Live-tier-wins ordering (a crash mid-fold leaves a duplicate, never
// a gap), so the segment lookup runs only after the live miss.
const packed = await this.segments?.readDelta(gen)
if (packed) {
const d = packed.delta as GenerationDelta
const entry = {
nouns: new Set(d.nouns),
verbs: new Set(d.verbs),
timestamp: packed.timestamp,
bytes: d.bytes ?? 0
}
this.setDelta(gen, entry)
return entry
}
throw new Error(
`Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` +
`(store corrupted or records removed outside compactHistory())`
@ -2213,6 +2326,94 @@ export class GenerationStore {
* @param options - Retention caps (see {@link CompactHistoryOptions}).
* @returns Count of removed record-sets and the new horizon.
*/
/**
* @description The REPACKER (D1+D3+repacking): fold cold live-tier
* generations into sealed segments — re-representation, never deletion.
* Every record and delta stays readable (asOf/chains unchanged); the
* per-generation directories are deleted only AFTER their segment is
* durable (crash between = duplicate representation, resolved
* live-tier-wins by every reader; never a gap). This is the transform that
* takes a 70k-file history to tens of segment files, and the ONLY history
* transform permitted under the archival profile.
*
* Folds oldest-first, contiguous from the packed boundary, in batches, and
* stops at the live window ({@link GenerationStore.REPACK_LIVE_WINDOW})
* or when `timeBudgetMs` is spent — an early stop is a consistent prefix;
* the next pass resumes.
*/
async repackHistory(options?: { timeBudgetMs?: number; batchGenerations?: number }): Promise<{
foldedGenerations: number
segmentsCreated: number
}> {
if (!this.segments) return { foldedGenerations: 0, segmentsCreated: 0 }
const segments = this.segments
return this.withMutex(async () => {
const deadline =
options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined
const batchSize = options?.batchGenerations ?? 512
const coldCeiling = this.committed - GenerationStore.REPACK_LIVE_WINDOW
const packedThrough =
segments.segments().length > 0
? segments.segments()[segments.segments().length - 1].lastGeneration
: 0
// Cold, unpacked, committed generations — ascending, contiguous scan.
const eligible: number[] = []
for (const gen of this.committedGensAsc()) {
if (gen > coldCeiling) break
if (gen <= packedThrough) continue // already packed (dup fold barred)
if (this.pendingBuffer.has(gen)) continue // un-flushed = live by definition
eligible.push(gen)
}
let folded = 0
let segmentsCreated = 0
for (let i = 0; i < eligible.length; i += batchSize) {
if (deadline !== undefined && Date.now() >= deadline) break
const batch = eligible.slice(i, i + batchSize)
const foldInput: FoldGeneration[] = []
for (const gen of batch) {
const delta = (await this.storage.readRawObject(
`${GENERATIONS_PREFIX}/${gen}/tx.json`
)) as GenerationDelta | null
if (delta === null) {
// Already folded by a prior crashed pass whose dirs were removed,
// or damage — getDelta's two-tier read decides which, loudly,
// when someone asks. Skip; never fold a generation we cannot read.
continue
}
const records: FoldGeneration['records'] = []
for (const [kind, ids] of [
['noun', delta.nouns] as const,
['verb', delta.verbs] as const
]) {
for (const id of ids) {
const record = await this.storage.readRawObject(
`${GENERATIONS_PREFIX}/${gen}/prev/${id}.json`
)
if (record) records.push({ kind, id, record })
}
}
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
}
if (foldInput.length === 0) continue
await segments.fold(foldInput)
segmentsCreated++
// Segment + manifest durable → the live copies retire.
for (const g of foldInput) {
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
}
folded += foldInput.length
}
if (folded > 0) {
prodLog.info(
`[GenerationStore] repacked ${folded} cold generation(s) into ${segmentsCreated} segment(s) — history preserved, file count reduced`
)
}
return { foldedGenerations: folded, segmentsCreated }
})
}
async compact(options?: CompactHistoryOptions): Promise<CompactHistoryResult> {
return this.withMutex(async () => {
const minPinned = this.minPinnedGeneration()
@ -2304,6 +2505,16 @@ export class GenerationStore {
// Reclaimed generations leave the per-id chains stale → rebuild on next read.
this.invalidateChains()
this.horizonGen = Math.max(this.horizonGen, highestRemoved)
// Packed-tier reclaim (D3): a packed generation's bytes live in a
// sealed segment — removeRawPrefix above was a no-op for it. Drop
// WHOLE segments now fully below the horizon; a partially-reclaimed
// segment keeps its bytes until the boundary passes it (the frozen
// partial-segments-wait rule; logical reclamation above still holds —
// the generations left committedRanges and asOf below the horizon
// throws regardless).
if (this.segments) {
await this.segments.dropSegmentsBelow(this.horizonGen + 1)
}
const manifest: GenerationManifest = {
version: 1,
generation: this.committed,