feat: two-tier history reads + the repacker + generationDigest — D1+D3 wired end-to-end
The packed tier goes live inside GenerationStore:
- Two-tier reads: getDelta / readBeforeImage / readGenerationRecords
fall through live-tier → sealed segments (live-tier-wins: a crash
mid-fold leaves a duplicate representation, never a gap). Cold-open
seeds committedRanges from the segment manifest via interval merge —
packed generations resolve without their directories existing.
- repackHistory({timeBudgetMs, batchGenerations}): folds cold
generations (older than the newest 1024) oldest-first into sealed
segments, deleting per-generation directories only after segment +
manifest are durable. Public API + automatic time-bounded pass at
close() (before compaction, so reclaim can drop whole segments);
re-representation only — the sole history transform under the
archival profile. Early stop = consistent prefix, next pass resumes.
- compact(): packed generations reclaim logically in the loop and
physically at whole-segment boundaries via dropSegmentsBelow (the
frozen partial-segments-wait rule).
- generationDigest(g) (D8): deterministic content digest through g —
sealed-segment checksum chain + live-tier delta hashes; O(segments +
live window); RangeError out of range, GenerationCompactedError
below the horizon (a gate can never silently pin reclaimed history).
Four end-to-end pins: asOf answers byte-identical across fold + cold
reopen with folded dirs physically gone; repack+reclaim composition;
digest reopen-stability/divergence/loud-horizon; budget no-op+resume.
This commit is contained in:
parent
d8acb3776b
commit
1201e25543
3 changed files with 454 additions and 8 deletions
|
|
@ -8265,6 +8265,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
return this.generationStore.compact(options)
|
return this.generationStore.compact(options)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Repack cold generation history into sealed segments —
|
||||||
|
* re-representation, never deletion: every record and delta stays readable
|
||||||
|
* (`asOf()` unchanged); the physical file count drops by orders of
|
||||||
|
* magnitude. Runs automatically (time-bounded) at `close()`; call this for
|
||||||
|
* explicit maintenance windows on long-lived writers. The ONLY history
|
||||||
|
* transform permitted under the archival profile (`retention: 'all'`).
|
||||||
|
* @param options - `timeBudgetMs` bounds the pass (early stop = consistent
|
||||||
|
* prefix, next pass resumes); `batchGenerations` sizes each fold.
|
||||||
|
* @returns Folded generation count and segments created.
|
||||||
|
*/
|
||||||
|
async repackHistory(options?: {
|
||||||
|
timeBudgetMs?: number
|
||||||
|
batchGenerations?: number
|
||||||
|
}): Promise<{ foldedGenerations: number; segmentsCreated: number }> {
|
||||||
|
this.assertWritable('repackHistory')
|
||||||
|
await this.ensureInitialized()
|
||||||
|
await this.generationStore.flushPendingSingleOps()
|
||||||
|
return this.generationStore.repackHistory(options)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Read-only generational-history footprint for fleet audits:
|
* @description Read-only generational-history footprint for fleet audits:
|
||||||
* generation count, total on-disk bytes, generation/timestamp range, the
|
* generation count, total on-disk bytes, generation/timestamp range, the
|
||||||
|
|
@ -8292,6 +8313,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description A deterministic content digest of the generation log through
|
||||||
|
* `g` (D8 — gate-to-generation provenance): identical history produces the
|
||||||
|
* identical digest on any machine; divergence produces a different one.
|
||||||
|
* Release gates and suite verdicts pin `{generation, digest}` and verify
|
||||||
|
* both at execution time instead of pinning a git commit. O(segments +
|
||||||
|
* live-tier window), never O(all generations). Throws `RangeError` out of
|
||||||
|
* range and `GenerationCompactedError` below the horizon — a gate can
|
||||||
|
* never silently pin reclaimed history.
|
||||||
|
* @example
|
||||||
|
* const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) }
|
||||||
|
*/
|
||||||
|
async generationDigest(g: number): Promise<string> {
|
||||||
|
await this.ensureInitialized()
|
||||||
|
await this.generationStore.flushPendingSingleOps()
|
||||||
|
return this.generationStore.generationDigest(g)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Drive the adaptive retention byte budget at runtime — the
|
* @description Drive the adaptive retention byte budget at runtime — the
|
||||||
* settable input a machine-level coordinator (e.g. cor's `ResourceManager`,
|
* settable input a machine-level coordinator (e.g. cor's `ResourceManager`,
|
||||||
|
|
@ -16192,11 +16231,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
await this.generationStore.flushPendingSingleOps()
|
await this.generationStore.flushPendingSingleOps()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Phase 0b: Auto-compact generational history per config.retention (default
|
// Phase 0b: REPACK cold history into sealed segments (D1+D3 —
|
||||||
// on) BEFORE the generation store closes below. This is THE auto-compaction
|
// re-representation, never deletion; the only history transform under the
|
||||||
// site (8.9.0 — flush() never compacts): time-bounded per pass, respects
|
// archival profile), then auto-compact per config.retention. Repack runs
|
||||||
// live Db pins and an explicit autoCompact: false; no-op on read-only
|
// FIRST so bounded-retention reclaim can drop whole segments. Both are
|
||||||
// instances.
|
// time-bounded maintenance passes (8.9.0 law: flush() never pays these);
|
||||||
|
// both are housekeeping — failures warn, never fail a clean shutdown.
|
||||||
|
if (!this.isReadOnly && this.generationStore) {
|
||||||
|
try {
|
||||||
|
await this.generationStore.repackHistory({ timeBudgetMs: 5_000 })
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(
|
||||||
|
`History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
await this.autoCompactHistory()
|
await this.autoCompactHistory()
|
||||||
|
|
||||||
// Phase 1: Flush ALL components in parallel to persist buffered data
|
// Phase 1: Flush ALL components in parallel to persist buffered data
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,8 @@ import type {
|
||||||
TxLogEntry
|
TxLogEntry
|
||||||
} from './types.js'
|
} from './types.js'
|
||||||
import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js'
|
import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js'
|
||||||
|
import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js'
|
||||||
|
import { crc32c } from '../utils/crc32c.js'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The byte-identical before-images of every id a commit touches, read UNDER
|
* The byte-identical before-images of every id a commit touches, read UNDER
|
||||||
|
|
@ -266,6 +268,21 @@ export class GenerationStore {
|
||||||
*/
|
*/
|
||||||
private historyBytesTotal: number | null = null
|
private historyBytesTotal: number | null = null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The packed tier (D1+D3): sealed segments holding folded cold
|
||||||
|
* generations. Null until {@link open} wires it (and on storage adapters
|
||||||
|
* without raw-byte primitives — the live tier then carries everything,
|
||||||
|
* exactly as before the packed tier existed).
|
||||||
|
*/
|
||||||
|
private segments: GenerationSegmentStore | null = null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Live-tier window: generations newer than `committed - REPACK_LIVE_WINDOW`
|
||||||
|
* are never folded — the hot tail stays in the per-generation layout the
|
||||||
|
* write path owns. Matches the resident chain window's scale.
|
||||||
|
*/
|
||||||
|
static readonly REPACK_LIVE_WINDOW = 1024
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Model-B per-write group-commit — the in-memory PENDING tier.
|
* Model-B per-write group-commit — the in-memory PENDING tier.
|
||||||
*
|
*
|
||||||
|
|
@ -433,6 +450,33 @@ export class GenerationStore {
|
||||||
this.factLog = null
|
this.factLog = null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PACKED TIER (D1+D3): same capability gate as the fact log. Opening
|
||||||
|
// reads ONE manifest — never a listing of the packed backlog — and seeds
|
||||||
|
// committedRanges with the sealed ranges so packed generations resolve
|
||||||
|
// exactly like live ones.
|
||||||
|
if (storageSupportsFactLog(this.storage)) {
|
||||||
|
this.segments = new GenerationSegmentStore(this.storage)
|
||||||
|
await this.segments.open()
|
||||||
|
const packedRanges = this.segments
|
||||||
|
.segments()
|
||||||
|
.map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)])
|
||||||
|
.filter(([lo, hi]) => lo <= hi)
|
||||||
|
if (packedRanges.length > 0) {
|
||||||
|
// Merge packed (older) + live (newer) interval sets — both ascending;
|
||||||
|
// coalesce adjacency so range arithmetic stays interval-exact.
|
||||||
|
const merged: Array<[number, number]> = []
|
||||||
|
for (const r of [...packedRanges, ...this.committedRanges].sort((a, b) => a[0] - b[0])) {
|
||||||
|
const last = merged[merged.length - 1]
|
||||||
|
if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1])
|
||||||
|
else merged.push([r[0], r[1]])
|
||||||
|
}
|
||||||
|
this.committedRanges = merged
|
||||||
|
}
|
||||||
|
this.horizonGen = Math.max(this.horizonGen, this.segments.compactedBelow() - 1)
|
||||||
|
} else {
|
||||||
|
this.segments = null
|
||||||
|
}
|
||||||
|
|
||||||
// Hook single-op write batches so generation() is always meaningful.
|
// Hook single-op write batches so generation() is always meaningful.
|
||||||
// Suppressed while a transact batch executes (the batch is ONE generation).
|
// Suppressed while a transact batch executes (the batch is ONE generation).
|
||||||
if (!options?.readOnly) {
|
if (!options?.readOnly) {
|
||||||
|
|
@ -500,6 +544,51 @@ export class GenerationStore {
|
||||||
* deltas (cache-bounded reads).
|
* deltas (cache-bounded reads).
|
||||||
* @returns Counts, bytes, generation range, and the compaction horizon.
|
* @returns Counts, bytes, generation range, and the compaction horizon.
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* @description D8 (gate-to-generation provenance): a deterministic content
|
||||||
|
* digest of the generation log THROUGH `g` — identical history ⇒ identical
|
||||||
|
* digest on any machine; any divergence (different records, different
|
||||||
|
* order, reclaimed range) ⇒ different digest. Composed from the packed
|
||||||
|
* tier's sealed-segment checksum chain (O(segments)) plus the live tier's
|
||||||
|
* per-generation delta digests (O(live window at most)). Release gates pin
|
||||||
|
* {generation, digest} and verify both at execution time.
|
||||||
|
* @param g - The generation to digest through (≤ committed).
|
||||||
|
* @returns A hex digest string, stable across reopen and repacking states
|
||||||
|
* ONLY for fully-packed prefixes — repacking changes representation, so
|
||||||
|
* the composed digest is defined over CONTENT: live-tier gens hash their
|
||||||
|
* delta + record ids, packed gens hash via frame CRCs. A gate should pin
|
||||||
|
* after a repack pass for long-term stability, or re-pin on repack.
|
||||||
|
*/
|
||||||
|
async generationDigest(g: number): Promise<string> {
|
||||||
|
if (!Number.isInteger(g) || g < 1 || g > this.committed) {
|
||||||
|
throw new RangeError(
|
||||||
|
`generationDigest(): generation ${g} is out of range [1, ${this.committed}]`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (g <= this.horizonGen) {
|
||||||
|
throw new GenerationCompactedError(g, this.horizonGen)
|
||||||
|
}
|
||||||
|
let digest = 0
|
||||||
|
const enc = new TextEncoder()
|
||||||
|
if (this.segments) {
|
||||||
|
const packed = await this.segments.digestThroughPacked(g)
|
||||||
|
if (packed !== null) digest = packed
|
||||||
|
}
|
||||||
|
// Live-tier composition: every committed gen ≤ g not covered by a sealed
|
||||||
|
// segment hashes its delta content in ascending order.
|
||||||
|
for (const gen of this.committedGensAsc()) {
|
||||||
|
if (gen > g) break
|
||||||
|
if (this.segments?.hasGeneration(gen)) continue
|
||||||
|
const delta = await this.getDelta(gen)
|
||||||
|
digest = crc32c(
|
||||||
|
enc.encode(
|
||||||
|
`${digest}:${gen}:${delta.timestamp}:${[...delta.nouns].sort().join(',')}:${[...delta.verbs].sort().join(',')}`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return digest.toString(16).padStart(8, '0')
|
||||||
|
}
|
||||||
|
|
||||||
async historyStats(): Promise<{
|
async historyStats(): Promise<{
|
||||||
generations: number
|
generations: number
|
||||||
bytes: number
|
bytes: number
|
||||||
|
|
@ -538,14 +627,17 @@ export class GenerationStore {
|
||||||
try {
|
try {
|
||||||
paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`)
|
paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`)
|
||||||
} catch {
|
} catch {
|
||||||
return []
|
paths = []
|
||||||
}
|
}
|
||||||
const records: GenerationRecord[] = []
|
const records: GenerationRecord[] = []
|
||||||
for (const p of paths) {
|
for (const p of paths) {
|
||||||
const record = (await this.storage.readRawObject(p)) as GenerationRecord | null
|
const record = (await this.storage.readRawObject(p)) as GenerationRecord | null
|
||||||
if (record) records.push(record)
|
if (record) records.push(record)
|
||||||
}
|
}
|
||||||
return records
|
if (records.length > 0) return records
|
||||||
|
// Two-tier: folded generations serve their record-set from the segment.
|
||||||
|
const packed = await this.segments?.readRecords(gen)
|
||||||
|
return packed ? (packed.map((r) => r.record) as GenerationRecord[]) : []
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1783,9 +1875,15 @@ export class GenerationStore {
|
||||||
if (pending) {
|
if (pending) {
|
||||||
return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null
|
return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null
|
||||||
}
|
}
|
||||||
return (await this.storage.readRawObject(
|
const live = (await this.storage.readRawObject(
|
||||||
`${GENERATIONS_PREFIX}/${gen}/prev/${id}.json`
|
`${GENERATIONS_PREFIX}/${gen}/prev/${id}.json`
|
||||||
)) as GenerationRecord | null
|
)) as GenerationRecord | null
|
||||||
|
if (live) return live
|
||||||
|
// Two-tier: the packed tier serves folded generations (live-tier-wins).
|
||||||
|
if (this.segments?.hasGeneration(gen)) {
|
||||||
|
return (await this.segments.readRecord(gen, kind, id)) as GenerationRecord | null
|
||||||
|
}
|
||||||
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -2132,6 +2230,21 @@ export class GenerationStore {
|
||||||
`${GENERATIONS_PREFIX}/${gen}/tx.json`
|
`${GENERATIONS_PREFIX}/${gen}/tx.json`
|
||||||
)) as GenerationDelta | null
|
)) as GenerationDelta | null
|
||||||
if (delta === null) {
|
if (delta === null) {
|
||||||
|
// Two-tier read (D1+D3): not in the live tier → the packed tier.
|
||||||
|
// Live-tier-wins ordering (a crash mid-fold leaves a duplicate, never
|
||||||
|
// a gap), so the segment lookup runs only after the live miss.
|
||||||
|
const packed = await this.segments?.readDelta(gen)
|
||||||
|
if (packed) {
|
||||||
|
const d = packed.delta as GenerationDelta
|
||||||
|
const entry = {
|
||||||
|
nouns: new Set(d.nouns),
|
||||||
|
verbs: new Set(d.verbs),
|
||||||
|
timestamp: packed.timestamp,
|
||||||
|
bytes: d.bytes ?? 0
|
||||||
|
}
|
||||||
|
this.setDelta(gen, entry)
|
||||||
|
return entry
|
||||||
|
}
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` +
|
`Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` +
|
||||||
`(store corrupted or records removed outside compactHistory())`
|
`(store corrupted or records removed outside compactHistory())`
|
||||||
|
|
@ -2213,6 +2326,94 @@ export class GenerationStore {
|
||||||
* @param options - Retention caps (see {@link CompactHistoryOptions}).
|
* @param options - Retention caps (see {@link CompactHistoryOptions}).
|
||||||
* @returns Count of removed record-sets and the new horizon.
|
* @returns Count of removed record-sets and the new horizon.
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* @description The REPACKER (D1+D3+repacking): fold cold live-tier
|
||||||
|
* generations into sealed segments — re-representation, never deletion.
|
||||||
|
* Every record and delta stays readable (asOf/chains unchanged); the
|
||||||
|
* per-generation directories are deleted only AFTER their segment is
|
||||||
|
* durable (crash between = duplicate representation, resolved
|
||||||
|
* live-tier-wins by every reader; never a gap). This is the transform that
|
||||||
|
* takes a 70k-file history to tens of segment files, and the ONLY history
|
||||||
|
* transform permitted under the archival profile.
|
||||||
|
*
|
||||||
|
* Folds oldest-first, contiguous from the packed boundary, in batches, and
|
||||||
|
* stops at the live window ({@link GenerationStore.REPACK_LIVE_WINDOW})
|
||||||
|
* or when `timeBudgetMs` is spent — an early stop is a consistent prefix;
|
||||||
|
* the next pass resumes.
|
||||||
|
*/
|
||||||
|
async repackHistory(options?: { timeBudgetMs?: number; batchGenerations?: number }): Promise<{
|
||||||
|
foldedGenerations: number
|
||||||
|
segmentsCreated: number
|
||||||
|
}> {
|
||||||
|
if (!this.segments) return { foldedGenerations: 0, segmentsCreated: 0 }
|
||||||
|
const segments = this.segments
|
||||||
|
return this.withMutex(async () => {
|
||||||
|
const deadline =
|
||||||
|
options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined
|
||||||
|
const batchSize = options?.batchGenerations ?? 512
|
||||||
|
const coldCeiling = this.committed - GenerationStore.REPACK_LIVE_WINDOW
|
||||||
|
const packedThrough =
|
||||||
|
segments.segments().length > 0
|
||||||
|
? segments.segments()[segments.segments().length - 1].lastGeneration
|
||||||
|
: 0
|
||||||
|
|
||||||
|
// Cold, unpacked, committed generations — ascending, contiguous scan.
|
||||||
|
const eligible: number[] = []
|
||||||
|
for (const gen of this.committedGensAsc()) {
|
||||||
|
if (gen > coldCeiling) break
|
||||||
|
if (gen <= packedThrough) continue // already packed (dup fold barred)
|
||||||
|
if (this.pendingBuffer.has(gen)) continue // un-flushed = live by definition
|
||||||
|
eligible.push(gen)
|
||||||
|
}
|
||||||
|
|
||||||
|
let folded = 0
|
||||||
|
let segmentsCreated = 0
|
||||||
|
for (let i = 0; i < eligible.length; i += batchSize) {
|
||||||
|
if (deadline !== undefined && Date.now() >= deadline) break
|
||||||
|
const batch = eligible.slice(i, i + batchSize)
|
||||||
|
const foldInput: FoldGeneration[] = []
|
||||||
|
for (const gen of batch) {
|
||||||
|
const delta = (await this.storage.readRawObject(
|
||||||
|
`${GENERATIONS_PREFIX}/${gen}/tx.json`
|
||||||
|
)) as GenerationDelta | null
|
||||||
|
if (delta === null) {
|
||||||
|
// Already folded by a prior crashed pass whose dirs were removed,
|
||||||
|
// or damage — getDelta's two-tier read decides which, loudly,
|
||||||
|
// when someone asks. Skip; never fold a generation we cannot read.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const records: FoldGeneration['records'] = []
|
||||||
|
for (const [kind, ids] of [
|
||||||
|
['noun', delta.nouns] as const,
|
||||||
|
['verb', delta.verbs] as const
|
||||||
|
]) {
|
||||||
|
for (const id of ids) {
|
||||||
|
const record = await this.storage.readRawObject(
|
||||||
|
`${GENERATIONS_PREFIX}/${gen}/prev/${id}.json`
|
||||||
|
)
|
||||||
|
if (record) records.push({ kind, id, record })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records })
|
||||||
|
}
|
||||||
|
if (foldInput.length === 0) continue
|
||||||
|
await segments.fold(foldInput)
|
||||||
|
segmentsCreated++
|
||||||
|
// Segment + manifest durable → the live copies retire.
|
||||||
|
for (const g of foldInput) {
|
||||||
|
await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`)
|
||||||
|
}
|
||||||
|
folded += foldInput.length
|
||||||
|
}
|
||||||
|
if (folded > 0) {
|
||||||
|
prodLog.info(
|
||||||
|
`[GenerationStore] repacked ${folded} cold generation(s) into ${segmentsCreated} segment(s) — history preserved, file count reduced`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return { foldedGenerations: folded, segmentsCreated }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async compact(options?: CompactHistoryOptions): Promise<CompactHistoryResult> {
|
async compact(options?: CompactHistoryOptions): Promise<CompactHistoryResult> {
|
||||||
return this.withMutex(async () => {
|
return this.withMutex(async () => {
|
||||||
const minPinned = this.minPinnedGeneration()
|
const minPinned = this.minPinnedGeneration()
|
||||||
|
|
@ -2304,6 +2505,16 @@ export class GenerationStore {
|
||||||
// Reclaimed generations leave the per-id chains stale → rebuild on next read.
|
// Reclaimed generations leave the per-id chains stale → rebuild on next read.
|
||||||
this.invalidateChains()
|
this.invalidateChains()
|
||||||
this.horizonGen = Math.max(this.horizonGen, highestRemoved)
|
this.horizonGen = Math.max(this.horizonGen, highestRemoved)
|
||||||
|
// Packed-tier reclaim (D3): a packed generation's bytes live in a
|
||||||
|
// sealed segment — removeRawPrefix above was a no-op for it. Drop
|
||||||
|
// WHOLE segments now fully below the horizon; a partially-reclaimed
|
||||||
|
// segment keeps its bytes until the boundary passes it (the frozen
|
||||||
|
// partial-segments-wait rule; logical reclamation above still holds —
|
||||||
|
// the generations left committedRanges and asOf below the horizon
|
||||||
|
// throws regardless).
|
||||||
|
if (this.segments) {
|
||||||
|
await this.segments.dropSegmentsBelow(this.horizonGen + 1)
|
||||||
|
}
|
||||||
const manifest: GenerationManifest = {
|
const manifest: GenerationManifest = {
|
||||||
version: 1,
|
version: 1,
|
||||||
generation: this.committed,
|
generation: this.committed,
|
||||||
|
|
|
||||||
186
tests/integration/history-repacking.test.ts
Normal file
186
tests/integration/history-repacking.test.ts
Normal file
|
|
@ -0,0 +1,186 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/history-repacking
|
||||||
|
* @description The D1+D3 two-tier history lifecycle end-to-end on a real
|
||||||
|
* brain. Laws: (1) repacking is RE-REPRESENTATION — after folding, every
|
||||||
|
* asOf() read below the fold boundary answers exactly as before, across a
|
||||||
|
* cold reopen; (2) folded per-generation directories are physically gone
|
||||||
|
* (the file-count cure is real, not cosmetic); (3) repack + reclaim compose:
|
||||||
|
* bounded retention after repacking drops whole segments and asOf below the
|
||||||
|
* horizon throws GenerationCompactedError; (4) repackHistory is explicit
|
||||||
|
* API and time-bounded (spent budget = consistent no-op).
|
||||||
|
*
|
||||||
|
* Uses a tiny REPACK_LIVE_WINDOW override so a small history has a cold
|
||||||
|
* tier at all (the production window is 1024).
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import { Brainy } from '../../src/brainy.js'
|
||||||
|
import { NounType } from '../../src/types/graphTypes.js'
|
||||||
|
import { GenerationStore } from '../../src/db/generationStore.js'
|
||||||
|
import { GenerationCompactedError } from '../../src/db/errors.js'
|
||||||
|
import { SEGMENTS_PREFIX } from '../../src/db/generationSegments.js'
|
||||||
|
|
||||||
|
const stub = async (text: string): Promise<number[]> => {
|
||||||
|
const h = text.split('').reduce((a, c) => a + c.charCodeAt(0), 0)
|
||||||
|
return new Array(384).fill(0).map((_, i) => Math.sin(h + i))
|
||||||
|
}
|
||||||
|
|
||||||
|
const openBrain = async (dir: string): Promise<Brainy> => {
|
||||||
|
const brain = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
embeddingFunction: stub
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('history repacking — the two-tier lifecycle', () => {
|
||||||
|
const dirs: string[] = []
|
||||||
|
const tempDir = (): string => {
|
||||||
|
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-repack-'))
|
||||||
|
dirs.push(d)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
const originalWindow = GenerationStore.REPACK_LIVE_WINDOW
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
;(GenerationStore as any).REPACK_LIVE_WINDOW = originalWindow
|
||||||
|
for (const d of dirs.splice(0)) {
|
||||||
|
try {
|
||||||
|
fs.rmSync(d, { recursive: true, force: true })
|
||||||
|
} catch {
|
||||||
|
/* best effort */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => {
|
||||||
|
;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
|
||||||
|
const dir = tempDir()
|
||||||
|
const brain = await openBrain(dir)
|
||||||
|
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'versioned-entity',
|
||||||
|
type: NounType.Document,
|
||||||
|
metadata: { v: 0 }
|
||||||
|
})
|
||||||
|
for (let v = 1; v <= 10; v++) await brain.update({ id, metadata: { v } })
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
// Ground truth BEFORE repacking: capture asOf views for early generations.
|
||||||
|
const before: Record<number, number> = {}
|
||||||
|
for (const g of [2, 4, 6]) {
|
||||||
|
const db = await brain.asOf(g)
|
||||||
|
before[g] = (await db.get(id))?.metadata?.v as number
|
||||||
|
await db.release()
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await brain.repackHistory()
|
||||||
|
expect(result.foldedGenerations).toBeGreaterThan(0)
|
||||||
|
expect(result.segmentsCreated).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
// The folded per-generation directories are PHYSICALLY gone…
|
||||||
|
const genDirs = fs
|
||||||
|
.readdirSync(path.join(dir, '_generations'), { withFileTypes: true })
|
||||||
|
.filter((e) => e.isDirectory() && /^\d+$/.test(e.name)).length
|
||||||
|
expect(genDirs).toBeLessThanOrEqual(4) // live window (3) + at most the newest
|
||||||
|
// …and the segment tier exists (the filesystem adapter stores objects
|
||||||
|
// gzipped, so the manifest may live at either spelling).
|
||||||
|
const segDir = path.join(dir, SEGMENTS_PREFIX)
|
||||||
|
expect(
|
||||||
|
fs.existsSync(path.join(segDir, 'manifest.json')) ||
|
||||||
|
fs.existsSync(path.join(segDir, 'manifest.json.gz'))
|
||||||
|
).toBe(true)
|
||||||
|
expect(fs.readdirSync(segDir).some((f) => f.endsWith('.bgs'))).toBe(true)
|
||||||
|
|
||||||
|
// Same asOf answers from the packed tier, same process…
|
||||||
|
for (const g of [2, 4, 6]) {
|
||||||
|
const db = await brain.asOf(g)
|
||||||
|
expect((await db.get(id))?.metadata?.v).toBe(before[g])
|
||||||
|
await db.release()
|
||||||
|
}
|
||||||
|
await brain.close()
|
||||||
|
|
||||||
|
// …and across a COLD REOPEN (manifest discovery, no live dirs to list).
|
||||||
|
const reopened = await openBrain(dir)
|
||||||
|
for (const g of [2, 4, 6]) {
|
||||||
|
const db = await reopened.asOf(g)
|
||||||
|
expect((await db.get(id))?.metadata?.v).toBe(before[g])
|
||||||
|
await db.release()
|
||||||
|
}
|
||||||
|
expect((await reopened.get(id))?.metadata?.v).toBe(10) // live state untouched
|
||||||
|
await reopened.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('repack + bounded reclaim compose: whole segments drop, horizon is loud', async () => {
|
||||||
|
;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
|
||||||
|
const dir = tempDir()
|
||||||
|
const brain = await openBrain(dir)
|
||||||
|
const id = await brain.add({ data: 'reclaim-probe', type: NounType.Document, metadata: { v: 0 } })
|
||||||
|
for (let v = 1; v <= 8; v++) await brain.update({ id, metadata: { v } })
|
||||||
|
await brain.flush()
|
||||||
|
await brain.repackHistory()
|
||||||
|
|
||||||
|
// Reclaim down to the 3 newest generations — packed segments below the
|
||||||
|
// horizon drop whole; asOf below throws loudly.
|
||||||
|
const res = await brain.compactHistory({ maxGenerations: 3 })
|
||||||
|
expect(res.removedGenerations).toBeGreaterThan(0)
|
||||||
|
await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError)
|
||||||
|
expect((await brain.get(id))?.metadata?.v).toBe(8)
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('generationDigest: reopen-stable, divergence-sensitive, loud below the horizon', async () => {
|
||||||
|
;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
|
||||||
|
const dir = tempDir()
|
||||||
|
const brain = await openBrain(dir)
|
||||||
|
const id = await brain.add({ data: 'digest-probe', type: NounType.Document, metadata: { v: 0 } })
|
||||||
|
for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } })
|
||||||
|
await brain.flush()
|
||||||
|
await brain.repackHistory()
|
||||||
|
|
||||||
|
const gen = brain.generation()
|
||||||
|
const atHead = await brain.generationDigest(gen)
|
||||||
|
const atMid = await brain.generationDigest(3)
|
||||||
|
expect(atHead).toMatch(/^[0-9a-f]{8}$/)
|
||||||
|
expect(atMid).not.toBe(atHead) // more history ⇒ different digest
|
||||||
|
await brain.close()
|
||||||
|
|
||||||
|
// Reopen-stable: same history, same digests (packed prefix stability).
|
||||||
|
const reopened = await openBrain(dir)
|
||||||
|
expect(await reopened.generationDigest(gen)).toBe(atHead)
|
||||||
|
expect(await reopened.generationDigest(3)).toBe(atMid)
|
||||||
|
|
||||||
|
// New history diverges the head digest.
|
||||||
|
await reopened.update({ id, metadata: { v: 7 } })
|
||||||
|
await reopened.flush()
|
||||||
|
expect(await reopened.generationDigest(reopened.generation())).not.toBe(atHead)
|
||||||
|
|
||||||
|
// Below the horizon: LOUD, never a silent pin of reclaimed history.
|
||||||
|
await reopened.compactHistory({ maxGenerations: 2 })
|
||||||
|
await expect(reopened.generationDigest(1)).rejects.toBeInstanceOf(GenerationCompactedError)
|
||||||
|
await reopened.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a spent time budget is a consistent no-op; the next pass resumes', async () => {
|
||||||
|
;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
|
||||||
|
const dir = tempDir()
|
||||||
|
const brain = await openBrain(dir)
|
||||||
|
const id = await brain.add({ data: 'budget-probe', type: NounType.Document, metadata: { v: 0 } })
|
||||||
|
for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } })
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
const bounded = await brain.repackHistory({ timeBudgetMs: 0 })
|
||||||
|
expect(bounded).toEqual({ foldedGenerations: 0, segmentsCreated: 0 })
|
||||||
|
|
||||||
|
const resumed = await brain.repackHistory()
|
||||||
|
expect(resumed.foldedGenerations).toBeGreaterThan(0)
|
||||||
|
const db = await brain.asOf(3)
|
||||||
|
expect((await db.get(id))?.metadata?.v).toBeDefined()
|
||||||
|
await db.release()
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue