fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip
All checks were successful
CI / Node 22 (push) Successful in 12m13s
CI / Node 24 (push) Successful in 12m9s
CI / Integration + conformance (Node 22) (push) Successful in 18m16s
CI / Bun (latest) (push) Successful in 12m20s

A production brain's first process boot after a live authority flip looked
hung and was restarted three times mid-recovery — three defects with one
scene. (1) THE FOLD MATERIALIZED THE LOG: peekFactsAbove(0) decoded every
fact into one array (GBs of after-images on a ~7k-fact log, a GC storm, a
starved write lane). The fold now STREAMS one segment-batch at a time —
memory is one segment at any log size — with structural ordering asserted
loudly. (2) THE FOLD WAS SILENT UNTIL DONE: minutes of boot work with zero
narration is what invited the restarts. It now announces itself BEFORE the
work ('do not restart, the fold is finite') and prints progress every
thousand facts. (3) THE CHAIN COULD ONLY ARM AT A CRASH: a live mid-session
flip left the fold checkpoint unfounded, so the brain's first unclean boot
paid a whole-log fold. Adoption now founds the checkpoint AT THE FLIP — one
paged full canonical barrier (bounded memory), then the stamp — so bounded
recovery holds from minute zero for every store that flips, at any size.

Pinned: a non-fresh flip stamps immediately; the first post-flip unclean
boot folds bounded (an unflushed at-ack fact above the checkpoint is
restored; a barrier-covered row below it is outside the fold). Kill matrix
and both adoption suites green alongside.
This commit is contained in:
David Snelling 2026-08-18 12:53:50 -07:00
parent 8fb6cb7e54
commit ed7d1db97e
4 changed files with 215 additions and 33 deletions

View file

@ -637,33 +637,63 @@ export class GenerationStore {
this.foldCheckpointChainValid = checkpoint !== null || this.committed === 0
this.foldCheckpoint = foldBound
if (uncleanOpen) this.foldCheckpointChainValid = true
const factsToReplay = uncleanOpen
? await this.factLog.peekFactsAbove(foldBound)
: orphans
if (factsToReplay.length > 0) {
let replayed = 0
for (const fact of factsToReplay) {
for (const op of fact.ops) {
const image =
op.record === null
? { metadata: null, vector: null }
: { metadata: op.record.metadata, vector: op.record.vector }
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
else await this.storage.writeNounRaw(op.id, image)
this.noteCheckpointDirty(op.kind, op.id)
}
replayed++
if (fact.generation > this.committed) {
this.committed = fact.generation
this.appendCommittedGen(fact.generation)
this.setDelta(fact.generation, {
nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)),
verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)),
timestamp: fact.timestamp,
bytes: 0
})
}
// THE FOLD STREAMS AND NARRATES. A production first boot after a live
// flip folded ~7k facts by materializing them all (GBs of decoded
// after-images, a GC storm, a starved write lane) in SILENCE — the
// operator restarted the process three times mid-fold, each restart
// making the next boot unclean again. Two laws from that day: the
// fold consumes the log one segment-batch at a time (memory = one
// segment, any log size), and it announces itself BEFORE the work
// with progress lines DURING it — an operator who can see a fold
// converging lets it finish.
const foldKind = uncleanOpen
? foldBound > 0
? `BOUNDED fold above checkpoint ${foldBound}`
: 'WHOLE-LOG fold'
: 'above-manifest replay'
let replayed = 0
const replayFact = async (fact: CommitFact): Promise<void> => {
for (const op of fact.ops) {
const image =
op.record === null
? { metadata: null, vector: null }
: { metadata: op.record.metadata, vector: op.record.vector }
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
else await this.storage.writeNounRaw(op.id, image)
this.noteCheckpointDirty(op.kind, op.id)
}
replayed++
if (replayed % 1000 === 0) {
prodLog.warn(
`[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` +
`(at generation ${fact.generation}); do not restart, the fold is finite`
)
}
if (fact.generation > this.committed) {
this.committed = fact.generation
this.appendCommittedGen(fact.generation)
this.setDelta(fact.generation, {
nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)),
verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)),
timestamp: fact.timestamp,
bytes: 0
})
}
}
if (uncleanOpen) {
prodLog.warn(
`[GenerationStore] log-authority recovery: ${foldKind} beginning ` +
`(unclean shutdown detected) — streaming replay, bounded memory, ` +
`progress every 1000 facts. Do not restart the process; a restart ` +
`re-pays the whole fold.`
)
for await (const batch of this.factLog.streamFactsAbove(foldBound)) {
for (const fact of batch) await replayFact(fact)
}
} else {
for (const fact of orphans) await replayFact(fact)
}
if (replayed > 0) {
if (this.counter < this.committed) this.counter = this.committed
await this.persistCounterUnlocked()
const manifest: GenerationManifest = {
@ -676,13 +706,7 @@ export class GenerationStore {
await this.storage.syncRawObjects([MANIFEST_PATH])
prodLog.warn(
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
`canonical (${
uncleanOpen
? foldBound > 0
? `BOUNDED fold above checkpoint ${foldBound} — unclean shutdown`
: 'WHOLE-LOG fold — unclean shutdown'
: 'above-manifest'
}; committed at ${this.committed}) an acked write is never lost`
`canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost`
)
}
// A recovery fold re-applied (and the barrier below re-syncs) every
@ -897,6 +921,44 @@ export class GenerationStore {
this.authorityIsLog = true
}
/** Whether the fold-checkpoint chain is armed (a bounded fold is possible). */
foldCheckpointChainArmed(): boolean {
return this.foldCheckpointChainValid
}
/**
* @description Stamp the fold checkpoint after the caller has completed a
* FULL canonical barrier (every live row's canonical bytes fsynced, paged
* the adoption path does this right after a non-fresh flip). The stamp
* asserts total coverage, so it may ONLY be called when the barrier walked
* everything; stamp-after-data is the caller's ordering to keep. Arms the
* chain: the brain's first unclean boot folds (checkpoint, head] instead of
* the whole log a production first boot after a live flip paid a full-log
* fold through three mid-fold restarts because the chain could previously
* only arm at a crash.
*/
async stampFoldCheckpointAfterFullBarrier(): Promise<void> {
return this.withMutex(async () => {
if (!this.authorityIsLog || !this.factLog) {
throw new Error(
'stampFoldCheckpointAfterFullBarrier: only a log-authority brain stamps a fold checkpoint'
)
}
this.foldCheckpointChainValid = true
// The full barrier supersedes any accumulated partial set.
this.checkpointDirtyNouns = new Set()
this.checkpointDirtyVerbs = new Set()
const target = this.committed
await this.storage.writeRawObject(FOLD_CHECKPOINT_PATH, { generation: target })
await this.storage.syncRawObjects([FOLD_CHECKPOINT_PATH])
this.foldCheckpoint = target
prodLog.info(
`[GenerationStore] fold checkpoint founded at generation ${target}` +
`crash recovery is bounded from this moment`
)
})
}
/**
* @description Adoption-time chain bootstrap, abort called when an
* adoption attempt throws or refuses after phase 1. Disarms the chain and