fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip
A production brain's first process boot after a live authority flip looked
hung and was restarted three times mid-recovery — three defects with one
scene. (1) THE FOLD MATERIALIZED THE LOG: peekFactsAbove(0) decoded every
fact into one array (GBs of after-images on a ~7k-fact log, a GC storm, a
starved write lane). The fold now STREAMS one segment-batch at a time —
memory is one segment at any log size — with structural ordering asserted
loudly. (2) THE FOLD WAS SILENT UNTIL DONE: minutes of boot work with zero
narration is what invited the restarts. It now announces itself BEFORE the
work ('do not restart, the fold is finite') and prints progress every
thousand facts. (3) THE CHAIN COULD ONLY ARM AT A CRASH: a live mid-session
flip left the fold checkpoint unfounded, so the brain's first unclean boot
paid a whole-log fold. Adoption now founds the checkpoint AT THE FLIP — one
paged full canonical barrier (bounded memory), then the stamp — so bounded
recovery holds from minute zero for every store that flips, at any size.
Pinned: a non-fresh flip stamps immediately; the first post-flip unclean
boot folds bounded (an unflushed at-ack fact above the checkpoint is
restored; a barrier-covered row below it is outside the fold). Kill matrix
and both adoption suites green alongside.
This commit is contained in:
parent
8fb6cb7e54
commit
ed7d1db97e
4 changed files with 215 additions and 33 deletions
|
|
@ -637,33 +637,63 @@ export class GenerationStore {
|
|||
this.foldCheckpointChainValid = checkpoint !== null || this.committed === 0
|
||||
this.foldCheckpoint = foldBound
|
||||
if (uncleanOpen) this.foldCheckpointChainValid = true
|
||||
const factsToReplay = uncleanOpen
|
||||
? await this.factLog.peekFactsAbove(foldBound)
|
||||
: orphans
|
||||
if (factsToReplay.length > 0) {
|
||||
let replayed = 0
|
||||
for (const fact of factsToReplay) {
|
||||
for (const op of fact.ops) {
|
||||
const image =
|
||||
op.record === null
|
||||
? { metadata: null, vector: null }
|
||||
: { metadata: op.record.metadata, vector: op.record.vector }
|
||||
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
|
||||
else await this.storage.writeNounRaw(op.id, image)
|
||||
this.noteCheckpointDirty(op.kind, op.id)
|
||||
}
|
||||
replayed++
|
||||
if (fact.generation > this.committed) {
|
||||
this.committed = fact.generation
|
||||
this.appendCommittedGen(fact.generation)
|
||||
this.setDelta(fact.generation, {
|
||||
nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)),
|
||||
verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)),
|
||||
timestamp: fact.timestamp,
|
||||
bytes: 0
|
||||
})
|
||||
}
|
||||
// THE FOLD STREAMS AND NARRATES. A production first boot after a live
|
||||
// flip folded ~7k facts by materializing them all (GBs of decoded
|
||||
// after-images, a GC storm, a starved write lane) in SILENCE — the
|
||||
// operator restarted the process three times mid-fold, each restart
|
||||
// making the next boot unclean again. Two laws from that day: the
|
||||
// fold consumes the log one segment-batch at a time (memory = one
|
||||
// segment, any log size), and it announces itself BEFORE the work
|
||||
// with progress lines DURING it — an operator who can see a fold
|
||||
// converging lets it finish.
|
||||
const foldKind = uncleanOpen
|
||||
? foldBound > 0
|
||||
? `BOUNDED fold above checkpoint ${foldBound}`
|
||||
: 'WHOLE-LOG fold'
|
||||
: 'above-manifest replay'
|
||||
let replayed = 0
|
||||
const replayFact = async (fact: CommitFact): Promise<void> => {
|
||||
for (const op of fact.ops) {
|
||||
const image =
|
||||
op.record === null
|
||||
? { metadata: null, vector: null }
|
||||
: { metadata: op.record.metadata, vector: op.record.vector }
|
||||
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
|
||||
else await this.storage.writeNounRaw(op.id, image)
|
||||
this.noteCheckpointDirty(op.kind, op.id)
|
||||
}
|
||||
replayed++
|
||||
if (replayed % 1000 === 0) {
|
||||
prodLog.warn(
|
||||
`[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` +
|
||||
`(at generation ${fact.generation}); do not restart, the fold is finite`
|
||||
)
|
||||
}
|
||||
if (fact.generation > this.committed) {
|
||||
this.committed = fact.generation
|
||||
this.appendCommittedGen(fact.generation)
|
||||
this.setDelta(fact.generation, {
|
||||
nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)),
|
||||
verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)),
|
||||
timestamp: fact.timestamp,
|
||||
bytes: 0
|
||||
})
|
||||
}
|
||||
}
|
||||
if (uncleanOpen) {
|
||||
prodLog.warn(
|
||||
`[GenerationStore] log-authority recovery: ${foldKind} beginning ` +
|
||||
`(unclean shutdown detected) — streaming replay, bounded memory, ` +
|
||||
`progress every 1000 facts. Do not restart the process; a restart ` +
|
||||
`re-pays the whole fold.`
|
||||
)
|
||||
for await (const batch of this.factLog.streamFactsAbove(foldBound)) {
|
||||
for (const fact of batch) await replayFact(fact)
|
||||
}
|
||||
} else {
|
||||
for (const fact of orphans) await replayFact(fact)
|
||||
}
|
||||
if (replayed > 0) {
|
||||
if (this.counter < this.committed) this.counter = this.committed
|
||||
await this.persistCounterUnlocked()
|
||||
const manifest: GenerationManifest = {
|
||||
|
|
@ -676,13 +706,7 @@ export class GenerationStore {
|
|||
await this.storage.syncRawObjects([MANIFEST_PATH])
|
||||
prodLog.warn(
|
||||
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
|
||||
`canonical (${
|
||||
uncleanOpen
|
||||
? foldBound > 0
|
||||
? `BOUNDED fold above checkpoint ${foldBound} — unclean shutdown`
|
||||
: 'WHOLE-LOG fold — unclean shutdown'
|
||||
: 'above-manifest'
|
||||
}; committed at ${this.committed}) — an acked write is never lost`
|
||||
`canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost`
|
||||
)
|
||||
}
|
||||
// A recovery fold re-applied (and the barrier below re-syncs) every
|
||||
|
|
@ -897,6 +921,44 @@ export class GenerationStore {
|
|||
this.authorityIsLog = true
|
||||
}
|
||||
|
||||
/** Whether the fold-checkpoint chain is armed (a bounded fold is possible). */
|
||||
foldCheckpointChainArmed(): boolean {
|
||||
return this.foldCheckpointChainValid
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Stamp the fold checkpoint after the caller has completed a
|
||||
* FULL canonical barrier (every live row's canonical bytes fsynced, paged —
|
||||
* the adoption path does this right after a non-fresh flip). The stamp
|
||||
* asserts total coverage, so it may ONLY be called when the barrier walked
|
||||
* everything; stamp-after-data is the caller's ordering to keep. Arms the
|
||||
* chain: the brain's first unclean boot folds (checkpoint, head] instead of
|
||||
* the whole log — a production first boot after a live flip paid a full-log
|
||||
* fold through three mid-fold restarts because the chain could previously
|
||||
* only arm at a crash.
|
||||
*/
|
||||
async stampFoldCheckpointAfterFullBarrier(): Promise<void> {
|
||||
return this.withMutex(async () => {
|
||||
if (!this.authorityIsLog || !this.factLog) {
|
||||
throw new Error(
|
||||
'stampFoldCheckpointAfterFullBarrier: only a log-authority brain stamps a fold checkpoint'
|
||||
)
|
||||
}
|
||||
this.foldCheckpointChainValid = true
|
||||
// The full barrier supersedes any accumulated partial set.
|
||||
this.checkpointDirtyNouns = new Set()
|
||||
this.checkpointDirtyVerbs = new Set()
|
||||
const target = this.committed
|
||||
await this.storage.writeRawObject(FOLD_CHECKPOINT_PATH, { generation: target })
|
||||
await this.storage.syncRawObjects([FOLD_CHECKPOINT_PATH])
|
||||
this.foldCheckpoint = target
|
||||
prodLog.info(
|
||||
`[GenerationStore] fold checkpoint founded at generation ${target} — ` +
|
||||
`crash recovery is bounded from this moment`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Adoption-time chain bootstrap, abort — called when an
|
||||
* adoption attempt throws or refuses after phase 1. Disarms the chain and
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue