feat(recovery): the fold-checkpoint bound — crash folds (checkpoint, head], never the whole log twice
The fold checkpoint (_system/fold-checkpoint.json) is stamped strictly after a canonical-sync barrier over every live entity touched since the last stamp (syncEntityCanonical: ids → canonical paths → fsync; an absent file fsyncs its parent directory so deletes are as durable as writes). An unclean open under log authority now folds only (checkpoint, head]; the chain bootstraps at an empty brain's adoption (three-phase hooks around adoptLogAuthority) or at a brain's first whole-log fold — existing brains converge at their first crash with zero regression. Rollback restores sync immediately; abort paths feed the barrier; a failed barrier retains the old bound (bigger fold later, never a lost write). Five structural pins including boundedness itself. Also: the production-shaped write-flow gate leg (mixed traffic racing flushes, crash mid-traffic, every ack survives — from a consumer-reported gate miss), and two release-ceremony cures (tag-first push so the publish never queues behind the release commit's CI run; raw-curl npmjs shasum probe with propagation grace instead of a one-shot false divergence).
This commit is contained in:
parent
cbe34d115e
commit
ff43de1ada
8 changed files with 695 additions and 12 deletions
|
|
@ -78,10 +78,21 @@ export const MANIFEST_PATH = '_system/manifest.json'
|
|||
/**
|
||||
* The clean-shutdown marker (log-authority recovery gate): written+fsynced at
|
||||
* a clean close carrying the committed generation; CONSUMED at every open.
|
||||
* Absent or generation-mismatched at open = unclean shutdown = the whole-log
|
||||
* replay fold. Its absence is always safe (costs one replay, loses nothing).
|
||||
* Absent or generation-mismatched at open = unclean shutdown = the replay
|
||||
* fold, bounded below by the fold checkpoint when one is stored (whole-log
|
||||
* without one). Its absence is always safe (costs one fold, loses nothing).
|
||||
*/
|
||||
export const CLEAN_SHUTDOWN_PATH = '_system/clean-shutdown.json'
|
||||
/**
|
||||
* The fold checkpoint (log-authority recovery BOUND): `{ generation: G }`
|
||||
* asserts that every entity whose latest fact is ≤ G has durable canonical
|
||||
* bytes — so an unclean open folds only `(G, head]` instead of the whole log.
|
||||
* Stamped strictly AFTER a canonical-sync barrier over every live entity
|
||||
* touched since the last stamp (stamp-after-data); absent or torn = fold from
|
||||
* 0 (always safe, just bigger). The chain of stamps starts only at a provable
|
||||
* point: an empty brain, or the end of a whole-log fold.
|
||||
*/
|
||||
export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json'
|
||||
/** Storage-root-relative prefix of the per-generation record directories. */
|
||||
export const GENERATIONS_PREFIX = '_generations'
|
||||
|
||||
|
|
@ -219,6 +230,37 @@ export class GenerationStore {
|
|||
/** Compaction horizon — record-sets ≤ this are reclaimed. */
|
||||
private horizonGen = 0
|
||||
|
||||
/**
|
||||
* Fold-checkpoint accumulator: every entity whose CANONICAL live bytes were
|
||||
* (re)written since the last stamped checkpoint. Drained by
|
||||
* {@link advanceFoldCheckpointUnlocked} — synced first, stamped after; on a
|
||||
* failed barrier the drained ids merge back so the checkpoint can never
|
||||
* advance past unsynced bytes. Fed only while the chain is valid (see
|
||||
* {@link foldCheckpointChainValid}) so tree-authority brains never grow it.
|
||||
*/
|
||||
private checkpointDirtyNouns = new Set<string>()
|
||||
/** @see checkpointDirtyNouns — the verb half of the accumulator. */
|
||||
private checkpointDirtyVerbs = new Set<string>()
|
||||
/**
|
||||
* Whether the checkpoint chain is PROVABLY sound for this brain: true when
|
||||
* a stored checkpoint exists (induction), the brain opened empty (vacuous),
|
||||
* or a whole-log fold just re-applied every fact (base case). While false,
|
||||
* checkpoints are never stamped and the fold bound stays 0 — the honest
|
||||
* 10.0 contract, upgraded at the brain's first recovery fold.
|
||||
*/
|
||||
private foldCheckpointChainValid = false
|
||||
/** Last stamped fold-checkpoint generation (0 = none / fold from origin). */
|
||||
private foldCheckpoint = 0
|
||||
/**
|
||||
* Whether this brain's stored authority is the log — set from the stored
|
||||
* artifact at open, or by {@link completeFoldCheckpointBootstrap} when an
|
||||
* in-session adoption flips it. Checkpoints are only ever STAMPED under log
|
||||
* authority (the artifact bounds the log fold, which only log-authority
|
||||
* recovery runs); the dirty accumulator may fill slightly earlier, during
|
||||
* an adoption in flight (see {@link beginFoldCheckpointBootstrap}).
|
||||
*/
|
||||
private authorityIsLog = false
|
||||
|
||||
/**
|
||||
* Committed generations whose record dirs exist, stored as a SORTED, DISJOINT,
|
||||
* ascending list of INCLUSIVE `[start, end]` intervals (a run-length set).
|
||||
|
|
@ -552,6 +594,7 @@ export class GenerationStore {
|
|||
// drift machinery at open — same as group-commit recovery.
|
||||
const authority = await readLogAuthority(this.storage)
|
||||
if (authority.authority === 'log') {
|
||||
this.authorityIsLog = true
|
||||
// TWO REPLAY TIERS, gated by the clean-shutdown marker:
|
||||
//
|
||||
// (1) ABOVE-MANIFEST (always): an intact fact above the manifest is
|
||||
|
|
@ -574,8 +617,22 @@ export class GenerationStore {
|
|||
const cleanShutdown = await this.readCleanShutdownMarker()
|
||||
const orphans = await this.factLog.peekFactsAbove(this.committed)
|
||||
const uncleanOpen = cleanShutdown === null || cleanShutdown !== this.committed
|
||||
// FOLD-CHECKPOINT BOUND: a stored checkpoint G proves every entity
|
||||
// whose latest fact is ≤ G has durable canonical bytes (each stamp
|
||||
// followed a canonical-sync barrier), so the unclean fold only needs
|
||||
// (G, head] — entities untouched since G are already safe, entities
|
||||
// touched after G get their latest after-image re-applied. Absent or
|
||||
// invalid checkpoint = fold from 0 (the 10.0 whole-log contract).
|
||||
const checkpoint = await this.readFoldCheckpoint()
|
||||
const foldBound = checkpoint ?? 0
|
||||
// Chain validity: induction (a stored stamp), vacuous truth (an empty
|
||||
// brain has no bytes to assert), or — set below — the base case (a
|
||||
// whole-log fold re-applies and re-syncs every entity in the log).
|
||||
this.foldCheckpointChainValid = checkpoint !== null || this.committed === 0
|
||||
this.foldCheckpoint = foldBound
|
||||
if (uncleanOpen) this.foldCheckpointChainValid = true
|
||||
const factsToReplay = uncleanOpen
|
||||
? await this.factLog.peekFactsAbove(0)
|
||||
? await this.factLog.peekFactsAbove(foldBound)
|
||||
: orphans
|
||||
if (factsToReplay.length > 0) {
|
||||
let replayed = 0
|
||||
|
|
@ -587,6 +644,7 @@ export class GenerationStore {
|
|||
: { metadata: op.record.metadata, vector: op.record.vector }
|
||||
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
|
||||
else await this.storage.writeNounRaw(op.id, image)
|
||||
this.noteCheckpointDirty(op.kind, op.id)
|
||||
}
|
||||
replayed++
|
||||
if (fact.generation > this.committed) {
|
||||
|
|
@ -612,10 +670,21 @@ export class GenerationStore {
|
|||
await this.storage.syncRawObjects([MANIFEST_PATH])
|
||||
prodLog.warn(
|
||||
`[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` +
|
||||
`canonical (${uncleanOpen ? 'WHOLE-LOG fold — unclean shutdown' : 'above-manifest'}; ` +
|
||||
`committed at ${this.committed}) — an acked write is never lost`
|
||||
`canonical (${
|
||||
uncleanOpen
|
||||
? foldBound > 0
|
||||
? `BOUNDED fold above checkpoint ${foldBound} — unclean shutdown`
|
||||
: 'WHOLE-LOG fold — unclean shutdown'
|
||||
: 'above-manifest'
|
||||
}; committed at ${this.committed}) — an acked write is never lost`
|
||||
)
|
||||
}
|
||||
// A recovery fold re-applied (and the barrier below re-syncs) every
|
||||
// entity in (bound, head] — stamp the checkpoint at the new committed
|
||||
// watermark so the NEXT crash folds only its own tail. This is also
|
||||
// the chain's base case: the first whole-log fold of a pre-checkpoint
|
||||
// brain covers every entity in the log, so its stamp is total.
|
||||
if (uncleanOpen) await this.advanceFoldCheckpointUnlocked()
|
||||
// The marker is consumed: any session that can write invalidates it
|
||||
// at first commit (see the commit paths); a clean close re-writes it.
|
||||
await this.clearCleanShutdownMarker()
|
||||
|
|
@ -672,6 +741,12 @@ export class GenerationStore {
|
|||
await this.flushPendingSingleOps()
|
||||
this.storage.setGenerationBumpHook(undefined)
|
||||
await this.persistCounterNow()
|
||||
// Fold-checkpoint barrier BEFORE the clean-shutdown marker: entities that
|
||||
// reached the accumulator outside the pending tier (transact commits,
|
||||
// aborted-write restores) get their canonical bytes synced and the stamp
|
||||
// advanced, so the marker below never vouches for bytes the checkpoint
|
||||
// chain hasn't proven durable.
|
||||
await this.advanceFoldCheckpoint()
|
||||
// Clean-shutdown marker (log-authority recovery gate): everything above
|
||||
// is durable; stamp the committed generation so the next open can adopt
|
||||
// instead of folding the log. Written LAST — a crash before this line is
|
||||
|
|
@ -705,6 +780,131 @@ export class GenerationStore {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the fold checkpoint's generation, or `null` when absent, torn, or
|
||||
* implausible (> committed) — every invalid shape degrades to the safe
|
||||
* whole-log fold, never to a bound that could skip an acked write.
|
||||
*/
|
||||
private async readFoldCheckpoint(): Promise<number | null> {
|
||||
try {
|
||||
const raw = (await this.storage.readRawObject(FOLD_CHECKPOINT_PATH)) as {
|
||||
generation?: number
|
||||
} | null
|
||||
const gen = raw?.generation
|
||||
if (!Number.isSafeInteger(gen) || (gen as number) < 0) return null
|
||||
if ((gen as number) > this.committed) {
|
||||
prodLog.warn(
|
||||
`[GenerationStore] fold checkpoint ${gen} is ahead of the manifest ` +
|
||||
`(${this.committed}) — ignoring it; recovery folds the whole log`
|
||||
)
|
||||
return null
|
||||
}
|
||||
return gen as number
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that an entity's canonical live bytes were (re)written and are not
|
||||
* yet covered by a checkpoint stamp. Gated on chain validity so brains
|
||||
* without a sound chain (tree authority, or log authority before its first
|
||||
* recovery fold) never accumulate — they keep the fold-from-0 contract.
|
||||
*/
|
||||
private noteCheckpointDirty(kind: 'noun' | 'verb', id: string): void {
|
||||
if (!this.foldCheckpointChainValid) return
|
||||
if (kind === 'verb') this.checkpointDirtyVerbs.add(id)
|
||||
else this.checkpointDirtyNouns.add(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* The canonical-sync barrier + checkpoint stamp (must run under the commit
|
||||
* mutex or in single-threaded open). Drains the dirty accumulator, makes
|
||||
* those entities' canonical bytes durable via the adapter barrier, and only
|
||||
* THEN stamps `_system/fold-checkpoint.json` at the committed watermark —
|
||||
* stamp-after-data, always. On any failure the drained ids merge back and
|
||||
* the stored checkpoint stays where it was: the bound can lag (a bigger
|
||||
* fold later) but can never overstate durability (a lost write, outlawed).
|
||||
*/
|
||||
private async advanceFoldCheckpointUnlocked(): Promise<void> {
|
||||
if (!this.foldCheckpointChainValid || !this.authorityIsLog || !this.factLog) return
|
||||
const nouns = [...this.checkpointDirtyNouns]
|
||||
const verbs = [...this.checkpointDirtyVerbs]
|
||||
const target = this.committed
|
||||
if (nouns.length === 0 && verbs.length === 0 && target === this.foldCheckpoint) return
|
||||
this.checkpointDirtyNouns = new Set()
|
||||
this.checkpointDirtyVerbs = new Set()
|
||||
try {
|
||||
if (nouns.length > 0 || verbs.length > 0) {
|
||||
await this.storage.syncEntityCanonical?.(nouns, verbs)
|
||||
}
|
||||
await this.storage.writeRawObject(FOLD_CHECKPOINT_PATH, { generation: target })
|
||||
await this.storage.syncRawObjects([FOLD_CHECKPOINT_PATH])
|
||||
this.foldCheckpoint = target
|
||||
} catch (err) {
|
||||
for (const id of nouns) this.checkpointDirtyNouns.add(id)
|
||||
for (const id of verbs) this.checkpointDirtyVerbs.add(id)
|
||||
prodLog.warn(
|
||||
`[GenerationStore] fold-checkpoint barrier failed at generation ${target} ` +
|
||||
`(${(err as Error).message}) — checkpoint stays at ${this.foldCheckpoint}; ` +
|
||||
`recovery would fold from there (bigger, never lossy). Will retry next flush.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Public, mutex-serialized fold-checkpoint advance — called by
|
||||
* `close()` after the final flush so entities touched by paths that do not
|
||||
* ride the pending tier (e.g. `transact()`) are covered before the
|
||||
* clean-shutdown marker is written.
|
||||
*/
|
||||
async advanceFoldCheckpoint(): Promise<void> {
|
||||
return this.withMutex(() => this.advanceFoldCheckpointUnlocked())
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Adoption-time chain bootstrap, phase 1 — called by
|
||||
* `adoptLogAuthority()` BEFORE its oracle/backfill passes. Only a FRESH
|
||||
* brain (committed === 0) may bootstrap here: with no committed
|
||||
* generations the chain's assertion is vacuously true, and arming it now
|
||||
* means the baseline backfill's own re-commits feed the dirty accumulator,
|
||||
* so the first stamp after the flip covers them. A non-fresh flip skips
|
||||
* this (returns false) — its chain starts at the brain's first recovery
|
||||
* fold instead, because only a whole-log fold can prove coverage of
|
||||
* entities written before the log existed.
|
||||
*/
|
||||
beginFoldCheckpointBootstrap(): boolean {
|
||||
if (this.committed !== 0 || this.foldCheckpointChainValid) {
|
||||
return this.foldCheckpointChainValid
|
||||
}
|
||||
this.foldCheckpointChainValid = true
|
||||
this.foldCheckpoint = 0
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Adoption-time chain bootstrap, phase 2 — called after
|
||||
* `flipToLogAuthority` records the flip. Opens the stamp gate; the next
|
||||
* flush/close barrier writes the first checkpoint.
|
||||
*/
|
||||
completeFoldCheckpointBootstrap(): void {
|
||||
this.authorityIsLog = true
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Adoption-time chain bootstrap, abort — called when an
|
||||
* adoption attempt throws or refuses after phase 1. Disarms the chain and
|
||||
* drops the accumulator so a tree-authority brain never accumulates or
|
||||
* stamps. (If the chain was valid BEFORE the attempt — a stored checkpoint
|
||||
* exists — it stays valid; only a phase-1 arm is undone.)
|
||||
*/
|
||||
abandonFoldCheckpointBootstrap(): void {
|
||||
if (this.authorityIsLog) return
|
||||
this.foldCheckpointChainValid = false
|
||||
this.checkpointDirtyNouns = new Set()
|
||||
this.checkpointDirtyVerbs = new Set()
|
||||
}
|
||||
|
||||
/**
|
||||
* @description TEST-ONLY: install (or clear, with `undefined`) a fault
|
||||
* injector that is invoked at each {@link CommitFaultPhase} of the commit
|
||||
|
|
@ -1238,6 +1438,13 @@ export class GenerationStore {
|
|||
this.historyBytesTotal += delta.bytes ?? 0
|
||||
}
|
||||
this.extendChains(gen, nouns, verbs)
|
||||
// Fold-checkpoint accounting: the write barrier above already synced
|
||||
// this batch's canonical footprint on adapters that have one, but the
|
||||
// accumulator entry is the belt — an adapter without a write barrier
|
||||
// still gets these ids covered by the next checkpoint barrier, and a
|
||||
// redundant fsync of already-durable bytes is cheap and idempotent.
|
||||
for (const id of nouns) this.noteCheckpointDirty('noun', id)
|
||||
for (const id of verbs) this.noteCheckpointDirty('verb', id)
|
||||
const logEntry: TxLogEntry = { generation: gen, timestamp, ...(args.meta && { meta: args.meta }) }
|
||||
await this.storage.appendTxLogLine(JSON.stringify(logEntry))
|
||||
|
||||
|
|
@ -1249,6 +1456,13 @@ export class GenerationStore {
|
|||
if (crashSimulated) {
|
||||
throw err
|
||||
}
|
||||
// Fold-checkpoint accounting: an abort's rollback restores are raw
|
||||
// canonical writes that never reach the transaction write barrier
|
||||
// (flushWriteBarrier only runs on the commit path) — feed them so the
|
||||
// next checkpoint barrier syncs the restored bytes before any stamp
|
||||
// vouches for them.
|
||||
for (const id of nouns) this.noteCheckpointDirty('noun', id)
|
||||
for (const id of verbs) this.noteCheckpointDirty('verb', id)
|
||||
// The trapdoor for a batch: if rollback FAILED to fully apply, canonical
|
||||
// storage may be inconsistent. A batch is never adopted forward (its
|
||||
// other ops were rolled back — partial commit would break atomicity), so
|
||||
|
|
@ -1476,6 +1690,13 @@ export class GenerationStore {
|
|||
await args.execute()
|
||||
} catch (err) {
|
||||
this.inTransact = false
|
||||
// Fold-checkpoint accounting: execute() ran, so canonical bytes for
|
||||
// the touched ids changed — whether they now hold the new images, a
|
||||
// restored rollback, or (the trapdoor) something indeterminate, the
|
||||
// next checkpoint stamp must not assert their durability without a
|
||||
// barrier over whatever is actually there.
|
||||
for (const id of nouns) this.noteCheckpointDirty('noun', id)
|
||||
for (const id of verbs) this.noteCheckpointDirty('verb', id)
|
||||
// A failed rollback (TransactionRollbackError) may have left canonical
|
||||
// storage inconsistent — the trapdoor. Reconcile against the
|
||||
// before-images to decide the honest response (David's ruling:
|
||||
|
|
@ -1530,6 +1751,10 @@ export class GenerationStore {
|
|||
throw err
|
||||
}
|
||||
this.inTransact = false
|
||||
// Fold-checkpoint accounting: the live canonical write is applied — it
|
||||
// must ride the next canonical-sync barrier before any stamp covers it.
|
||||
for (const id of nouns) this.noteCheckpointDirty('noun', id)
|
||||
for (const id of verbs) this.noteCheckpointDirty('verb', id)
|
||||
// Test-only crash simulation (direct call — a throw propagates with no
|
||||
// cleanup, exactly like a process death; recovery-on-open restores the
|
||||
// contract). A crash here must cost only the never-returned ack: the
|
||||
|
|
@ -1801,6 +2026,16 @@ export class GenerationStore {
|
|||
for (const entry of logEntries) {
|
||||
await this.storage.appendTxLogLine(JSON.stringify(entry))
|
||||
}
|
||||
|
||||
// Fold-checkpoint barrier: the window's LIVE canonical bytes (the acked
|
||||
// writes themselves — the staging sync above covered only their history
|
||||
// copies) become durable here, and only then does the checkpoint stamp
|
||||
// advance to the new committed watermark. This is what keeps crash
|
||||
// recovery's log fold bounded to (checkpoint, head] instead of the
|
||||
// whole log. A failure inside is absorbed by the barrier (it warns,
|
||||
// retains the accumulator, and leaves the old bound standing) — history
|
||||
// durability above already succeeded, so the flush itself is good.
|
||||
await this.advanceFoldCheckpointUnlocked()
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -2961,6 +3196,8 @@ export class GenerationStore {
|
|||
private async rollBackUncommittedGeneration(gen: number): Promise<void> {
|
||||
const dir = `${GENERATIONS_PREFIX}/${gen}`
|
||||
const prevPaths = await this.storage.listRawObjects(`${dir}/prev`)
|
||||
const restoredNouns: string[] = []
|
||||
const restoredVerbs: string[] = []
|
||||
for (const recordPath of prevPaths) {
|
||||
const id = recordIdFromPath(recordPath)
|
||||
if (id === null) continue
|
||||
|
|
@ -2969,10 +3206,20 @@ export class GenerationStore {
|
|||
const image = { metadata: record.metadata, vector: record.vector }
|
||||
if (record.kind === 'verb') {
|
||||
await this.storage.writeVerbRaw(id, image)
|
||||
restoredVerbs.push(id)
|
||||
} else {
|
||||
await this.storage.writeNounRaw(id, image)
|
||||
restoredNouns.push(id)
|
||||
}
|
||||
}
|
||||
// Make the restores durable IMMEDIATELY (this runs at open, before the
|
||||
// fold-checkpoint chain state is even read): a restored before-image
|
||||
// replaces bytes a stored checkpoint may already vouch for, so it must
|
||||
// reach disk with the same certainty — otherwise a power cut could let
|
||||
// the rolled-back write's bytes resurrect past a bounded fold.
|
||||
if (restoredNouns.length > 0 || restoredVerbs.length > 0) {
|
||||
await this.storage.syncEntityCanonical?.(restoredNouns, restoredVerbs)
|
||||
}
|
||||
await this.storage.removeRawPrefix(dir)
|
||||
prodLog.warn(
|
||||
`[GenerationStore] rolled back uncommitted generation ${gen} ` +
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue