fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it
All checks were successful
CI / Node 22 (push) Successful in 12m13s
CI / Node 24 (push) Successful in 12m8s
CI / Bun (latest) (push) Successful in 12m20s

Two defects with one root, found by the fold-checkpoint work's first
integration gate. (1) THE RACE: restore() never quiesced the generation
store, so a background flush could write into _system/ while the swap was
removing it — observed as ENOTEMPTY mid-swap when a checkpoint stamp landed
between readdir and rmdir. The swap now runs inside the store's exclusive
section (runStateReplacement): flush timer disarmed, pending tier and
checkpoint accumulator discarded BEFORE any directory moves. (2) THE
INHERITED ASSERTION: a snapshot carries its source brain's clean-shutdown
marker and fold checkpoint, but the restored files were bulk-copied without
per-file fsync — the inherited stamps would suppress exactly the recovery
fold that cures a post-restore power cut. reopenAfterRestore now deletes
both stamps before reopening: the open treats the store as uncleanly shut,
folds the restored log into canonical, barrier-syncs what it re-applied,
and stamps fresh — the restored state is durably founded at restore time
instead of borrowing assertions about bytes this disk never synced.

Pinned: restore under in-flight traffic completes; the pre-restore stamp
does not survive; the post-restore stamp is the reopen fold's own, at the
restored watermark.
This commit is contained in:
David Snelling 2026-08-13 09:19:14 -07:00
parent ff43de1ada
commit 9ca80667c3
3 changed files with 85 additions and 1 deletions

View file

@ -9229,7 +9229,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
} }
const floorGeneration = this.generationStore.generation() const floorGeneration = this.generationStore.generation()
await this.storage.restoreFromDirectory(path) // The swap runs inside the generation store's exclusive section: pending
// flush timers are disarmed and buffers discarded BEFORE any directory is
// removed, so a background flush can never write into `_system/` mid-swap
// (the ENOTEMPTY race a checkpoint stamp once hit).
await this.generationStore.runStateReplacement(() =>
this.storage.restoreFromDirectory(path)
)
await this.generationStore.reopenAfterRestore(floorGeneration) await this.generationStore.reopenAfterRestore(floorGeneration)
// If the entity-id mapper is a NATIVE provider with a `rebuild()`, reload it // If the entity-id mapper is a NATIVE provider with a `rebuild()`, reload it

View file

@ -3127,6 +3127,28 @@ export class GenerationStore {
* are never reissued. * are never reissued.
* @param floorGeneration - The counter value before the restore. * @param floorGeneration - The counter value before the restore.
*/ */
/**
* @description Run a wholesale state replacement (the restore swap)
* EXCLUSIVELY: under the commit mutex, with the pending flush timer
* disarmed and the pending tier + fold-checkpoint accumulator discarded
* FIRST so no background flush can write into `_system/` while the
* replacement is removing and swapping directories. Observed without this:
* a checkpoint stamp raced restore's directory removal and the swap died
* ENOTEMPTY mid-flight. The discarded in-memory state describes the store
* being replaced `reopenAfterRestore` (which the caller runs next)
* rebuilds everything from the restored bytes.
*/
async runStateReplacement(replace: () => Promise<void>): Promise<void> {
return this.withMutex(async () => {
this.clearPendingFlushTimer()
this.pendingGens = []
this.pendingBuffer.clear()
this.checkpointDirtyNouns = new Set()
this.checkpointDirtyVerbs = new Set()
await replace()
})
}
async reopenAfterRestore(floorGeneration: number): Promise<void> { async reopenAfterRestore(floorGeneration: number): Promise<void> {
await this.withMutex(async () => { await this.withMutex(async () => {
this.deltaCache.clear() this.deltaCache.clear()
@ -3139,6 +3161,27 @@ export class GenerationStore {
this.clearPendingFlushTimer() this.clearPendingFlushTimer()
this.pendingGens = [] this.pendingGens = []
this.pendingBuffer.clear() this.pendingBuffer.clear()
// The fold-checkpoint accumulator described the replaced state too.
this.checkpointDirtyNouns = new Set()
this.checkpointDirtyVerbs = new Set()
this.foldCheckpointChainValid = false
this.foldCheckpoint = 0
// A RESTORE IS AN UNCLEAN EVENT, by construction: the snapshot's files
// were just bulk-copied WITHOUT per-file fsync, so a power cut here can
// tear them — yet the snapshot may CARRY the source brain's
// clean-shutdown marker and fold checkpoint, which would together
// suppress exactly the recovery fold that cures such a tear. Delete
// both BEFORE reopening: the open below then treats the store as
// uncleanly shut, folds the restored log into canonical, barrier-syncs
// what it re-applied, and stamps a FRESH checkpoint — the restored
// state becomes durably founded at restore time instead of inheriting
// the source brain's assertions about bytes this disk never synced.
try {
await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH)
} catch { /* absent is fine — same outcome */ }
try {
await this.storage.deleteRawObject(FOLD_CHECKPOINT_PATH)
} catch { /* absent is fine — fold from 0 */ }
this.opened = false this.opened = false
// open() re-reads counter/manifest and re-registers the bump hook. // open() re-reads counter/manifest and re-registers the bump hook.
await this.open() await this.open()

View file

@ -172,6 +172,41 @@ describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], nev
expect(readCheckpoint(dir)).toBeNull() expect(readCheckpoint(dir)).toBeNull()
}, 120000) }, 120000)
it('restore is an UNCLEAN event: the snapshots stamps do not survive — the reopen fold re-founds and re-stamps the restored state', async () => {
const dir = trackDir()
const brain = await openBrain(dir, { logAuthority: 'adopt' })
liveBrains.push(brain)
const idA = await brain.add({ data: 'survives the restore', type: NounType.Document, metadata: { n: 1 } })
await brain.flush()
const snapDir = join(trackDir(), 'snap')
const db = brain.now()
await (db as unknown as { persist(p: string): Promise<void> }).persist(snapDir)
await (db as unknown as { release(): Promise<void> }).release()
// Advance the live brain past the snapshot: a later write, a later flush,
// a later checkpoint stamp — none of which may survive the restore.
const idB = await brain.add({ data: 'must not survive', type: NounType.Document, metadata: { n: 2 } })
await brain.flush()
const stampBeforeRestore = readCheckpoint(dir)
expect(stampBeforeRestore).toBe(committedOf(brain))
// Unflushed traffic in flight at restore time — the quiesced swap discards
// it under the mutex instead of letting its flush timer race the swap
// (the ENOTEMPTY class).
await brain.add({ data: 'in-flight at restore', type: NounType.Document, metadata: { n: 3 } })
await brain.restore(snapDir, { confirm: true })
expect(await brain.get(idA), 'snapshot state restored').not.toBeNull()
expect(await brain.get(idB), 'post-snapshot state replaced').toBeNull()
// The stamp on disk is the REOPEN FOLD's fresh assertion about the
// restored (and now barrier-synced) bytes — at the restored watermark,
// strictly below the pre-restore stamp that must not survive.
const stampAfterRestore = readCheckpoint(dir)
expect(stampAfterRestore).toBe(committedOf(brain))
expect(stampAfterRestore!).toBeLessThan(stampBeforeRestore!)
}, 120000)
it('a delete rides the barrier: the tombstoned id is in the synced set and the stamp advances past it', async () => { it('a delete rides the barrier: the tombstoned id is in the synced set and the stamp advances past it', async () => {
const dir = trackDir() const dir = trackDir()
const brain = await openBrain(dir, { logAuthority: 'adopt' }) const brain = await openBrain(dir, { logAuthority: 'adopt' })