fix: non-destructive, crash-resumable restore

restore() removed the entire live brain directory and then fs.cp'd the
snapshot in, so any copy failure left the store destroyed with only a partial
copy. The sharpest edge: fs.cp materialized the holes of sparse mmap blob
files, so a snapshot that fits on disk could balloon and ENOSPC mid-copy — the
recovery tool destroying the brain it was asked to recover.

Rewrite restoreFromDirectory as stage → verify → atomic swap:
- Copy the snapshot into a _restore_staging area BEFORE touching live data,
  sparse-aware (copyTreeSparse/copyFileSparse skip all-zero 4 MiB chunks, so a
  mostly-hole store restores at its true allocated size, not its apparent size).
- On any copy failure (ENOSPC included) remove only the half-written staging
  area and throw — the live store is left exactly as it was.
- Only after the copy succeeds, fsync a completion marker naming the staged
  entries, then swapStagedRestoreIn(): an idempotent per-entry
  rm-old + rename-staged-in (same-filesystem renames that cannot ENOSPC),
  removing stale live entries the snapshot lacks.
- completeInterruptedRestore(), wired into init() right after the root dir is
  ensured, finishes a crash mid-swap FORWARD (resume a committed swap) or
  discards an uncommitted staging area (live still authoritative).

_restore_staging is excluded from snapshots. persist() (hard-link snapshot)
was already safe and is unchanged; no public API change.

Regression (tests/integration/restore-nondestructive.test.ts): a forced copy
failure leaves live data fully intact and cleans staging; a normal restore
round-trips to the snapshot; an interrupted-but-committed restore completes on
reopen; an uncommitted staging area is discarded on open; the sparse copy is
byte-identical with allocated blocks far below apparent size.
This commit is contained in:
David Snelling 2026-07-12 09:10:35 -07:00
parent b3e8d47d46
commit a2f4f6a550
2 changed files with 388 additions and 13 deletions

View file

@ -223,6 +223,11 @@ export class FileSystemStorage extends BaseStorage {
// Create the root directory if it doesn't exist
await this.ensureDirectoryExists(this.rootDir)
// Finish any restore interrupted by a crash (resume the staged swap, or
// discard an uncommitted staging area) BEFORE counts/derived state load,
// so the rest of startup sees the completed store.
await this.completeInterruptedRestore()
// Create the nouns directory if it doesn't exist
await this.ensureDirectoryExists(this.nounsDir)
@ -587,9 +592,30 @@ export class FileSystemStorage extends BaseStorage {
/**
* Top-level directories excluded from snapshots: process-local lock state
* (writer lock, flush-request RPC files) must never travel with the data.
* (writer lock, flush-request RPC files) must never travel with the data, and
* the restore staging area ({@link RESTORE_STAGING_DIR}) is transient scratch
* that must never be captured or restored.
*/
private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set<string>(['locks'])
private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set<string>(['locks', '_restore_staging'])
/**
* Transient top-level directory holding a restore-in-progress: the snapshot is
* fully copied here (sparse-aware) BEFORE any live data is touched, then an
* atomic per-entry swap moves it into place. Its presence + the completion
* marker let {@link completeInterruptedRestore} resume a crashed restore.
*/
private static readonly RESTORE_STAGING_DIR = '_restore_staging'
/**
* Written+fsync'd inside the staging dir ONLY after the whole snapshot has
* copied successfully. Its presence authorizes the swap (and its resume): a
* staging dir WITHOUT this marker is an interrupted copy discardable debris,
* live data still authoritative.
*/
private static readonly RESTORE_MARKER = '.restore-manifest.json'
/** Chunk size for sparse-aware copying (holes are preserved at this grain). */
private static readonly SPARSE_CHUNK_BYTES = 4 * 1024 * 1024
/** A zero buffer the size of one sparse chunk, for all-zero (hole) detection. */
private static readonly SPARSE_ZERO_CHUNK = Buffer.alloc(4 * 1024 * 1024)
/**
* Remove every object under a storage-root-relative prefix one recursive
@ -899,6 +925,16 @@ export class FileSystemStorage extends BaseStorage {
*
* @param sourcePath - Absolute path of a directory produced by
* {@link FileSystemStorage.snapshotToDirectory}.
*
* Non-destructive: the snapshot is copied into a staging area (sparse-aware,
* so a store of mostly-hole mmap blobs cannot balloon and ENOSPC) BEFORE any
* live data is touched. Only once the full copy has succeeded and a completion
* marker is fsync'd does an atomic per-entry swap move it into place. A copy
* failure (including ENOSPC) leaves the live store exactly as it was; a crash
* mid-swap is resumed forward on the next {@link init} by
* {@link completeInterruptedRestore}. The previous implementation removed the
* live store first and then `fs.cp`'d a copy failure destroyed the brain it
* was meant to recover.
*/
public async restoreFromDirectory(sourcePath: string): Promise<void> {
await this.ensureInitialized()
@ -908,23 +944,194 @@ export class FileSystemStorage extends BaseStorage {
throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`)
}
// Clear current contents, preserving live lock state.
const currentEntries = await fs.promises.readdir(this.rootDir)
for (const entry of currentEntries) {
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
// Discard any staging left by a prior aborted restore, then stage fresh.
await fs.promises.rm(staging, { recursive: true, force: true })
await fs.promises.mkdir(staging, { recursive: true })
// Copy the snapshot into staging (sparse-aware). ANY failure here — most
// importantly ENOSPC — leaves the live store untouched: we remove only the
// half-written staging area and re-throw.
const staged: string[] = []
try {
const sourceEntries = await fs.promises.readdir(sourcePath)
for (const entry of sourceEntries) {
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
await this.copyTreeSparse(
path.join(sourcePath, entry),
path.join(staging, entry)
)
staged.push(entry)
}
// Commit point: fsync a marker naming the fully-staged entries. Only after
// this does the swap (and its resume) become authorized.
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
await fs.promises.writeFile(markerPath, JSON.stringify({ entries: staged }))
const mfh = await fs.promises.open(markerPath, 'r')
try {
await mfh.sync()
} finally {
await mfh.close()
}
const dfh = await fs.promises.open(staging, 'r').catch(() => null)
if (dfh) {
try {
await dfh.sync()
} catch {
// platform may reject directory fsync — best effort
} finally {
await dfh.close()
}
}
} catch (error: any) {
await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
throw new Error(
`restoreFromDirectory: staging copy failed, live store left untouched: ${error.message}`
)
}
// Atomic per-entry swap (metadata-only renames within rootDir — cannot ENOSPC).
await this.swapStagedRestoreIn()
await this.reloadDerivedState()
}
/**
* Move a fully-staged, marker-committed restore into place, then clear the
* staging area. Idempotent and resumable: driven by the marker's entry list
* and by which staged entries remain, so a crash at any point is completed by
* simply calling it again (from {@link completeInterruptedRestore} on the next
* open). Every step is a same-filesystem rename or a remove no operation can
* fail for disk space, so once the marker exists the store is guaranteed to
* reach the restored state.
*/
private async swapStagedRestoreIn(): Promise<void> {
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
let staged: string[]
try {
const marker = JSON.parse(await fs.promises.readFile(markerPath, 'utf-8'))
staged = Array.isArray(marker?.entries) ? marker.entries : []
} catch {
return // no committed marker — nothing to swap
}
const stagedSet = new Set(staged)
// 1. Remove stale live entries the snapshot does not contain (excluding
// process-local dirs and the staging area itself). An already-placed
// staged entry is in stagedSet, so it is kept.
for (const entry of await fs.promises.readdir(this.rootDir)) {
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
if (entry === FileSystemStorage.RESTORE_STAGING_DIR) continue
if (stagedSet.has(entry)) continue
await fs.promises.rm(path.join(this.rootDir, entry), { recursive: true, force: true })
}
// Byte-copy the snapshot in (defensively skipping lock dirs in old snapshots).
const sourceEntries = await fs.promises.readdir(sourcePath)
for (const entry of sourceEntries) {
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
await fs.promises.cp(path.join(sourcePath, entry), path.join(this.rootDir, entry), {
recursive: true
})
// 2. Place each staged entry (idempotent: a prior attempt that already moved
// it leaves staging/entry absent, so we skip). `rm` the old first — a
// rename onto an existing non-empty directory is not allowed; the staged
// copy is the durable source until it is placed, so a crash between the
// rm and the rename is recovered forward on the next call.
for (const entry of staged) {
const from = path.join(staging, entry)
const to = path.join(this.rootDir, entry)
const exists = await fs.promises.lstat(from).then(() => true, () => false)
if (!exists) continue
await fs.promises.rm(to, { recursive: true, force: true })
await fs.promises.rename(from, to)
}
await this.reloadDerivedState()
// 3. Clear the staging area (marker last-standing entry).
await fs.promises.rm(staging, { recursive: true, force: true })
}
/**
* On open, finish any restore interrupted by a crash. A staging dir WITH the
* completion marker means the copy had succeeded resume the swap forward
* (loudly). A staging dir WITHOUT the marker is an interrupted copy pure
* debris; the live store is authoritative, so discard it. Called from
* {@link init} before counts/derived state load, so recovery is invisible to
* the rest of startup. Returns `true` if a swap was resumed.
*/
private async completeInterruptedRestore(): Promise<boolean> {
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
const stagingStat = await fs.promises.stat(staging).catch(() => null)
if (!stagingStat || !stagingStat.isDirectory()) return false
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
const hasMarker = await fs.promises
.stat(markerPath)
.then(() => true, () => false)
if (!hasMarker) {
// Interrupted before the copy committed — live data untouched, discard.
await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
return false
}
console.log('♻️ Resuming an interrupted restore (completing the staged swap)')
await this.swapStagedRestoreIn()
return true
}
/**
* Recursively copy `src` to `dest`, preserving holes (sparse regions). Files
* are copied chunk-by-chunk skipping all-zero chunks, so a store of
* mostly-hole mmap blobs restores at its true allocated size instead of
* materializing every hole (the failure that made `fs.cp` ENOSPC a restore
* that would otherwise fit). Directories recurse; symlinks are recreated.
*/
private async copyTreeSparse(src: string, dest: string): Promise<void> {
const stat = await fs.promises.lstat(src)
if (stat.isDirectory()) {
await fs.promises.mkdir(dest, { recursive: true })
for (const child of await fs.promises.readdir(src)) {
await this.copyTreeSparse(path.join(src, child), path.join(dest, child))
}
} else if (stat.isSymbolicLink()) {
await fs.promises.symlink(await fs.promises.readlink(src), dest)
} else if (stat.isFile()) {
await this.copyFileSparse(src, dest, stat.size, stat.mode)
}
// Other node types (sockets, devices) do not occur in a brain store.
}
/** Sparse-aware single-file copy — see {@link copyTreeSparse}. */
private async copyFileSparse(
src: string,
dest: string,
size: number,
mode: number
): Promise<void> {
const CHUNK = FileSystemStorage.SPARSE_CHUNK_BYTES
const srcFh = await fs.promises.open(src, 'r')
try {
const destFh = await fs.promises.open(dest, 'w', mode)
try {
// Pre-size the destination so unwritten regions are holes.
await destFh.truncate(size)
const buf = Buffer.allocUnsafe(CHUNK)
let pos = 0
while (pos < size) {
const { bytesRead } = await srcFh.read(buf, 0, CHUNK, pos)
if (bytesRead === 0) break
const chunk = buf.subarray(0, bytesRead)
// Skip all-zero chunks: leaving them unwritten preserves the hole.
const isHole = chunk.equals(
FileSystemStorage.SPARSE_ZERO_CHUNK.subarray(0, bytesRead)
)
if (!isHole) {
await destFh.write(chunk, 0, bytesRead, pos)
}
pos += bytesRead
}
} finally {
await destFh.close()
}
} finally {
await srcFh.close()
}
}
// ===========================================================================

View file

@ -0,0 +1,168 @@
/**
* @module tests/integration/restore-nondestructive
* @description Regression for a consumer-reported recovery hazard: `restore()`
* removed the entire live brain directory and THEN `fs.cp`'d the snapshot in, so
* a copy failure (most dangerously ENOSPC `fs.cp` also materializes the holes
* of sparse mmap blobs, ballooning a snapshot that would otherwise fit) left the
* store destroyed with only a partial copy: the recovery tool could destroy the
* brain it was meant to recover.
*
* Fix: the snapshot is copied into a staging area (sparse-aware) BEFORE any live
* data is touched; only after the copy succeeds and a marker is fsync'd does an
* atomic per-entry swap move it into place. A copy failure leaves the live store
* exactly as it was; a crash mid-swap is resumed forward on the next open.
*
* These tests exercise the property directly: a forced copy failure leaves live
* data intact, a normal restore round-trips correctly, an interrupted-but-
* committed staging area is completed on reopen, and the sparse copy preserves
* holes byte-for-byte.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
const STAGING = '_restore_staging'
const MARKER = '.restore-manifest.json'
describe('non-destructive restore (BRAINY-RESTORE-DESTRUCTIVE)', () => {
let root: string
let snap: string
let brain: any
const openBrain = async (dir: string) => {
const b = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
dimensions: 384,
silent: true
})
await b.init()
return b
}
beforeEach(async () => {
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
const base = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-restore-'))
root = path.join(base, 'store')
snap = path.join(base, 'snapshot')
brain = await openBrain(root)
})
afterEach(async () => {
try { await brain.close() } catch { /* already closed in a test */ }
fs.rmSync(path.dirname(root), { recursive: true, force: true })
})
it('a copy failure leaves the LIVE store completely intact (the core property)', async () => {
const keep = await brain.add({ id: '00000000-0000-4000-8000-0000000000c1', data: 'in snapshot', type: NounType.Thing })
await brain.flush()
await brain['storage'].snapshotToDirectory(snap)
// Mutate live AFTER the snapshot — this state must survive a failed restore.
const extra = await brain.add({ id: '00000000-0000-4000-8000-0000000000c2', data: 'live only', type: NounType.Thing })
// Force the staging copy to fail on its second entry (partial staging).
const storage = brain['storage']
const realCopy = storage.copyTreeSparse.bind(storage)
let calls = 0
storage.copyTreeSparse = async (src: string, dest: string) => {
if (++calls === 2) throw new Error('simulated ENOSPC')
return realCopy(src, dest)
}
await expect(brain.restore(snap, { confirm: true })).rejects.toThrow(/untouched/)
// Live data — BOTH the snapshot entity and the live-only mutation — survives.
expect(await brain.get(keep)).not.toBeNull()
expect(await brain.get(extra)).not.toBeNull()
// The half-written staging area was cleaned up.
expect(fs.existsSync(path.join(root, STAGING))).toBe(false)
})
it('a normal restore round-trips to the snapshot state', async () => {
const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c3', data: 'A', type: NounType.Thing })
await brain.flush()
await brain['storage'].snapshotToDirectory(snap)
// Diverge from the snapshot: add B, remove A.
const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c4', data: 'B', type: NounType.Thing })
await brain.remove(a)
expect(await brain.get(a)).toBeNull()
expect(await brain.get(b)).not.toBeNull()
await brain.restore(snap, { confirm: true })
// Back to exactly the snapshot: A present, B gone.
expect(await brain.get(a)).not.toBeNull()
expect(await brain.get(b)).toBeNull()
expect(fs.existsSync(path.join(root, STAGING))).toBe(false)
})
it('an interrupted-but-committed restore is completed on the next open (resume forward)', async () => {
const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c5', data: 'A', type: NounType.Thing })
await brain.flush()
await brain['storage'].snapshotToDirectory(snap)
const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c6', data: 'B', type: NounType.Thing })
await brain.flush()
await brain.close()
// Simulate the state right after the staging copy committed but before the
// swap ran: a _restore_staging holding the snapshot's entries + the marker.
const staging = path.join(root, STAGING)
fs.mkdirSync(staging, { recursive: true })
const entries: string[] = []
for (const entry of fs.readdirSync(snap)) {
if (entry === 'locks' || entry === STAGING) continue
fs.cpSync(path.join(snap, entry), path.join(staging, entry), { recursive: true })
entries.push(entry)
}
fs.writeFileSync(path.join(staging, MARKER), JSON.stringify({ entries }))
// Reopen → init resumes the swap → the store is the snapshot (A, no B).
brain = await openBrain(root)
expect(await brain.get(a)).not.toBeNull()
expect(await brain.get(b)).toBeNull()
expect(fs.existsSync(staging)).toBe(false)
})
it('an uncommitted staging area (no marker) is discarded on open, live untouched', async () => {
const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c7', data: 'A live', type: NounType.Thing })
await brain.flush()
await brain.close()
// A staging dir with NO marker = a copy that never committed → debris.
const staging = path.join(root, STAGING)
fs.mkdirSync(staging, { recursive: true })
fs.writeFileSync(path.join(staging, 'entities'), 'garbage partial copy')
brain = await openBrain(root)
expect(await brain.get(a)).not.toBeNull() // live authoritative
expect(fs.existsSync(staging)).toBe(false) // debris removed
})
it('the sparse copy preserves holes: content byte-identical, allocation far below apparent size', async () => {
const base = path.dirname(root)
const src = path.join(base, 'sparse-src.bin')
const dest = path.join(base, 'sparse-dest.bin')
const SIZE = 32 * 1024 * 1024 // 32 MiB apparent
const DATA = Buffer.from('hello sparse world')
// Create a sparse source: 32 MiB of holes with a little real data at the start.
const fh = fs.openSync(src, 'w')
fs.ftruncateSync(fh, SIZE)
fs.writeSync(fh, DATA, 0, DATA.length, 0)
fs.closeSync(fh)
const storage = brain['storage']
const stat = fs.statSync(src)
await storage.copyFileSparse(src, dest, stat.size, stat.mode)
// Same apparent size and byte-identical content…
const destStat = fs.statSync(dest)
expect(destStat.size).toBe(SIZE)
expect(fs.readFileSync(dest).equals(fs.readFileSync(src))).toBe(true)
// …but the destination is actually sparse (allocated bytes far below size).
expect(destStat.blocks * 512).toBeLessThan(SIZE / 2)
})
})