From a2f4f6a55039a5c4366f40bb2ab1228d49101cb1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 12 Jul 2026 09:10:35 -0700 Subject: [PATCH] fix: non-destructive, crash-resumable restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restore() removed the entire live brain directory and then fs.cp'd the snapshot in, so any copy failure left the store destroyed with only a partial copy. The sharpest edge: fs.cp materialized the holes of sparse mmap blob files, so a snapshot that fits on disk could balloon and ENOSPC mid-copy — the recovery tool destroying the brain it was asked to recover. Rewrite restoreFromDirectory as stage → verify → atomic swap: - Copy the snapshot into a _restore_staging area BEFORE touching live data, sparse-aware (copyTreeSparse/copyFileSparse skip all-zero 4 MiB chunks, so a mostly-hole store restores at its true allocated size, not its apparent size). - On any copy failure (ENOSPC included) remove only the half-written staging area and throw — the live store is left exactly as it was. - Only after the copy succeeds, fsync a completion marker naming the staged entries, then swapStagedRestoreIn(): an idempotent per-entry rm-old + rename-staged-in (same-filesystem renames that cannot ENOSPC), removing stale live entries the snapshot lacks. - completeInterruptedRestore(), wired into init() right after the root dir is ensured, finishes a crash mid-swap FORWARD (resume a committed swap) or discards an uncommitted staging area (live still authoritative). _restore_staging is excluded from snapshots. persist() (hard-link snapshot) was already safe and is unchanged; no public API change. Regression (tests/integration/restore-nondestructive.test.ts): a forced copy failure leaves live data fully intact and cleans staging; a normal restore round-trips to the snapshot; an interrupted-but-committed restore completes on reopen; an uncommitted staging area is discarded on open; the sparse copy is byte-identical with allocated blocks far below apparent size. --- src/storage/adapters/fileSystemStorage.ts | 233 +++++++++++++++++- .../restore-nondestructive.test.ts | 168 +++++++++++++ 2 files changed, 388 insertions(+), 13 deletions(-) create mode 100644 tests/integration/restore-nondestructive.test.ts diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index b0bc4ab7..e728cc41 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -223,6 +223,11 @@ export class FileSystemStorage extends BaseStorage { // Create the root directory if it doesn't exist await this.ensureDirectoryExists(this.rootDir) + // Finish any restore interrupted by a crash (resume the staged swap, or + // discard an uncommitted staging area) BEFORE counts/derived state load, + // so the rest of startup sees the completed store. + await this.completeInterruptedRestore() + // Create the nouns directory if it doesn't exist await this.ensureDirectoryExists(this.nounsDir) @@ -587,9 +592,30 @@ export class FileSystemStorage extends BaseStorage { /** * Top-level directories excluded from snapshots: process-local lock state - * (writer lock, flush-request RPC files) must never travel with the data. + * (writer lock, flush-request RPC files) must never travel with the data, and + * the restore staging area ({@link RESTORE_STAGING_DIR}) is transient scratch + * that must never be captured or restored. */ - private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set(['locks']) + private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set(['locks', '_restore_staging']) + + /** + * Transient top-level directory holding a restore-in-progress: the snapshot is + * fully copied here (sparse-aware) BEFORE any live data is touched, then an + * atomic per-entry swap moves it into place. Its presence + the completion + * marker let {@link completeInterruptedRestore} resume a crashed restore. + */ + private static readonly RESTORE_STAGING_DIR = '_restore_staging' + /** + * Written+fsync'd inside the staging dir ONLY after the whole snapshot has + * copied successfully. Its presence authorizes the swap (and its resume): a + * staging dir WITHOUT this marker is an interrupted copy — discardable debris, + * live data still authoritative. + */ + private static readonly RESTORE_MARKER = '.restore-manifest.json' + /** Chunk size for sparse-aware copying (holes are preserved at this grain). */ + private static readonly SPARSE_CHUNK_BYTES = 4 * 1024 * 1024 + /** A zero buffer the size of one sparse chunk, for all-zero (hole) detection. */ + private static readonly SPARSE_ZERO_CHUNK = Buffer.alloc(4 * 1024 * 1024) /** * Remove every object under a storage-root-relative prefix — one recursive @@ -899,6 +925,16 @@ export class FileSystemStorage extends BaseStorage { * * @param sourcePath - Absolute path of a directory produced by * {@link FileSystemStorage.snapshotToDirectory}. + * + * Non-destructive: the snapshot is copied into a staging area (sparse-aware, + * so a store of mostly-hole mmap blobs cannot balloon and ENOSPC) BEFORE any + * live data is touched. Only once the full copy has succeeded and a completion + * marker is fsync'd does an atomic per-entry swap move it into place. A copy + * failure (including ENOSPC) leaves the live store exactly as it was; a crash + * mid-swap is resumed forward on the next {@link init} by + * {@link completeInterruptedRestore}. The previous implementation removed the + * live store first and then `fs.cp`'d — a copy failure destroyed the brain it + * was meant to recover. */ public async restoreFromDirectory(sourcePath: string): Promise { await this.ensureInitialized() @@ -908,23 +944,194 @@ export class FileSystemStorage extends BaseStorage { throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`) } - // Clear current contents, preserving live lock state. - const currentEntries = await fs.promises.readdir(this.rootDir) - for (const entry of currentEntries) { + const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) + + // Discard any staging left by a prior aborted restore, then stage fresh. + await fs.promises.rm(staging, { recursive: true, force: true }) + await fs.promises.mkdir(staging, { recursive: true }) + + // Copy the snapshot into staging (sparse-aware). ANY failure here — most + // importantly ENOSPC — leaves the live store untouched: we remove only the + // half-written staging area and re-throw. + const staged: string[] = [] + try { + const sourceEntries = await fs.promises.readdir(sourcePath) + for (const entry of sourceEntries) { + if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue + await this.copyTreeSparse( + path.join(sourcePath, entry), + path.join(staging, entry) + ) + staged.push(entry) + } + // Commit point: fsync a marker naming the fully-staged entries. Only after + // this does the swap (and its resume) become authorized. + const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) + await fs.promises.writeFile(markerPath, JSON.stringify({ entries: staged })) + const mfh = await fs.promises.open(markerPath, 'r') + try { + await mfh.sync() + } finally { + await mfh.close() + } + const dfh = await fs.promises.open(staging, 'r').catch(() => null) + if (dfh) { + try { + await dfh.sync() + } catch { + // platform may reject directory fsync — best effort + } finally { + await dfh.close() + } + } + } catch (error: any) { + await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {}) + throw new Error( + `restoreFromDirectory: staging copy failed, live store left untouched: ${error.message}` + ) + } + + // Atomic per-entry swap (metadata-only renames within rootDir — cannot ENOSPC). + await this.swapStagedRestoreIn() + await this.reloadDerivedState() + } + + /** + * Move a fully-staged, marker-committed restore into place, then clear the + * staging area. Idempotent and resumable: driven by the marker's entry list + * and by which staged entries remain, so a crash at any point is completed by + * simply calling it again (from {@link completeInterruptedRestore} on the next + * open). Every step is a same-filesystem rename or a remove — no operation can + * fail for disk space, so once the marker exists the store is guaranteed to + * reach the restored state. + */ + private async swapStagedRestoreIn(): Promise { + const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) + const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) + + let staged: string[] + try { + const marker = JSON.parse(await fs.promises.readFile(markerPath, 'utf-8')) + staged = Array.isArray(marker?.entries) ? marker.entries : [] + } catch { + return // no committed marker — nothing to swap + } + const stagedSet = new Set(staged) + + // 1. Remove stale live entries the snapshot does not contain (excluding + // process-local dirs and the staging area itself). An already-placed + // staged entry is in stagedSet, so it is kept. + for (const entry of await fs.promises.readdir(this.rootDir)) { if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue + if (entry === FileSystemStorage.RESTORE_STAGING_DIR) continue + if (stagedSet.has(entry)) continue await fs.promises.rm(path.join(this.rootDir, entry), { recursive: true, force: true }) } - // Byte-copy the snapshot in (defensively skipping lock dirs in old snapshots). - const sourceEntries = await fs.promises.readdir(sourcePath) - for (const entry of sourceEntries) { - if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue - await fs.promises.cp(path.join(sourcePath, entry), path.join(this.rootDir, entry), { - recursive: true - }) + // 2. Place each staged entry (idempotent: a prior attempt that already moved + // it leaves staging/entry absent, so we skip). `rm` the old first — a + // rename onto an existing non-empty directory is not allowed; the staged + // copy is the durable source until it is placed, so a crash between the + // rm and the rename is recovered forward on the next call. + for (const entry of staged) { + const from = path.join(staging, entry) + const to = path.join(this.rootDir, entry) + const exists = await fs.promises.lstat(from).then(() => true, () => false) + if (!exists) continue + await fs.promises.rm(to, { recursive: true, force: true }) + await fs.promises.rename(from, to) } - await this.reloadDerivedState() + // 3. Clear the staging area (marker last-standing entry). + await fs.promises.rm(staging, { recursive: true, force: true }) + } + + /** + * On open, finish any restore interrupted by a crash. A staging dir WITH the + * completion marker means the copy had succeeded — resume the swap forward + * (loudly). A staging dir WITHOUT the marker is an interrupted copy — pure + * debris; the live store is authoritative, so discard it. Called from + * {@link init} before counts/derived state load, so recovery is invisible to + * the rest of startup. Returns `true` if a swap was resumed. + */ + private async completeInterruptedRestore(): Promise { + const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR) + const stagingStat = await fs.promises.stat(staging).catch(() => null) + if (!stagingStat || !stagingStat.isDirectory()) return false + + const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER) + const hasMarker = await fs.promises + .stat(markerPath) + .then(() => true, () => false) + + if (!hasMarker) { + // Interrupted before the copy committed — live data untouched, discard. + await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {}) + return false + } + + console.log('♻️ Resuming an interrupted restore (completing the staged swap)') + await this.swapStagedRestoreIn() + return true + } + + /** + * Recursively copy `src` to `dest`, preserving holes (sparse regions). Files + * are copied chunk-by-chunk skipping all-zero chunks, so a store of + * mostly-hole mmap blobs restores at its true allocated size instead of + * materializing every hole (the failure that made `fs.cp` ENOSPC a restore + * that would otherwise fit). Directories recurse; symlinks are recreated. + */ + private async copyTreeSparse(src: string, dest: string): Promise { + const stat = await fs.promises.lstat(src) + if (stat.isDirectory()) { + await fs.promises.mkdir(dest, { recursive: true }) + for (const child of await fs.promises.readdir(src)) { + await this.copyTreeSparse(path.join(src, child), path.join(dest, child)) + } + } else if (stat.isSymbolicLink()) { + await fs.promises.symlink(await fs.promises.readlink(src), dest) + } else if (stat.isFile()) { + await this.copyFileSparse(src, dest, stat.size, stat.mode) + } + // Other node types (sockets, devices) do not occur in a brain store. + } + + /** Sparse-aware single-file copy — see {@link copyTreeSparse}. */ + private async copyFileSparse( + src: string, + dest: string, + size: number, + mode: number + ): Promise { + const CHUNK = FileSystemStorage.SPARSE_CHUNK_BYTES + const srcFh = await fs.promises.open(src, 'r') + try { + const destFh = await fs.promises.open(dest, 'w', mode) + try { + // Pre-size the destination so unwritten regions are holes. + await destFh.truncate(size) + const buf = Buffer.allocUnsafe(CHUNK) + let pos = 0 + while (pos < size) { + const { bytesRead } = await srcFh.read(buf, 0, CHUNK, pos) + if (bytesRead === 0) break + const chunk = buf.subarray(0, bytesRead) + // Skip all-zero chunks: leaving them unwritten preserves the hole. + const isHole = chunk.equals( + FileSystemStorage.SPARSE_ZERO_CHUNK.subarray(0, bytesRead) + ) + if (!isHole) { + await destFh.write(chunk, 0, bytesRead, pos) + } + pos += bytesRead + } + } finally { + await destFh.close() + } + } finally { + await srcFh.close() + } } // =========================================================================== diff --git a/tests/integration/restore-nondestructive.test.ts b/tests/integration/restore-nondestructive.test.ts new file mode 100644 index 00000000..b36d2620 --- /dev/null +++ b/tests/integration/restore-nondestructive.test.ts @@ -0,0 +1,168 @@ +/** + * @module tests/integration/restore-nondestructive + * @description Regression for a consumer-reported recovery hazard: `restore()` + * removed the entire live brain directory and THEN `fs.cp`'d the snapshot in, so + * a copy failure (most dangerously ENOSPC — `fs.cp` also materializes the holes + * of sparse mmap blobs, ballooning a snapshot that would otherwise fit) left the + * store destroyed with only a partial copy: the recovery tool could destroy the + * brain it was meant to recover. + * + * Fix: the snapshot is copied into a staging area (sparse-aware) BEFORE any live + * data is touched; only after the copy succeeds and a marker is fsync'd does an + * atomic per-entry swap move it into place. A copy failure leaves the live store + * exactly as it was; a crash mid-swap is resumed forward on the next open. + * + * These tests exercise the property directly: a forced copy failure leaves live + * data intact, a normal restore round-trips correctly, an interrupted-but- + * committed staging area is completed on reopen, and the sparse copy preserves + * holes byte-for-byte. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const STAGING = '_restore_staging' +const MARKER = '.restore-manifest.json' + +describe('non-destructive restore (BRAINY-RESTORE-DESTRUCTIVE)', () => { + let root: string + let snap: string + let brain: any + + const openBrain = async (dir: string) => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-restore-')) + root = path.join(base, 'store') + snap = path.join(base, 'snapshot') + brain = await openBrain(root) + }) + + afterEach(async () => { + try { await brain.close() } catch { /* already closed in a test */ } + fs.rmSync(path.dirname(root), { recursive: true, force: true }) + }) + + it('a copy failure leaves the LIVE store completely intact (the core property)', async () => { + const keep = await brain.add({ id: '00000000-0000-4000-8000-0000000000c1', data: 'in snapshot', type: NounType.Thing }) + await brain.flush() + await brain['storage'].snapshotToDirectory(snap) + // Mutate live AFTER the snapshot — this state must survive a failed restore. + const extra = await brain.add({ id: '00000000-0000-4000-8000-0000000000c2', data: 'live only', type: NounType.Thing }) + + // Force the staging copy to fail on its second entry (partial staging). + const storage = brain['storage'] + const realCopy = storage.copyTreeSparse.bind(storage) + let calls = 0 + storage.copyTreeSparse = async (src: string, dest: string) => { + if (++calls === 2) throw new Error('simulated ENOSPC') + return realCopy(src, dest) + } + + await expect(brain.restore(snap, { confirm: true })).rejects.toThrow(/untouched/) + + // Live data — BOTH the snapshot entity and the live-only mutation — survives. + expect(await brain.get(keep)).not.toBeNull() + expect(await brain.get(extra)).not.toBeNull() + // The half-written staging area was cleaned up. + expect(fs.existsSync(path.join(root, STAGING))).toBe(false) + }) + + it('a normal restore round-trips to the snapshot state', async () => { + const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c3', data: 'A', type: NounType.Thing }) + await brain.flush() + await brain['storage'].snapshotToDirectory(snap) + // Diverge from the snapshot: add B, remove A. + const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c4', data: 'B', type: NounType.Thing }) + await brain.remove(a) + expect(await brain.get(a)).toBeNull() + expect(await brain.get(b)).not.toBeNull() + + await brain.restore(snap, { confirm: true }) + + // Back to exactly the snapshot: A present, B gone. + expect(await brain.get(a)).not.toBeNull() + expect(await brain.get(b)).toBeNull() + expect(fs.existsSync(path.join(root, STAGING))).toBe(false) + }) + + it('an interrupted-but-committed restore is completed on the next open (resume forward)', async () => { + const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c5', data: 'A', type: NounType.Thing }) + await brain.flush() + await brain['storage'].snapshotToDirectory(snap) + const b = await brain.add({ id: '00000000-0000-4000-8000-0000000000c6', data: 'B', type: NounType.Thing }) + await brain.flush() + await brain.close() + + // Simulate the state right after the staging copy committed but before the + // swap ran: a _restore_staging holding the snapshot's entries + the marker. + const staging = path.join(root, STAGING) + fs.mkdirSync(staging, { recursive: true }) + const entries: string[] = [] + for (const entry of fs.readdirSync(snap)) { + if (entry === 'locks' || entry === STAGING) continue + fs.cpSync(path.join(snap, entry), path.join(staging, entry), { recursive: true }) + entries.push(entry) + } + fs.writeFileSync(path.join(staging, MARKER), JSON.stringify({ entries })) + + // Reopen → init resumes the swap → the store is the snapshot (A, no B). + brain = await openBrain(root) + expect(await brain.get(a)).not.toBeNull() + expect(await brain.get(b)).toBeNull() + expect(fs.existsSync(staging)).toBe(false) + }) + + it('an uncommitted staging area (no marker) is discarded on open, live untouched', async () => { + const a = await brain.add({ id: '00000000-0000-4000-8000-0000000000c7', data: 'A live', type: NounType.Thing }) + await brain.flush() + await brain.close() + + // A staging dir with NO marker = a copy that never committed → debris. + const staging = path.join(root, STAGING) + fs.mkdirSync(staging, { recursive: true }) + fs.writeFileSync(path.join(staging, 'entities'), 'garbage partial copy') + + brain = await openBrain(root) + expect(await brain.get(a)).not.toBeNull() // live authoritative + expect(fs.existsSync(staging)).toBe(false) // debris removed + }) + + it('the sparse copy preserves holes: content byte-identical, allocation far below apparent size', async () => { + const base = path.dirname(root) + const src = path.join(base, 'sparse-src.bin') + const dest = path.join(base, 'sparse-dest.bin') + const SIZE = 32 * 1024 * 1024 // 32 MiB apparent + const DATA = Buffer.from('hello sparse world') + + // Create a sparse source: 32 MiB of holes with a little real data at the start. + const fh = fs.openSync(src, 'w') + fs.ftruncateSync(fh, SIZE) + fs.writeSync(fh, DATA, 0, DATA.length, 0) + fs.closeSync(fh) + + const storage = brain['storage'] + const stat = fs.statSync(src) + await storage.copyFileSparse(src, dest, stat.size, stat.mode) + + // Same apparent size and byte-identical content… + const destStat = fs.statSync(dest) + expect(destStat.size).toBe(SIZE) + expect(fs.readFileSync(dest).equals(fs.readFileSync(src))).toBe(true) + // …but the destination is actually sparse (allocated bytes far below size). + expect(destStat.blocks * 512).toBeLessThan(SIZE / 2) + }) +})