fix: non-destructive, crash-resumable restore
restore() removed the entire live brain directory and then fs.cp'd the snapshot in, so any copy failure left the store destroyed with only a partial copy. The sharpest edge: fs.cp materialized the holes of sparse mmap blob files, so a snapshot that fits on disk could balloon and ENOSPC mid-copy — the recovery tool destroying the brain it was asked to recover. Rewrite restoreFromDirectory as stage → verify → atomic swap: - Copy the snapshot into a _restore_staging area BEFORE touching live data, sparse-aware (copyTreeSparse/copyFileSparse skip all-zero 4 MiB chunks, so a mostly-hole store restores at its true allocated size, not its apparent size). - On any copy failure (ENOSPC included) remove only the half-written staging area and throw — the live store is left exactly as it was. - Only after the copy succeeds, fsync a completion marker naming the staged entries, then swapStagedRestoreIn(): an idempotent per-entry rm-old + rename-staged-in (same-filesystem renames that cannot ENOSPC), removing stale live entries the snapshot lacks. - completeInterruptedRestore(), wired into init() right after the root dir is ensured, finishes a crash mid-swap FORWARD (resume a committed swap) or discards an uncommitted staging area (live still authoritative). _restore_staging is excluded from snapshots. persist() (hard-link snapshot) was already safe and is unchanged; no public API change. Regression (tests/integration/restore-nondestructive.test.ts): a forced copy failure leaves live data fully intact and cleans staging; a normal restore round-trips to the snapshot; an interrupted-but-committed restore completes on reopen; an uncommitted staging area is discarded on open; the sparse copy is byte-identical with allocated blocks far below apparent size.
This commit is contained in:
parent
b3e8d47d46
commit
a2f4f6a550
2 changed files with 388 additions and 13 deletions
|
|
@ -223,6 +223,11 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// Create the root directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.rootDir)
|
||||
|
||||
// Finish any restore interrupted by a crash (resume the staged swap, or
|
||||
// discard an uncommitted staging area) BEFORE counts/derived state load,
|
||||
// so the rest of startup sees the completed store.
|
||||
await this.completeInterruptedRestore()
|
||||
|
||||
// Create the nouns directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
|
||||
|
|
@ -587,9 +592,30 @@ export class FileSystemStorage extends BaseStorage {
|
|||
|
||||
/**
|
||||
* Top-level directories excluded from snapshots: process-local lock state
|
||||
* (writer lock, flush-request RPC files) must never travel with the data.
|
||||
* (writer lock, flush-request RPC files) must never travel with the data, and
|
||||
* the restore staging area ({@link RESTORE_STAGING_DIR}) is transient scratch
|
||||
* that must never be captured or restored.
|
||||
*/
|
||||
private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set<string>(['locks'])
|
||||
private static readonly SNAPSHOT_EXCLUDED_TOP_DIRS = new Set<string>(['locks', '_restore_staging'])
|
||||
|
||||
/**
|
||||
* Transient top-level directory holding a restore-in-progress: the snapshot is
|
||||
* fully copied here (sparse-aware) BEFORE any live data is touched, then an
|
||||
* atomic per-entry swap moves it into place. Its presence + the completion
|
||||
* marker let {@link completeInterruptedRestore} resume a crashed restore.
|
||||
*/
|
||||
private static readonly RESTORE_STAGING_DIR = '_restore_staging'
|
||||
/**
|
||||
* Written+fsync'd inside the staging dir ONLY after the whole snapshot has
|
||||
* copied successfully. Its presence authorizes the swap (and its resume): a
|
||||
* staging dir WITHOUT this marker is an interrupted copy — discardable debris,
|
||||
* live data still authoritative.
|
||||
*/
|
||||
private static readonly RESTORE_MARKER = '.restore-manifest.json'
|
||||
/** Chunk size for sparse-aware copying (holes are preserved at this grain). */
|
||||
private static readonly SPARSE_CHUNK_BYTES = 4 * 1024 * 1024
|
||||
/** A zero buffer the size of one sparse chunk, for all-zero (hole) detection. */
|
||||
private static readonly SPARSE_ZERO_CHUNK = Buffer.alloc(4 * 1024 * 1024)
|
||||
|
||||
/**
|
||||
* Remove every object under a storage-root-relative prefix — one recursive
|
||||
|
|
@ -899,6 +925,16 @@ export class FileSystemStorage extends BaseStorage {
|
|||
*
|
||||
* @param sourcePath - Absolute path of a directory produced by
|
||||
* {@link FileSystemStorage.snapshotToDirectory}.
|
||||
*
|
||||
* Non-destructive: the snapshot is copied into a staging area (sparse-aware,
|
||||
* so a store of mostly-hole mmap blobs cannot balloon and ENOSPC) BEFORE any
|
||||
* live data is touched. Only once the full copy has succeeded and a completion
|
||||
* marker is fsync'd does an atomic per-entry swap move it into place. A copy
|
||||
* failure (including ENOSPC) leaves the live store exactly as it was; a crash
|
||||
* mid-swap is resumed forward on the next {@link init} by
|
||||
* {@link completeInterruptedRestore}. The previous implementation removed the
|
||||
* live store first and then `fs.cp`'d — a copy failure destroyed the brain it
|
||||
* was meant to recover.
|
||||
*/
|
||||
public async restoreFromDirectory(sourcePath: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -908,23 +944,194 @@ export class FileSystemStorage extends BaseStorage {
|
|||
throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`)
|
||||
}
|
||||
|
||||
// Clear current contents, preserving live lock state.
|
||||
const currentEntries = await fs.promises.readdir(this.rootDir)
|
||||
for (const entry of currentEntries) {
|
||||
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
|
||||
|
||||
// Discard any staging left by a prior aborted restore, then stage fresh.
|
||||
await fs.promises.rm(staging, { recursive: true, force: true })
|
||||
await fs.promises.mkdir(staging, { recursive: true })
|
||||
|
||||
// Copy the snapshot into staging (sparse-aware). ANY failure here — most
|
||||
// importantly ENOSPC — leaves the live store untouched: we remove only the
|
||||
// half-written staging area and re-throw.
|
||||
const staged: string[] = []
|
||||
try {
|
||||
const sourceEntries = await fs.promises.readdir(sourcePath)
|
||||
for (const entry of sourceEntries) {
|
||||
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
|
||||
await this.copyTreeSparse(
|
||||
path.join(sourcePath, entry),
|
||||
path.join(staging, entry)
|
||||
)
|
||||
staged.push(entry)
|
||||
}
|
||||
// Commit point: fsync a marker naming the fully-staged entries. Only after
|
||||
// this does the swap (and its resume) become authorized.
|
||||
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
|
||||
await fs.promises.writeFile(markerPath, JSON.stringify({ entries: staged }))
|
||||
const mfh = await fs.promises.open(markerPath, 'r')
|
||||
try {
|
||||
await mfh.sync()
|
||||
} finally {
|
||||
await mfh.close()
|
||||
}
|
||||
const dfh = await fs.promises.open(staging, 'r').catch(() => null)
|
||||
if (dfh) {
|
||||
try {
|
||||
await dfh.sync()
|
||||
} catch {
|
||||
// platform may reject directory fsync — best effort
|
||||
} finally {
|
||||
await dfh.close()
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
|
||||
throw new Error(
|
||||
`restoreFromDirectory: staging copy failed, live store left untouched: ${error.message}`
|
||||
)
|
||||
}
|
||||
|
||||
// Atomic per-entry swap (metadata-only renames within rootDir — cannot ENOSPC).
|
||||
await this.swapStagedRestoreIn()
|
||||
await this.reloadDerivedState()
|
||||
}
|
||||
|
||||
/**
|
||||
* Move a fully-staged, marker-committed restore into place, then clear the
|
||||
* staging area. Idempotent and resumable: driven by the marker's entry list
|
||||
* and by which staged entries remain, so a crash at any point is completed by
|
||||
* simply calling it again (from {@link completeInterruptedRestore} on the next
|
||||
* open). Every step is a same-filesystem rename or a remove — no operation can
|
||||
* fail for disk space, so once the marker exists the store is guaranteed to
|
||||
* reach the restored state.
|
||||
*/
|
||||
private async swapStagedRestoreIn(): Promise<void> {
|
||||
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
|
||||
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
|
||||
|
||||
let staged: string[]
|
||||
try {
|
||||
const marker = JSON.parse(await fs.promises.readFile(markerPath, 'utf-8'))
|
||||
staged = Array.isArray(marker?.entries) ? marker.entries : []
|
||||
} catch {
|
||||
return // no committed marker — nothing to swap
|
||||
}
|
||||
const stagedSet = new Set(staged)
|
||||
|
||||
// 1. Remove stale live entries the snapshot does not contain (excluding
|
||||
// process-local dirs and the staging area itself). An already-placed
|
||||
// staged entry is in stagedSet, so it is kept.
|
||||
for (const entry of await fs.promises.readdir(this.rootDir)) {
|
||||
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
|
||||
if (entry === FileSystemStorage.RESTORE_STAGING_DIR) continue
|
||||
if (stagedSet.has(entry)) continue
|
||||
await fs.promises.rm(path.join(this.rootDir, entry), { recursive: true, force: true })
|
||||
}
|
||||
|
||||
// Byte-copy the snapshot in (defensively skipping lock dirs in old snapshots).
|
||||
const sourceEntries = await fs.promises.readdir(sourcePath)
|
||||
for (const entry of sourceEntries) {
|
||||
if (FileSystemStorage.SNAPSHOT_EXCLUDED_TOP_DIRS.has(entry)) continue
|
||||
await fs.promises.cp(path.join(sourcePath, entry), path.join(this.rootDir, entry), {
|
||||
recursive: true
|
||||
})
|
||||
// 2. Place each staged entry (idempotent: a prior attempt that already moved
|
||||
// it leaves staging/entry absent, so we skip). `rm` the old first — a
|
||||
// rename onto an existing non-empty directory is not allowed; the staged
|
||||
// copy is the durable source until it is placed, so a crash between the
|
||||
// rm and the rename is recovered forward on the next call.
|
||||
for (const entry of staged) {
|
||||
const from = path.join(staging, entry)
|
||||
const to = path.join(this.rootDir, entry)
|
||||
const exists = await fs.promises.lstat(from).then(() => true, () => false)
|
||||
if (!exists) continue
|
||||
await fs.promises.rm(to, { recursive: true, force: true })
|
||||
await fs.promises.rename(from, to)
|
||||
}
|
||||
|
||||
await this.reloadDerivedState()
|
||||
// 3. Clear the staging area (marker last-standing entry).
|
||||
await fs.promises.rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
/**
|
||||
* On open, finish any restore interrupted by a crash. A staging dir WITH the
|
||||
* completion marker means the copy had succeeded — resume the swap forward
|
||||
* (loudly). A staging dir WITHOUT the marker is an interrupted copy — pure
|
||||
* debris; the live store is authoritative, so discard it. Called from
|
||||
* {@link init} before counts/derived state load, so recovery is invisible to
|
||||
* the rest of startup. Returns `true` if a swap was resumed.
|
||||
*/
|
||||
private async completeInterruptedRestore(): Promise<boolean> {
|
||||
const staging = path.join(this.rootDir, FileSystemStorage.RESTORE_STAGING_DIR)
|
||||
const stagingStat = await fs.promises.stat(staging).catch(() => null)
|
||||
if (!stagingStat || !stagingStat.isDirectory()) return false
|
||||
|
||||
const markerPath = path.join(staging, FileSystemStorage.RESTORE_MARKER)
|
||||
const hasMarker = await fs.promises
|
||||
.stat(markerPath)
|
||||
.then(() => true, () => false)
|
||||
|
||||
if (!hasMarker) {
|
||||
// Interrupted before the copy committed — live data untouched, discard.
|
||||
await fs.promises.rm(staging, { recursive: true, force: true }).catch(() => {})
|
||||
return false
|
||||
}
|
||||
|
||||
console.log('♻️ Resuming an interrupted restore (completing the staged swap)')
|
||||
await this.swapStagedRestoreIn()
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively copy `src` to `dest`, preserving holes (sparse regions). Files
|
||||
* are copied chunk-by-chunk skipping all-zero chunks, so a store of
|
||||
* mostly-hole mmap blobs restores at its true allocated size instead of
|
||||
* materializing every hole (the failure that made `fs.cp` ENOSPC a restore
|
||||
* that would otherwise fit). Directories recurse; symlinks are recreated.
|
||||
*/
|
||||
private async copyTreeSparse(src: string, dest: string): Promise<void> {
|
||||
const stat = await fs.promises.lstat(src)
|
||||
if (stat.isDirectory()) {
|
||||
await fs.promises.mkdir(dest, { recursive: true })
|
||||
for (const child of await fs.promises.readdir(src)) {
|
||||
await this.copyTreeSparse(path.join(src, child), path.join(dest, child))
|
||||
}
|
||||
} else if (stat.isSymbolicLink()) {
|
||||
await fs.promises.symlink(await fs.promises.readlink(src), dest)
|
||||
} else if (stat.isFile()) {
|
||||
await this.copyFileSparse(src, dest, stat.size, stat.mode)
|
||||
}
|
||||
// Other node types (sockets, devices) do not occur in a brain store.
|
||||
}
|
||||
|
||||
/** Sparse-aware single-file copy — see {@link copyTreeSparse}. */
|
||||
private async copyFileSparse(
|
||||
src: string,
|
||||
dest: string,
|
||||
size: number,
|
||||
mode: number
|
||||
): Promise<void> {
|
||||
const CHUNK = FileSystemStorage.SPARSE_CHUNK_BYTES
|
||||
const srcFh = await fs.promises.open(src, 'r')
|
||||
try {
|
||||
const destFh = await fs.promises.open(dest, 'w', mode)
|
||||
try {
|
||||
// Pre-size the destination so unwritten regions are holes.
|
||||
await destFh.truncate(size)
|
||||
const buf = Buffer.allocUnsafe(CHUNK)
|
||||
let pos = 0
|
||||
while (pos < size) {
|
||||
const { bytesRead } = await srcFh.read(buf, 0, CHUNK, pos)
|
||||
if (bytesRead === 0) break
|
||||
const chunk = buf.subarray(0, bytesRead)
|
||||
// Skip all-zero chunks: leaving them unwritten preserves the hole.
|
||||
const isHole = chunk.equals(
|
||||
FileSystemStorage.SPARSE_ZERO_CHUNK.subarray(0, bytesRead)
|
||||
)
|
||||
if (!isHole) {
|
||||
await destFh.write(chunk, 0, bytesRead, pos)
|
||||
}
|
||||
pos += bytesRead
|
||||
}
|
||||
} finally {
|
||||
await destFh.close()
|
||||
}
|
||||
} finally {
|
||||
await srcFh.close()
|
||||
}
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue