From a30ed72dc3c9866937960cd7b27936e1cd8ce6f7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 2 Jul 2026 13:35:43 -0700 Subject: [PATCH] fix(8.0): byte-copy _id_mapper/* in the pre-upgrade backup (cor's write-new nuance) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cor confirmed the migration write-new / tmp+rename invariant the hard-link backup relies on — with one exception: the native shared mmap `_id_mapper/*` is mutated IN PLACE (msync + a rebuild truncate+re-inject), the same category as the tx-log. A hard-linked snapshot of it would be corrupted by a live in-place mutation reaching through the shared inode, so the backup would not be a faithful pre-upgrade copy. Add a SNAPSHOT_BYTE_COPY_DIRS set (`_id_mapper`) so snapshotToDirectory byte-copies every file under it (the directory analogue of SNAPSHOT_BYTE_COPY_PATHS); the id map is bounded, so the copy cost is small. Everything else (SSTables / .dkann / manifests / objects, all tmp+rename) still hard-links. Test: `_id_mapper/*` gets an independent inode + faithful content; a regular file stays hard-linked. --- src/storage/adapters/fileSystemStorage.ts | 23 +++++++++++++++++- tests/integration/migration-backup.test.ts | 27 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index f1203417..a9a530c0 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -940,6 +940,20 @@ export class FileSystemStorage extends BaseStorage { `${SYSTEM_DIR}/tx-log.jsonl` ]) + /** + * Top-level directories whose EVERY file is mutated in place (not tmp+rename) + * and must therefore be byte-copied into a snapshot, not hard-linked — the + * directory analogue of {@link SNAPSHOT_BYTE_COPY_PATHS}. + * + * `_id_mapper` holds the native provider's shared mmap `BinaryIdMapper`, which + * `flush()` msyncs and a migration rebuild can truncate+re-inject IN PLACE + * (confirmed by the native side). A hard link shares the inode, so an in-place + * msync/truncate on the live file would reach through into the pre-upgrade + * backup — byte-copy keeps the snapshot a faithful, independent copy. It is + * bounded (the id map), not the large index files, so the copy cost is small. + */ + private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set(['_id_mapper']) + /** * Top-level directories excluded from snapshots: process-local lock state * (writer lock, flush-request RPC files) must never travel with the data. @@ -1081,7 +1095,14 @@ export class FileSystemStorage extends BaseStorage { // relative path with separators unified, so `_system/tx-log.jsonl` // matches on every platform. const normalized = relPath.split(path.sep).join('/') - if (FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized)) { + // Byte-copy (never hard-link) files that are mutated in place: the exact + // append-in-place paths, and every file under a mmap-mutated directory + // (e.g. the native `_id_mapper/*`). A shared inode would let a live + // msync/truncate reach through into the snapshot. + if ( + FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) || + FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) + ) { await fs.promises.copyFile(sourceFile, targetFile) continue } diff --git a/tests/integration/migration-backup.test.ts b/tests/integration/migration-backup.test.ts index 504d5eca..61b76b96 100644 --- a/tests/integration/migration-backup.test.ts +++ b/tests/integration/migration-backup.test.ts @@ -54,6 +54,33 @@ async function buildBrainWithData(dir: string, staleAfter = false): Promise { + it('byte-copies _id_mapper/* (mmap, mutated in place) but hard-links the rest', async () => { + // cor confirmed the native _id_mapper mmap is msync/truncate-in-place, so a + // hard-linked snapshot of it would be corrupted by a live mutation reaching + // through the shared inode — it must be byte-copied. Everything else (written + // tmp+rename) hard-links safely. + const dir = mkTmp() + await buildBrainWithData(dir) + fs.mkdirSync(path.join(dir, '_id_mapper'), { recursive: true }) + fs.writeFileSync(path.join(dir, '_id_mapper', 'map.bin'), Buffer.from([1, 2, 3, 4])) + fs.writeFileSync(path.join(dir, 'regular.bin'), Buffer.from([9, 9, 9, 9])) + + const storage: any = new FileSystemStorage(dir) + await storage.init() + const backup = await storage.createMigrationBackup() + + // _id_mapper file → byte-copied: DIFFERENT inode (a live msync can't reach it) + faithful content + const mapperLive = fs.statSync(path.join(dir, '_id_mapper', 'map.bin')).ino + const mapperBackup = fs.statSync(path.join(backup, '_id_mapper', 'map.bin')).ino + expect(mapperBackup).not.toBe(mapperLive) + expect(fs.readFileSync(path.join(backup, '_id_mapper', 'map.bin'))).toEqual(Buffer.from([1, 2, 3, 4])) + + // a regular (tmp+rename) file → hard-linked: SAME inode (zero-copy) + const regLive = fs.statSync(path.join(dir, 'regular.bin')).ino + const regBackup = fs.statSync(path.join(backup, 'regular.bin')).ino + expect(regBackup).toBe(regLive) + }) + it('FileSystemStorage.createMigrationBackup hard-links the store; removeMigrationBackup deletes it without touching live data', async () => { const dir = mkTmp() const id = await buildBrainWithData(dir)