fix(8.0): byte-copy _id_mapper/* in the pre-upgrade backup (cor's write-new nuance)

cor confirmed the migration write-new / tmp+rename invariant the hard-link backup
relies on — with one exception: the native shared mmap `_id_mapper/*` is mutated
IN PLACE (msync + a rebuild truncate+re-inject), the same category as the tx-log.
A hard-linked snapshot of it would be corrupted by a live in-place mutation
reaching through the shared inode, so the backup would not be a faithful
pre-upgrade copy.

Add a SNAPSHOT_BYTE_COPY_DIRS set (`_id_mapper`) so snapshotToDirectory byte-copies
every file under it (the directory analogue of SNAPSHOT_BYTE_COPY_PATHS); the id
map is bounded, so the copy cost is small. Everything else (SSTables / .dkann /
manifests / objects, all tmp+rename) still hard-links. Test: `_id_mapper/*` gets
an independent inode + faithful content; a regular file stays hard-linked.
This commit is contained in:
David Snelling 2026-07-02 13:35:43 -07:00
parent 79e8709c35
commit a30ed72dc3
2 changed files with 49 additions and 1 deletions

View file

@ -940,6 +940,20 @@ export class FileSystemStorage extends BaseStorage {
`${SYSTEM_DIR}/tx-log.jsonl` `${SYSTEM_DIR}/tx-log.jsonl`
]) ])
/**
* Top-level directories whose EVERY file is mutated in place (not tmp+rename)
* and must therefore be byte-copied into a snapshot, not hard-linked the
* directory analogue of {@link SNAPSHOT_BYTE_COPY_PATHS}.
*
* `_id_mapper` holds the native provider's shared mmap `BinaryIdMapper`, which
* `flush()` msyncs and a migration rebuild can truncate+re-inject IN PLACE
* (confirmed by the native side). A hard link shares the inode, so an in-place
* msync/truncate on the live file would reach through into the pre-upgrade
* backup byte-copy keeps the snapshot a faithful, independent copy. It is
* bounded (the id map), not the large index files, so the copy cost is small.
*/
private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set<string>(['_id_mapper'])
/** /**
* Top-level directories excluded from snapshots: process-local lock state * Top-level directories excluded from snapshots: process-local lock state
* (writer lock, flush-request RPC files) must never travel with the data. * (writer lock, flush-request RPC files) must never travel with the data.
@ -1081,7 +1095,14 @@ export class FileSystemStorage extends BaseStorage {
// relative path with separators unified, so `_system/tx-log.jsonl` // relative path with separators unified, so `_system/tx-log.jsonl`
// matches on every platform. // matches on every platform.
const normalized = relPath.split(path.sep).join('/') const normalized = relPath.split(path.sep).join('/')
if (FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized)) { // Byte-copy (never hard-link) files that are mutated in place: the exact
// append-in-place paths, and every file under a mmap-mutated directory
// (e.g. the native `_id_mapper/*`). A shared inode would let a live
// msync/truncate reach through into the snapshot.
if (
FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) ||
FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0])
) {
await fs.promises.copyFile(sourceFile, targetFile) await fs.promises.copyFile(sourceFile, targetFile)
continue continue
} }

View file

@ -54,6 +54,33 @@ async function buildBrainWithData(dir: string, staleAfter = false): Promise<stri
} }
describe('Pre-migration backup (#42)', () => { describe('Pre-migration backup (#42)', () => {
it('byte-copies _id_mapper/* (mmap, mutated in place) but hard-links the rest', async () => {
// cor confirmed the native _id_mapper mmap is msync/truncate-in-place, so a
// hard-linked snapshot of it would be corrupted by a live mutation reaching
// through the shared inode — it must be byte-copied. Everything else (written
// tmp+rename) hard-links safely.
const dir = mkTmp()
await buildBrainWithData(dir)
fs.mkdirSync(path.join(dir, '_id_mapper'), { recursive: true })
fs.writeFileSync(path.join(dir, '_id_mapper', 'map.bin'), Buffer.from([1, 2, 3, 4]))
fs.writeFileSync(path.join(dir, 'regular.bin'), Buffer.from([9, 9, 9, 9]))
const storage: any = new FileSystemStorage(dir)
await storage.init()
const backup = await storage.createMigrationBackup()
// _id_mapper file → byte-copied: DIFFERENT inode (a live msync can't reach it) + faithful content
const mapperLive = fs.statSync(path.join(dir, '_id_mapper', 'map.bin')).ino
const mapperBackup = fs.statSync(path.join(backup, '_id_mapper', 'map.bin')).ino
expect(mapperBackup).not.toBe(mapperLive)
expect(fs.readFileSync(path.join(backup, '_id_mapper', 'map.bin'))).toEqual(Buffer.from([1, 2, 3, 4]))
// a regular (tmp+rename) file → hard-linked: SAME inode (zero-copy)
const regLive = fs.statSync(path.join(dir, 'regular.bin')).ino
const regBackup = fs.statSync(path.join(backup, 'regular.bin')).ino
expect(regBackup).toBe(regLive)
})
it('FileSystemStorage.createMigrationBackup hard-links the store; removeMigrationBackup deletes it without touching live data', async () => { it('FileSystemStorage.createMigrationBackup hard-links the store; removeMigrationBackup deletes it without touching live data', async () => {
const dir = mkTmp() const dir = mkTmp()
const id = await buildBrainWithData(dir) const id = await buildBrainWithData(dir)