fix(8.0): byte-copy _id_mapper/* in the pre-upgrade backup (cor's write-new nuance)
cor confirmed the migration write-new / tmp+rename invariant the hard-link backup relies on — with one exception: the native shared mmap `_id_mapper/*` is mutated IN PLACE (msync + a rebuild truncate+re-inject), the same category as the tx-log. A hard-linked snapshot of it would be corrupted by a live in-place mutation reaching through the shared inode, so the backup would not be a faithful pre-upgrade copy. Add a SNAPSHOT_BYTE_COPY_DIRS set (`_id_mapper`) so snapshotToDirectory byte-copies every file under it (the directory analogue of SNAPSHOT_BYTE_COPY_PATHS); the id map is bounded, so the copy cost is small. Everything else (SSTables / .dkann / manifests / objects, all tmp+rename) still hard-links. Test: `_id_mapper/*` gets an independent inode + faithful content; a regular file stays hard-linked.
This commit is contained in:
parent
79e8709c35
commit
a30ed72dc3
2 changed files with 49 additions and 1 deletions
|
|
@ -940,6 +940,20 @@ export class FileSystemStorage extends BaseStorage {
|
|||
`${SYSTEM_DIR}/tx-log.jsonl`
|
||||
])
|
||||
|
||||
/**
|
||||
* Top-level directories whose EVERY file is mutated in place (not tmp+rename)
|
||||
* and must therefore be byte-copied into a snapshot, not hard-linked — the
|
||||
* directory analogue of {@link SNAPSHOT_BYTE_COPY_PATHS}.
|
||||
*
|
||||
* `_id_mapper` holds the native provider's shared mmap `BinaryIdMapper`, which
|
||||
* `flush()` msyncs and a migration rebuild can truncate+re-inject IN PLACE
|
||||
* (confirmed by the native side). A hard link shares the inode, so an in-place
|
||||
* msync/truncate on the live file would reach through into the pre-upgrade
|
||||
* backup — byte-copy keeps the snapshot a faithful, independent copy. It is
|
||||
* bounded (the id map), not the large index files, so the copy cost is small.
|
||||
*/
|
||||
private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set<string>(['_id_mapper'])
|
||||
|
||||
/**
|
||||
* Top-level directories excluded from snapshots: process-local lock state
|
||||
* (writer lock, flush-request RPC files) must never travel with the data.
|
||||
|
|
@ -1081,7 +1095,14 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// relative path with separators unified, so `_system/tx-log.jsonl`
|
||||
// matches on every platform.
|
||||
const normalized = relPath.split(path.sep).join('/')
|
||||
if (FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized)) {
|
||||
// Byte-copy (never hard-link) files that are mutated in place: the exact
|
||||
// append-in-place paths, and every file under a mmap-mutated directory
|
||||
// (e.g. the native `_id_mapper/*`). A shared inode would let a live
|
||||
// msync/truncate reach through into the snapshot.
|
||||
if (
|
||||
FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) ||
|
||||
FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0])
|
||||
) {
|
||||
await fs.promises.copyFile(sourceFile, targetFile)
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,33 @@ async function buildBrainWithData(dir: string, staleAfter = false): Promise<stri
|
|||
}
|
||||
|
||||
describe('Pre-migration backup (#42)', () => {
|
||||
it('byte-copies _id_mapper/* (mmap, mutated in place) but hard-links the rest', async () => {
|
||||
// cor confirmed the native _id_mapper mmap is msync/truncate-in-place, so a
|
||||
// hard-linked snapshot of it would be corrupted by a live mutation reaching
|
||||
// through the shared inode — it must be byte-copied. Everything else (written
|
||||
// tmp+rename) hard-links safely.
|
||||
const dir = mkTmp()
|
||||
await buildBrainWithData(dir)
|
||||
fs.mkdirSync(path.join(dir, '_id_mapper'), { recursive: true })
|
||||
fs.writeFileSync(path.join(dir, '_id_mapper', 'map.bin'), Buffer.from([1, 2, 3, 4]))
|
||||
fs.writeFileSync(path.join(dir, 'regular.bin'), Buffer.from([9, 9, 9, 9]))
|
||||
|
||||
const storage: any = new FileSystemStorage(dir)
|
||||
await storage.init()
|
||||
const backup = await storage.createMigrationBackup()
|
||||
|
||||
// _id_mapper file → byte-copied: DIFFERENT inode (a live msync can't reach it) + faithful content
|
||||
const mapperLive = fs.statSync(path.join(dir, '_id_mapper', 'map.bin')).ino
|
||||
const mapperBackup = fs.statSync(path.join(backup, '_id_mapper', 'map.bin')).ino
|
||||
expect(mapperBackup).not.toBe(mapperLive)
|
||||
expect(fs.readFileSync(path.join(backup, '_id_mapper', 'map.bin'))).toEqual(Buffer.from([1, 2, 3, 4]))
|
||||
|
||||
// a regular (tmp+rename) file → hard-linked: SAME inode (zero-copy)
|
||||
const regLive = fs.statSync(path.join(dir, 'regular.bin')).ino
|
||||
const regBackup = fs.statSync(path.join(backup, 'regular.bin')).ino
|
||||
expect(regBackup).toBe(regLive)
|
||||
})
|
||||
|
||||
it('FileSystemStorage.createMigrationBackup hard-links the store; removeMigrationBackup deletes it without touching live data', async () => {
|
||||
const dir = mkTmp()
|
||||
const id = await buildBrainWithData(dir)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue