diff --git a/src/brainy.ts b/src/brainy.ts index efa0516f..e3630e56 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -432,6 +432,10 @@ export class Brainy implements BrainyInterface { * not migrating — the fallback `elapsedMs` a provider that omits `migrationStatus()` * timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */ private _migrationObservedAt: number | null = null + /** Path of the pre-upgrade backup taken this open (default-on, opt-out via + * `migrationBackup: false`), or `null` if none. Removed once the 7.x→8.0 + * upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */ + private _migrationBackupPath: string | null = null /** * 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat} * marker (`_system/brain-format.json`) as read during the store-open phase, @@ -856,6 +860,16 @@ export class Brainy implements BrainyInterface { this._indexEpochStale = this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH + // Pre-upgrade backup (default-on, opt-out via `migrationBackup: false`): the + // on-disk format is stale, so a one-time 7.x → 8.0 rebuild will run below. + // Snapshot the brain dir NOW — before any provider (native or JS) rebuilds a + // derived index — so a migration bug can be rolled back. Removed once the + // upgrade verifies + stamps; retained on failure. No-op for a reader, for + // non-filesystem storage, or for a brain with no persisted data. + if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { + await this.createMigrationBackupIfNeeded() + } + // Provider: embeddings (reassign embedder if plugin provides one) const embeddingProvider = this.pluginRegistry.getProvider('embeddings') if (embeddingProvider) { @@ -6489,6 +6503,9 @@ export class Brainy implements BrainyInterface { await writeBrainFormat(this.storage) this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } this._indexEpochStale = false + // The upgrade has verified + stamped — drop the pre-upgrade backup (no-op if + // none was taken, e.g. a non-migrating stamp of a fresh brain's initial marker). + await this.removeMigrationBackupSafe() } /** @@ -12991,6 +13008,8 @@ export class Brainy implements BrainyInterface { // Migration LOCK wait budget — left undefined when omitted so // awaitMigrationLock() applies its 30 s default at the read site. migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined, + // Pre-upgrade backup — default-on; opt out with `migrationBackup: false`. + migrationBackup: config?.migrationBackup ?? true, // Vector index configuration (8.0) — algorithm-neutral surface with // `recall` preset + `persistMode` (folded in from 7.x's hnswPersistMode). // See BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2. @@ -13400,6 +13419,8 @@ export class Brainy implements BrainyInterface { await writeBrainFormat(this.storage) this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } this._indexEpochStale = false + // The JS inline rebuild verified + stamped — drop the pre-upgrade backup. + await this.removeMigrationBackupSafe() } /** @@ -13560,6 +13581,61 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Take the pre-upgrade backup (default-on) before a native provider + * rebuilds anything for a 7.x → 8.0 migration. Called at open, once the on-disk + * format is known stale, BEFORE the providers init. Feature-detected + best-effort: + * a backend without `createMigrationBackup` (memory) or a store with no data is a + * no-op, and a backup failure NEVER blocks the upgrade (the migration is itself + * safe — it only reads canonical, and self-heals on re-open). The snapshot is + * removed once the upgrade verifies ({@link removeMigrationBackupSafe}) and + * retained on failure for rollback. + */ + private async createMigrationBackupIfNeeded(): Promise { + const storage = this.storage as StorageAdapter & { + createMigrationBackup?: () => Promise + } + if (typeof storage.createMigrationBackup !== 'function') return + try { + const backupPath = await storage.createMigrationBackup() + if (backupPath) { + this._migrationBackupPath = backupPath + prodLog.info( + `[Brainy] Pre-upgrade backup created at ${backupPath} (hard-link snapshot; ~zero ` + + `cost/space). Removed automatically once the 7.x→8.0 upgrade verifies; retained ` + + `for rollback if it fails. Opt out with { migrationBackup: false }.` + ) + } + } catch (error) { + prodLog.warn( + `[Brainy] Pre-upgrade backup could not be created (continuing — the upgrade is safe ` + + `and self-heals): ${error instanceof Error ? error.message : String(error)}` + ) + } + } + + /** + * @description Remove the pre-upgrade backup once the migration has verified and + * stamped the format marker. Called from both stamp paths (the JS inline rebuild + * and the native provider's `stampBrainFormat()`), gated on a backup having been + * taken this open. Best-effort: a removal failure leaves a harmless backup dir. + */ + private async removeMigrationBackupSafe(): Promise { + if (!this._migrationBackupPath) return + const backupPath = this._migrationBackupPath + this._migrationBackupPath = null + const storage = this.storage as StorageAdapter & { + removeMigrationBackup?: (location: string) => Promise + } + if (typeof storage.removeMigrationBackup !== 'function') return + try { + await storage.removeMigrationBackup(backupPath) + prodLog.info(`[Brainy] Upgrade verified — pre-upgrade backup at ${backupPath} removed.`) + } catch { + // best-effort — a leftover backup dir is harmless + } + } + /** * Check health of metadata indexes * diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 45caebb3..cce06e8a 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -1138,4 +1138,20 @@ export interface StorageAdapter { * @returns Promise that resolves to the total number of verbs */ getVerbCount(): Promise + + /** + * OPTIONAL — create a pre-upgrade backup of the whole store and return its + * location, or `null` when there is nothing to back up (empty store). On the + * filesystem adapter this is a zero-copy hard-link snapshot into a sibling + * directory; other backends may omit it. Callers feature-detect. Paired with + * {@link removeMigrationBackup}. Used by the 7.x → 8.0 migration lifecycle. + */ + createMigrationBackup?(): Promise + + /** + * OPTIONAL — remove a backup created by {@link createMigrationBackup}. + * Best-effort: a missing `location` is a no-op. Removing a hard-link snapshot + * never affects the live store (shared inodes; only the extra links are gone). + */ + removeMigrationBackup?(location: string): Promise } diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index dcb326cc..f1203417 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1106,6 +1106,56 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * @description Pre-upgrade backup: a hard-link snapshot of the whole store into + * a SIBLING directory (`.migration-backup`, outside `rootDir` so the + * snapshot never recurses into itself), taken before a 7.x → 8.0 upgrade + * rebuilds the derived indexes. Zero-copy (shared inodes; the store is + * immutable-by-rename) and instant even at scale. Returns the backup path, or + * `null` when the store holds no canonical nouns (nothing to protect). If a + * backup already exists from a prior FAILED upgrade, it is REUSED as-is (that + * is the true pre-upgrade state; a retry must not overwrite it). + */ + public async createMigrationBackup(): Promise { + await this.ensureInitialized() + + // Nothing to protect if the store has no canonical nouns (e.g. a brand-new + // brain whose marker is simply absent — `epochStale` is true but there is no + // 7.x data to migrate). + const probe = await this.getNouns({ pagination: { limit: 1 } }) + if ((probe.totalCount || 0) === 0 && probe.items.length === 0) { + return null + } + + const backupPath = `${this.rootDir}.migration-backup` + + // Reuse an existing backup (a prior failed upgrade's pre-state) rather than + // overwrite it — snapshotToDirectory also refuses a non-empty target. + try { + const existing = await fs.promises.readdir(backupPath) + if (existing.length > 0) return backupPath + } catch (error: any) { + if (error.code !== 'ENOENT') throw error + } + + await this.snapshotToDirectory(backupPath) + return backupPath + } + + /** + * @description Remove a {@link createMigrationBackup} snapshot. Best-effort: a + * missing path is a no-op, and any error is swallowed (a leftover backup dir is + * harmless — the operator can delete it). Removing the hard-links never touches + * the live store's bytes (shared inodes; only the extra links go away). + */ + public async removeMigrationBackup(location: string): Promise { + try { + await fs.promises.rm(location, { recursive: true, force: true }) + } catch { + // best-effort — leaving the backup behind is safe + } + } + /** * Recursively collect snapshot-eligible files under `dirAbs`, excluding the * lock directory and in-flight `*.tmp.*` write files. diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index b8fda568..b895c309 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1673,6 +1673,21 @@ export interface BrainyConfig { */ migrationWaitTimeoutMs?: number + /** + * Take an automatic **pre-upgrade backup** before a one-time 7.x → 8.0 + * migration rebuilds the derived indexes, and remove it once the upgrade + * verifies (retain it on failure, for rollback). On the filesystem adapter + * this is a **hard-link snapshot** of the brain directory — near-zero cost and + * space (shared inodes; the store is immutable-by-rename), even at scale. + * The migration is already structurally safe (it only reads canonical records, + * derived indexes are fully reconstructable, and a failed upgrade self-heals on + * re-open), so this is catastrophe-insurance against a migration *bug*, not a + * data-loss guard — and NOT a substitute for an off-device backup. No-op for + * non-filesystem storage and for a brain with no persisted data. + * Default: `true`. Set `false` to opt out (e.g. you run your own backup). + */ + migrationBackup?: boolean + /** * Vector index configuration (Brainy 8.0). * diff --git a/tests/integration/migration-backup.test.ts b/tests/integration/migration-backup.test.ts new file mode 100644 index 00000000..504d5eca --- /dev/null +++ b/tests/integration/migration-backup.test.ts @@ -0,0 +1,162 @@ +/** + * @module tests/integration/migration-backup + * @description Proof suite for the pre-upgrade backup (#42): before a one-time + * 7.x → 8.0 rebuild mutates the derived indexes, brainy hard-link-snapshots the + * brain directory (default-on, opt-out via `migrationBackup: false`), removes it + * once the upgrade verifies + stamps, and retains it on failure for rollback. + * + * Covers: the FileSystemStorage machinery (hard-link snapshot, reuse, remove + * without touching live data, empty→null); the brainy lifecycle on a stale-epoch + * reopen (backup created then removed on success); the opt-out; and the memory + * feature-detect (no machinery → graceful no-op). + * + * Staleness is forced by stamping a marker with a mismatched `indexEpoch` (what a + * pre-8.0 brain looks like), via the same `writeRawObject` surface the marker is + * stored through — not by deleting an on-disk file (the marker path is encoded). + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' +import { NounType } from '../../src/types/graphTypes.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-mig-backup-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) { + fs.rmSync(d, { recursive: true, force: true }) + fs.rmSync(`${d}.migration-backup`, { recursive: true, force: true }) + } +}) + +/** Build a persisted brain with one entity. `staleAfter` overwrites the marker + * with a mismatched epoch so the NEXT open detects a 7.x→8.0 upgrade. */ +async function buildBrainWithData(dir: string, staleAfter = false): Promise { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await brain.init() + const id = await brain.add({ data: 'protect me during the upgrade', type: NounType.Concept }) + await brain.flush() + if (staleAfter) { + await brain.storage.writeRawObject('_system/brain-format.json', { dataFormat: '7.0', indexEpoch: 0 }) + } + await brain.close() + return id +} + +describe('Pre-migration backup (#42)', () => { + it('FileSystemStorage.createMigrationBackup hard-links the store; removeMigrationBackup deletes it without touching live data', async () => { + const dir = mkTmp() + const id = await buildBrainWithData(dir) + + const storage: any = new FileSystemStorage(dir) + await storage.init() + + const backupPath = await storage.createMigrationBackup() + expect(backupPath).toBe(`${dir}.migration-backup`) + expect(fs.existsSync(backupPath)).toBe(true) + expect(fs.readdirSync(backupPath).length).toBeGreaterThan(0) // snapshot has content + + // Reuse: a second call returns the same path (a prior failed upgrade's + // pre-state is preserved, not overwritten) and does not throw. + expect(await storage.createMigrationBackup()).toBe(backupPath) + + // Remove: gone from disk, and the LIVE store is untouched (shared inodes). + await storage.removeMigrationBackup(backupPath) + expect(fs.existsSync(backupPath)).toBe(false) + + const reopened: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await reopened.init() + expect(await reopened.get(id)).not.toBeNull() // live data survived the backup+remove + await reopened.close() + }) + + it('returns null for an empty store — nothing to protect', async () => { + const dir = mkTmp() + const storage: any = new FileSystemStorage(dir) // raw store, no brain → no VFS root + await storage.init() + expect(await storage.createMigrationBackup()).toBeNull() + }) + + it('takes a backup on a stale-epoch reopen (default-on) and removes it once the upgrade verifies', async () => { + const dir = mkTmp() + await buildBrainWithData(dir, /* staleAfter */ true) + + const created: string[] = [] + const removed: string[] = [] + const proto = FileSystemStorage.prototype as any + const origCreate = proto.createMigrationBackup + const origRemove = proto.removeMigrationBackup + proto.createMigrationBackup = async function () { + const p = await origCreate.call(this) + if (p) created.push(p) + return p + } + proto.removeMigrationBackup = async function (p: string) { + removed.push(p) + return origRemove.call(this, p) + } + try { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true + }) + await brain.init() // epochStale → backup created; inline rebuild stamps → backup removed + await brain.close() + + expect(created.length).toBe(1) // a backup WAS taken before the rebuild + expect(removed).toContain(created[0]) // and removed once the upgrade stamped + expect(fs.existsSync(created[0])).toBe(false) // gone from disk + } finally { + proto.createMigrationBackup = origCreate + proto.removeMigrationBackup = origRemove + } + }) + + it('opt-out (migrationBackup:false) takes no backup; memory storage is a graceful no-op', async () => { + const dir = mkTmp() + await buildBrainWithData(dir, /* staleAfter */ true) + + const proto = FileSystemStorage.prototype as any + const origCreate = proto.createMigrationBackup + let calls = 0 + proto.createMigrationBackup = async function () { + calls++ + return origCreate.call(this) + } + try { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + migrationBackup: false, + silent: true + }) + await brain.init() + await brain.close() + expect(calls).toBe(0) + } finally { + proto.createMigrationBackup = origCreate + } + + // Memory storage has no backup machinery — default-on must not crash. + const mem: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await mem.init() + expect(mem.isInitialized).toBe(true) + expect(mem.config.migrationBackup).toBe(true) // default resolved + await mem.close() + }) +})