feat(8.0): auto pre-upgrade backup — hard-link snapshot before the 7.x→8.0 migration

David's ask: back up the brain before the one-time upgrade, drop it after. On
open, when the on-disk format is stale (a 7.x→8.0 rebuild will run) and the store
holds data, brainy hard-link-snapshots the brain dir to a sibling
`<brainDir>.migration-backup` BEFORE any provider rebuilds — near-zero cost/space
(shared inodes; the store is immutable-by-rename), instant even at scale. Removed
automatically once the upgrade verifies + stamps the marker; retained on failure
for rollback. Default-on; opt out with `migrationBackup: false`.

- Reuses the existing snapshotToDirectory() hard-link farm via new optional
  adapter methods createMigrationBackup()/removeMigrationBackup() (filesystem
  only — feature-detected; memory + non-fs are a graceful no-op).
- Taken before the native provider inits, so it captures true pre-migration
  state; a prior failed upgrade's backup is reused, not overwritten.
- Best-effort: a backup failure never blocks the upgrade (the migration is itself
  safe — reads canonical, reconstructable, self-heals). Catastrophe-insurance
  against a migration bug, not a data-loss guard or an off-device backup.

4 lifecycle tests (adapter hard-link / reuse / remove-without-touching-live /
empty→null; stale-epoch reopen create+remove; opt-out; memory no-op). Gates:
typecheck 0, build 0, test:unit 1753/1753, layout-migration suite 5/5.
This commit is contained in:
David Snelling 2026-07-01 15:04:01 -07:00
parent ed178e2ce9
commit 1aad1f67de
5 changed files with 319 additions and 0 deletions

View file

@ -432,6 +432,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
* not migrating the fallback `elapsedMs` a provider that omits `migrationStatus()` * not migrating the fallback `elapsedMs` a provider that omits `migrationStatus()`
* timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */ * timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */
private _migrationObservedAt: number | null = null private _migrationObservedAt: number | null = null
/** Path of the pre-upgrade backup taken this open (default-on, opt-out via
* `migrationBackup: false`), or `null` if none. Removed once the 7.x8.0
* upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */
private _migrationBackupPath: string | null = null
/** /**
* 8.0 native-provider version handshake the on-disk {@link BrainFormat} * 8.0 native-provider version handshake the on-disk {@link BrainFormat}
* marker (`_system/brain-format.json`) as read during the store-open phase, * marker (`_system/brain-format.json`) as read during the store-open phase,
@ -856,6 +860,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
this._indexEpochStale = this._indexEpochStale =
this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH
// Pre-upgrade backup (default-on, opt-out via `migrationBackup: false`): the
// on-disk format is stale, so a one-time 7.x → 8.0 rebuild will run below.
// Snapshot the brain dir NOW — before any provider (native or JS) rebuilds a
// derived index — so a migration bug can be rolled back. Removed once the
// upgrade verifies + stamps; retained on failure. No-op for a reader, for
// non-filesystem storage, or for a brain with no persisted data.
if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) {
await this.createMigrationBackupIfNeeded()
}
// Provider: embeddings (reassign embedder if plugin provides one) // Provider: embeddings (reassign embedder if plugin provides one)
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings') const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
if (embeddingProvider) { if (embeddingProvider) {
@ -6489,6 +6503,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
await writeBrainFormat(this.storage) await writeBrainFormat(this.storage)
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
this._indexEpochStale = false this._indexEpochStale = false
// The upgrade has verified + stamped — drop the pre-upgrade backup (no-op if
// none was taken, e.g. a non-migrating stamp of a fresh brain's initial marker).
await this.removeMigrationBackupSafe()
} }
/** /**
@ -12991,6 +13008,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Migration LOCK wait budget — left undefined when omitted so // Migration LOCK wait budget — left undefined when omitted so
// awaitMigrationLock() applies its 30 s default at the read site. // awaitMigrationLock() applies its 30 s default at the read site.
migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined, migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined,
// Pre-upgrade backup — default-on; opt out with `migrationBackup: false`.
migrationBackup: config?.migrationBackup ?? true,
// Vector index configuration (8.0) — algorithm-neutral surface with // Vector index configuration (8.0) — algorithm-neutral surface with
// `recall` preset + `persistMode` (folded in from 7.x's hnswPersistMode). // `recall` preset + `persistMode` (folded in from 7.x's hnswPersistMode).
// See BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2. // See BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2.
@ -13400,6 +13419,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
await writeBrainFormat(this.storage) await writeBrainFormat(this.storage)
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
this._indexEpochStale = false this._indexEpochStale = false
// The JS inline rebuild verified + stamped — drop the pre-upgrade backup.
await this.removeMigrationBackupSafe()
} }
/** /**
@ -13560,6 +13581,61 @@ export class Brainy<T = any> implements BrainyInterface<T> {
} }
} }
/**
* @description Take the pre-upgrade backup (default-on) before a native provider
* rebuilds anything for a 7.x 8.0 migration. Called at open, once the on-disk
* format is known stale, BEFORE the providers init. Feature-detected + best-effort:
* a backend without `createMigrationBackup` (memory) or a store with no data is a
* no-op, and a backup failure NEVER blocks the upgrade (the migration is itself
* safe it only reads canonical, and self-heals on re-open). The snapshot is
* removed once the upgrade verifies ({@link removeMigrationBackupSafe}) and
* retained on failure for rollback.
*/
private async createMigrationBackupIfNeeded(): Promise<void> {
const storage = this.storage as StorageAdapter & {
createMigrationBackup?: () => Promise<string | null>
}
if (typeof storage.createMigrationBackup !== 'function') return
try {
const backupPath = await storage.createMigrationBackup()
if (backupPath) {
this._migrationBackupPath = backupPath
prodLog.info(
`[Brainy] Pre-upgrade backup created at ${backupPath} (hard-link snapshot; ~zero ` +
`cost/space). Removed automatically once the 7.x→8.0 upgrade verifies; retained ` +
`for rollback if it fails. Opt out with { migrationBackup: false }.`
)
}
} catch (error) {
prodLog.warn(
`[Brainy] Pre-upgrade backup could not be created (continuing — the upgrade is safe ` +
`and self-heals): ${error instanceof Error ? error.message : String(error)}`
)
}
}
/**
* @description Remove the pre-upgrade backup once the migration has verified and
* stamped the format marker. Called from both stamp paths (the JS inline rebuild
* and the native provider's `stampBrainFormat()`), gated on a backup having been
* taken this open. Best-effort: a removal failure leaves a harmless backup dir.
*/
private async removeMigrationBackupSafe(): Promise<void> {
if (!this._migrationBackupPath) return
const backupPath = this._migrationBackupPath
this._migrationBackupPath = null
const storage = this.storage as StorageAdapter & {
removeMigrationBackup?: (location: string) => Promise<void>
}
if (typeof storage.removeMigrationBackup !== 'function') return
try {
await storage.removeMigrationBackup(backupPath)
prodLog.info(`[Brainy] Upgrade verified — pre-upgrade backup at ${backupPath} removed.`)
} catch {
// best-effort — a leftover backup dir is harmless
}
}
/** /**
* Check health of metadata indexes * Check health of metadata indexes
* *

View file

@ -1138,4 +1138,20 @@ export interface StorageAdapter {
* @returns Promise that resolves to the total number of verbs * @returns Promise that resolves to the total number of verbs
*/ */
getVerbCount(): Promise<number> getVerbCount(): Promise<number>
/**
* OPTIONAL create a pre-upgrade backup of the whole store and return its
* location, or `null` when there is nothing to back up (empty store). On the
* filesystem adapter this is a zero-copy hard-link snapshot into a sibling
* directory; other backends may omit it. Callers feature-detect. Paired with
* {@link removeMigrationBackup}. Used by the 7.x 8.0 migration lifecycle.
*/
createMigrationBackup?(): Promise<string | null>
/**
* OPTIONAL remove a backup created by {@link createMigrationBackup}.
* Best-effort: a missing `location` is a no-op. Removing a hard-link snapshot
* never affects the live store (shared inodes; only the extra links are gone).
*/
removeMigrationBackup?(location: string): Promise<void>
} }

View file

@ -1106,6 +1106,56 @@ export class FileSystemStorage extends BaseStorage {
} }
} }
/**
* @description Pre-upgrade backup: a hard-link snapshot of the whole store into
* a SIBLING directory (`<rootDir>.migration-backup`, outside `rootDir` so the
* snapshot never recurses into itself), taken before a 7.x 8.0 upgrade
* rebuilds the derived indexes. Zero-copy (shared inodes; the store is
* immutable-by-rename) and instant even at scale. Returns the backup path, or
* `null` when the store holds no canonical nouns (nothing to protect). If a
* backup already exists from a prior FAILED upgrade, it is REUSED as-is (that
* is the true pre-upgrade state; a retry must not overwrite it).
*/
public async createMigrationBackup(): Promise<string | null> {
await this.ensureInitialized()
// Nothing to protect if the store has no canonical nouns (e.g. a brand-new
// brain whose marker is simply absent — `epochStale` is true but there is no
// 7.x data to migrate).
const probe = await this.getNouns({ pagination: { limit: 1 } })
if ((probe.totalCount || 0) === 0 && probe.items.length === 0) {
return null
}
const backupPath = `${this.rootDir}.migration-backup`
// Reuse an existing backup (a prior failed upgrade's pre-state) rather than
// overwrite it — snapshotToDirectory also refuses a non-empty target.
try {
const existing = await fs.promises.readdir(backupPath)
if (existing.length > 0) return backupPath
} catch (error: any) {
if (error.code !== 'ENOENT') throw error
}
await this.snapshotToDirectory(backupPath)
return backupPath
}
/**
* @description Remove a {@link createMigrationBackup} snapshot. Best-effort: a
* missing path is a no-op, and any error is swallowed (a leftover backup dir is
* harmless the operator can delete it). Removing the hard-links never touches
* the live store's bytes (shared inodes; only the extra links go away).
*/
public async removeMigrationBackup(location: string): Promise<void> {
try {
await fs.promises.rm(location, { recursive: true, force: true })
} catch {
// best-effort — leaving the backup behind is safe
}
}
/** /**
* Recursively collect snapshot-eligible files under `dirAbs`, excluding the * Recursively collect snapshot-eligible files under `dirAbs`, excluding the
* lock directory and in-flight `*.tmp.*` write files. * lock directory and in-flight `*.tmp.*` write files.

View file

@ -1673,6 +1673,21 @@ export interface BrainyConfig {
*/ */
migrationWaitTimeoutMs?: number migrationWaitTimeoutMs?: number
/**
* Take an automatic **pre-upgrade backup** before a one-time 7.x 8.0
* migration rebuilds the derived indexes, and remove it once the upgrade
* verifies (retain it on failure, for rollback). On the filesystem adapter
* this is a **hard-link snapshot** of the brain directory near-zero cost and
* space (shared inodes; the store is immutable-by-rename), even at scale.
* The migration is already structurally safe (it only reads canonical records,
* derived indexes are fully reconstructable, and a failed upgrade self-heals on
* re-open), so this is catastrophe-insurance against a migration *bug*, not a
* data-loss guard and NOT a substitute for an off-device backup. No-op for
* non-filesystem storage and for a brain with no persisted data.
* Default: `true`. Set `false` to opt out (e.g. you run your own backup).
*/
migrationBackup?: boolean
/** /**
* Vector index configuration (Brainy 8.0). * Vector index configuration (Brainy 8.0).
* *

View file

@ -0,0 +1,162 @@
/**
* @module tests/integration/migration-backup
* @description Proof suite for the pre-upgrade backup (#42): before a one-time
* 7.x 8.0 rebuild mutates the derived indexes, brainy hard-link-snapshots the
* brain directory (default-on, opt-out via `migrationBackup: false`), removes it
* once the upgrade verifies + stamps, and retains it on failure for rollback.
*
* Covers: the FileSystemStorage machinery (hard-link snapshot, reuse, remove
* without touching live data, emptynull); the brainy lifecycle on a stale-epoch
* reopen (backup created then removed on success); the opt-out; and the memory
* feature-detect (no machinery graceful no-op).
*
* Staleness is forced by stamping a marker with a mismatched `indexEpoch` (what a
* pre-8.0 brain looks like), via the same `writeRawObject` surface the marker is
* stored through not by deleting an on-disk file (the marker path is encoded).
*/
import { describe, it, expect, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
import { NounType } from '../../src/types/graphTypes.js'
const tmpDirs: string[] = []
function mkTmp(): string {
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-mig-backup-'))
tmpDirs.push(d)
return d
}
afterEach(() => {
for (const d of tmpDirs.splice(0)) {
fs.rmSync(d, { recursive: true, force: true })
fs.rmSync(`${d}.migration-backup`, { recursive: true, force: true })
}
})
/** Build a persisted brain with one entity. `staleAfter` overwrites the marker
* with a mismatched epoch so the NEXT open detects a 7.x8.0 upgrade. */
async function buildBrainWithData(dir: string, staleAfter = false): Promise<string> {
const brain: any = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true
})
await brain.init()
const id = await brain.add({ data: 'protect me during the upgrade', type: NounType.Concept })
await brain.flush()
if (staleAfter) {
await brain.storage.writeRawObject('_system/brain-format.json', { dataFormat: '7.0', indexEpoch: 0 })
}
await brain.close()
return id
}
describe('Pre-migration backup (#42)', () => {
it('FileSystemStorage.createMigrationBackup hard-links the store; removeMigrationBackup deletes it without touching live data', async () => {
const dir = mkTmp()
const id = await buildBrainWithData(dir)
const storage: any = new FileSystemStorage(dir)
await storage.init()
const backupPath = await storage.createMigrationBackup()
expect(backupPath).toBe(`${dir}.migration-backup`)
expect(fs.existsSync(backupPath)).toBe(true)
expect(fs.readdirSync(backupPath).length).toBeGreaterThan(0) // snapshot has content
// Reuse: a second call returns the same path (a prior failed upgrade's
// pre-state is preserved, not overwritten) and does not throw.
expect(await storage.createMigrationBackup()).toBe(backupPath)
// Remove: gone from disk, and the LIVE store is untouched (shared inodes).
await storage.removeMigrationBackup(backupPath)
expect(fs.existsSync(backupPath)).toBe(false)
const reopened: any = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true
})
await reopened.init()
expect(await reopened.get(id)).not.toBeNull() // live data survived the backup+remove
await reopened.close()
})
it('returns null for an empty store — nothing to protect', async () => {
const dir = mkTmp()
const storage: any = new FileSystemStorage(dir) // raw store, no brain → no VFS root
await storage.init()
expect(await storage.createMigrationBackup()).toBeNull()
})
it('takes a backup on a stale-epoch reopen (default-on) and removes it once the upgrade verifies', async () => {
const dir = mkTmp()
await buildBrainWithData(dir, /* staleAfter */ true)
const created: string[] = []
const removed: string[] = []
const proto = FileSystemStorage.prototype as any
const origCreate = proto.createMigrationBackup
const origRemove = proto.removeMigrationBackup
proto.createMigrationBackup = async function () {
const p = await origCreate.call(this)
if (p) created.push(p)
return p
}
proto.removeMigrationBackup = async function (p: string) {
removed.push(p)
return origRemove.call(this, p)
}
try {
const brain: any = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true
})
await brain.init() // epochStale → backup created; inline rebuild stamps → backup removed
await brain.close()
expect(created.length).toBe(1) // a backup WAS taken before the rebuild
expect(removed).toContain(created[0]) // and removed once the upgrade stamped
expect(fs.existsSync(created[0])).toBe(false) // gone from disk
} finally {
proto.createMigrationBackup = origCreate
proto.removeMigrationBackup = origRemove
}
})
it('opt-out (migrationBackup:false) takes no backup; memory storage is a graceful no-op', async () => {
const dir = mkTmp()
await buildBrainWithData(dir, /* staleAfter */ true)
const proto = FileSystemStorage.prototype as any
const origCreate = proto.createMigrationBackup
let calls = 0
proto.createMigrationBackup = async function () {
calls++
return origCreate.call(this)
}
try {
const brain: any = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
migrationBackup: false,
silent: true
})
await brain.init()
await brain.close()
expect(calls).toBe(0)
} finally {
proto.createMigrationBackup = origCreate
}
// Memory storage has no backup machinery — default-on must not crash.
const mem: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
await mem.init()
expect(mem.isInitialized).toBe(true)
expect(mem.config.migrationBackup).toBe(true) // default resolved
await mem.close()
})
})