feat(8.0): auto pre-upgrade backup — hard-link snapshot before the 7.x→8.0 migration
David's ask: back up the brain before the one-time upgrade, drop it after. On open, when the on-disk format is stale (a 7.x→8.0 rebuild will run) and the store holds data, brainy hard-link-snapshots the brain dir to a sibling `<brainDir>.migration-backup` BEFORE any provider rebuilds — near-zero cost/space (shared inodes; the store is immutable-by-rename), instant even at scale. Removed automatically once the upgrade verifies + stamps the marker; retained on failure for rollback. Default-on; opt out with `migrationBackup: false`. - Reuses the existing snapshotToDirectory() hard-link farm via new optional adapter methods createMigrationBackup()/removeMigrationBackup() (filesystem only — feature-detected; memory + non-fs are a graceful no-op). - Taken before the native provider inits, so it captures true pre-migration state; a prior failed upgrade's backup is reused, not overwritten. - Best-effort: a backup failure never blocks the upgrade (the migration is itself safe — reads canonical, reconstructable, self-heals). Catastrophe-insurance against a migration bug, not a data-loss guard or an off-device backup. 4 lifecycle tests (adapter hard-link / reuse / remove-without-touching-live / empty→null; stale-epoch reopen create+remove; opt-out; memory no-op). Gates: typecheck 0, build 0, test:unit 1753/1753, layout-migration suite 5/5.
This commit is contained in:
parent
ed178e2ce9
commit
1aad1f67de
5 changed files with 319 additions and 0 deletions
|
|
@ -432,6 +432,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* not migrating — the fallback `elapsedMs` a provider that omits `migrationStatus()`
|
||||
* timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */
|
||||
private _migrationObservedAt: number | null = null
|
||||
/** Path of the pre-upgrade backup taken this open (default-on, opt-out via
|
||||
* `migrationBackup: false`), or `null` if none. Removed once the 7.x→8.0
|
||||
* upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */
|
||||
private _migrationBackupPath: string | null = null
|
||||
/**
|
||||
* 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat}
|
||||
* marker (`_system/brain-format.json`) as read during the store-open phase,
|
||||
|
|
@ -856,6 +860,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this._indexEpochStale =
|
||||
this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH
|
||||
|
||||
// Pre-upgrade backup (default-on, opt-out via `migrationBackup: false`): the
|
||||
// on-disk format is stale, so a one-time 7.x → 8.0 rebuild will run below.
|
||||
// Snapshot the brain dir NOW — before any provider (native or JS) rebuilds a
|
||||
// derived index — so a migration bug can be rolled back. Removed once the
|
||||
// upgrade verifies + stamps; retained on failure. No-op for a reader, for
|
||||
// non-filesystem storage, or for a brain with no persisted data.
|
||||
if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) {
|
||||
await this.createMigrationBackupIfNeeded()
|
||||
}
|
||||
|
||||
// Provider: embeddings (reassign embedder if plugin provides one)
|
||||
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
|
||||
if (embeddingProvider) {
|
||||
|
|
@ -6489,6 +6503,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await writeBrainFormat(this.storage)
|
||||
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
|
||||
this._indexEpochStale = false
|
||||
// The upgrade has verified + stamped — drop the pre-upgrade backup (no-op if
|
||||
// none was taken, e.g. a non-migrating stamp of a fresh brain's initial marker).
|
||||
await this.removeMigrationBackupSafe()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -12991,6 +13008,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Migration LOCK wait budget — left undefined when omitted so
|
||||
// awaitMigrationLock() applies its 30 s default at the read site.
|
||||
migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined,
|
||||
// Pre-upgrade backup — default-on; opt out with `migrationBackup: false`.
|
||||
migrationBackup: config?.migrationBackup ?? true,
|
||||
// Vector index configuration (8.0) — algorithm-neutral surface with
|
||||
// `recall` preset + `persistMode` (folded in from 7.x's hnswPersistMode).
|
||||
// See BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2.
|
||||
|
|
@ -13400,6 +13419,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await writeBrainFormat(this.storage)
|
||||
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
|
||||
this._indexEpochStale = false
|
||||
// The JS inline rebuild verified + stamped — drop the pre-upgrade backup.
|
||||
await this.removeMigrationBackupSafe()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -13560,6 +13581,61 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Take the pre-upgrade backup (default-on) before a native provider
|
||||
* rebuilds anything for a 7.x → 8.0 migration. Called at open, once the on-disk
|
||||
* format is known stale, BEFORE the providers init. Feature-detected + best-effort:
|
||||
* a backend without `createMigrationBackup` (memory) or a store with no data is a
|
||||
* no-op, and a backup failure NEVER blocks the upgrade (the migration is itself
|
||||
* safe — it only reads canonical, and self-heals on re-open). The snapshot is
|
||||
* removed once the upgrade verifies ({@link removeMigrationBackupSafe}) and
|
||||
* retained on failure for rollback.
|
||||
*/
|
||||
private async createMigrationBackupIfNeeded(): Promise<void> {
|
||||
const storage = this.storage as StorageAdapter & {
|
||||
createMigrationBackup?: () => Promise<string | null>
|
||||
}
|
||||
if (typeof storage.createMigrationBackup !== 'function') return
|
||||
try {
|
||||
const backupPath = await storage.createMigrationBackup()
|
||||
if (backupPath) {
|
||||
this._migrationBackupPath = backupPath
|
||||
prodLog.info(
|
||||
`[Brainy] Pre-upgrade backup created at ${backupPath} (hard-link snapshot; ~zero ` +
|
||||
`cost/space). Removed automatically once the 7.x→8.0 upgrade verifies; retained ` +
|
||||
`for rollback if it fails. Opt out with { migrationBackup: false }.`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
prodLog.warn(
|
||||
`[Brainy] Pre-upgrade backup could not be created (continuing — the upgrade is safe ` +
|
||||
`and self-heals): ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Remove the pre-upgrade backup once the migration has verified and
|
||||
* stamped the format marker. Called from both stamp paths (the JS inline rebuild
|
||||
* and the native provider's `stampBrainFormat()`), gated on a backup having been
|
||||
* taken this open. Best-effort: a removal failure leaves a harmless backup dir.
|
||||
*/
|
||||
private async removeMigrationBackupSafe(): Promise<void> {
|
||||
if (!this._migrationBackupPath) return
|
||||
const backupPath = this._migrationBackupPath
|
||||
this._migrationBackupPath = null
|
||||
const storage = this.storage as StorageAdapter & {
|
||||
removeMigrationBackup?: (location: string) => Promise<void>
|
||||
}
|
||||
if (typeof storage.removeMigrationBackup !== 'function') return
|
||||
try {
|
||||
await storage.removeMigrationBackup(backupPath)
|
||||
prodLog.info(`[Brainy] Upgrade verified — pre-upgrade backup at ${backupPath} removed.`)
|
||||
} catch {
|
||||
// best-effort — a leftover backup dir is harmless
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check health of metadata indexes
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1138,4 +1138,20 @@ export interface StorageAdapter {
|
|||
* @returns Promise that resolves to the total number of verbs
|
||||
*/
|
||||
getVerbCount(): Promise<number>
|
||||
|
||||
/**
|
||||
* OPTIONAL — create a pre-upgrade backup of the whole store and return its
|
||||
* location, or `null` when there is nothing to back up (empty store). On the
|
||||
* filesystem adapter this is a zero-copy hard-link snapshot into a sibling
|
||||
* directory; other backends may omit it. Callers feature-detect. Paired with
|
||||
* {@link removeMigrationBackup}. Used by the 7.x → 8.0 migration lifecycle.
|
||||
*/
|
||||
createMigrationBackup?(): Promise<string | null>
|
||||
|
||||
/**
|
||||
* OPTIONAL — remove a backup created by {@link createMigrationBackup}.
|
||||
* Best-effort: a missing `location` is a no-op. Removing a hard-link snapshot
|
||||
* never affects the live store (shared inodes; only the extra links are gone).
|
||||
*/
|
||||
removeMigrationBackup?(location: string): Promise<void>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1106,6 +1106,56 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Pre-upgrade backup: a hard-link snapshot of the whole store into
|
||||
* a SIBLING directory (`<rootDir>.migration-backup`, outside `rootDir` so the
|
||||
* snapshot never recurses into itself), taken before a 7.x → 8.0 upgrade
|
||||
* rebuilds the derived indexes. Zero-copy (shared inodes; the store is
|
||||
* immutable-by-rename) and instant even at scale. Returns the backup path, or
|
||||
* `null` when the store holds no canonical nouns (nothing to protect). If a
|
||||
* backup already exists from a prior FAILED upgrade, it is REUSED as-is (that
|
||||
* is the true pre-upgrade state; a retry must not overwrite it).
|
||||
*/
|
||||
public async createMigrationBackup(): Promise<string | null> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Nothing to protect if the store has no canonical nouns (e.g. a brand-new
|
||||
// brain whose marker is simply absent — `epochStale` is true but there is no
|
||||
// 7.x data to migrate).
|
||||
const probe = await this.getNouns({ pagination: { limit: 1 } })
|
||||
if ((probe.totalCount || 0) === 0 && probe.items.length === 0) {
|
||||
return null
|
||||
}
|
||||
|
||||
const backupPath = `${this.rootDir}.migration-backup`
|
||||
|
||||
// Reuse an existing backup (a prior failed upgrade's pre-state) rather than
|
||||
// overwrite it — snapshotToDirectory also refuses a non-empty target.
|
||||
try {
|
||||
const existing = await fs.promises.readdir(backupPath)
|
||||
if (existing.length > 0) return backupPath
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') throw error
|
||||
}
|
||||
|
||||
await this.snapshotToDirectory(backupPath)
|
||||
return backupPath
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Remove a {@link createMigrationBackup} snapshot. Best-effort: a
|
||||
* missing path is a no-op, and any error is swallowed (a leftover backup dir is
|
||||
* harmless — the operator can delete it). Removing the hard-links never touches
|
||||
* the live store's bytes (shared inodes; only the extra links go away).
|
||||
*/
|
||||
public async removeMigrationBackup(location: string): Promise<void> {
|
||||
try {
|
||||
await fs.promises.rm(location, { recursive: true, force: true })
|
||||
} catch {
|
||||
// best-effort — leaving the backup behind is safe
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively collect snapshot-eligible files under `dirAbs`, excluding the
|
||||
* lock directory and in-flight `*.tmp.*` write files.
|
||||
|
|
|
|||
|
|
@ -1673,6 +1673,21 @@ export interface BrainyConfig {
|
|||
*/
|
||||
migrationWaitTimeoutMs?: number
|
||||
|
||||
/**
|
||||
* Take an automatic **pre-upgrade backup** before a one-time 7.x → 8.0
|
||||
* migration rebuilds the derived indexes, and remove it once the upgrade
|
||||
* verifies (retain it on failure, for rollback). On the filesystem adapter
|
||||
* this is a **hard-link snapshot** of the brain directory — near-zero cost and
|
||||
* space (shared inodes; the store is immutable-by-rename), even at scale.
|
||||
* The migration is already structurally safe (it only reads canonical records,
|
||||
* derived indexes are fully reconstructable, and a failed upgrade self-heals on
|
||||
* re-open), so this is catastrophe-insurance against a migration *bug*, not a
|
||||
* data-loss guard — and NOT a substitute for an off-device backup. No-op for
|
||||
* non-filesystem storage and for a brain with no persisted data.
|
||||
* Default: `true`. Set `false` to opt out (e.g. you run your own backup).
|
||||
*/
|
||||
migrationBackup?: boolean
|
||||
|
||||
/**
|
||||
* Vector index configuration (Brainy 8.0).
|
||||
*
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue