feat: add migration system with error handling, validation, and enterprise hardening
- MigrationRunner: per-entity error tracking (non-fatal), maxErrors bail-out, static validateMigrations() called in constructor, branch error propagation - RefManager: updateRefMetadata() method for clean metadata updates - brainy.ts: eliminate as-any casts in migration methods, use updateRefMetadata, forward maxErrors through full chain including branch migrations - Types: MigrationError interface, errors field on MigrationResult, maxErrors on MigrateOptions - Package exports: MigrationError type, migrate() on BrainyInterface, autoMigrate config - 31 integration tests covering error handling, validation, branch error propagation - Documentation: docs/guides/schema-migrations.md
This commit is contained in:
parent
e9f6a1b461
commit
39b099cafc
12 changed files with 2022 additions and 1 deletions
221
src/brainy.ts
221
src/brainy.ts
|
|
@ -88,6 +88,8 @@ import {
|
|||
import { NounType, VerbType } from './types/graphTypes.js'
|
||||
import { BrainyInterface } from './types/brainyInterface.js'
|
||||
import type { IntegrationHub } from './integrations/core/IntegrationHub.js'
|
||||
import { MigrationRunner } from './migration/MigrationRunner.js'
|
||||
import type { MigrationPreview, MigrationResult, MigrateOptions } from './migration/types.js'
|
||||
|
||||
/**
|
||||
* Stopwords for semantic highlighting
|
||||
|
|
@ -169,6 +171,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
private _vfs?: VirtualFileSystem
|
||||
private _vfsInitialized = false // Track VFS init completion separately
|
||||
private _hub?: IntegrationHub // Integration Hub for external tools
|
||||
private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets
|
||||
|
||||
// State
|
||||
private initialized = false
|
||||
|
|
@ -351,6 +354,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Rebuild indexes if needed for existing data
|
||||
await this.rebuildIndexesIfNeeded()
|
||||
|
||||
// Check for pending data migrations
|
||||
await this.checkMigrations()
|
||||
|
||||
// Connect distributed components to storage
|
||||
await this.connectDistributedStorage()
|
||||
|
||||
|
|
@ -3551,6 +3557,217 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await refManager.deleteRef(branch)
|
||||
}
|
||||
|
||||
// ─── Migration API ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run pending data migrations, or preview what would change.
|
||||
*
|
||||
* @param options - Pass { dryRun: true } to preview without writing
|
||||
* @returns Migration result (or preview if dryRun)
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Preview what would change
|
||||
* const preview = await brain.migrate({ dryRun: true })
|
||||
* console.log(preview.affectedEntities)
|
||||
*
|
||||
* // Apply migrations (auto-forks a backup branch first)
|
||||
* const result = await brain.migrate()
|
||||
* console.log(result.backupBranch) // 'pre-migration-7.17.0'
|
||||
*
|
||||
* // Rollback if needed
|
||||
* await brain.checkout('pre-migration-7.17.0')
|
||||
* ```
|
||||
*/
|
||||
async migrate(options?: MigrateOptions): Promise<MigrationResult | MigrationPreview> {
|
||||
await this.ensureInitialized()
|
||||
const runner = this._pendingMigrationRunner || new MigrationRunner(this.storage)
|
||||
|
||||
if (options?.dryRun) {
|
||||
return runner.preview()
|
||||
}
|
||||
|
||||
return this.migrateInternal(runner, options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for pending migrations during init().
|
||||
* Runs inline for small datasets if autoMigrate is enabled,
|
||||
* otherwise logs a warning.
|
||||
*/
|
||||
private async checkMigrations(): Promise<void> {
|
||||
const runner = new MigrationRunner(this.storage)
|
||||
|
||||
if (!(await runner.hasPendingMigrations())) {
|
||||
return
|
||||
}
|
||||
|
||||
const count = await runner.pendingCount()
|
||||
|
||||
if (this.config.autoMigrate) {
|
||||
// Quick entity count check to decide inline vs deferred
|
||||
const probe = await this.storage.getNouns({ pagination: { limit: 1 } })
|
||||
const totalEstimate = probe.totalCount ?? (probe.hasMore ? 10001 : probe.items.length)
|
||||
|
||||
if (totalEstimate < 10000) {
|
||||
// Small dataset — migrate inline during init
|
||||
await this.migrateInternal(runner)
|
||||
} else {
|
||||
// Large dataset — defer to explicit brain.migrate() call
|
||||
this._pendingMigrationRunner = runner
|
||||
if (!this.config.silent) {
|
||||
console.log(`[brainy] ${count} pending migration(s) detected. Call brain.migrate() to apply (dataset too large for inline migration).`)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (!this.config.silent) {
|
||||
console.log(`[brainy] ${count} pending migration(s) available. Set autoMigrate: true or call brain.migrate() to apply.`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal: fork backup, run migrations on current branch + all other branches.
|
||||
*
|
||||
* Branch strategy:
|
||||
* - getNouns() uses listObjectsInBranch() which only returns branch-local entities
|
||||
* - So migrating main only transforms main's entities; branch-local entities are untouched
|
||||
* - After main, we iterate all other user branches and run the same transforms
|
||||
* - Lightweight: just switches storage.currentBranch (no full checkout/index rebuild)
|
||||
* - Transforms are idempotent (return null when already applied), so this is safe
|
||||
*/
|
||||
private async migrateInternal(runner: MigrationRunner, options?: MigrateOptions): Promise<MigrationResult> {
|
||||
// 0. Clean up old migration backup branches (by metadata tag, not name)
|
||||
await runner.cleanupOldBackups()
|
||||
|
||||
// 1. Fork backup branch (uses existing COW — instant)
|
||||
const version = runner.nextMigrationVersion()
|
||||
const backupName = `pre-migration-${version}`
|
||||
let backupCreated = false
|
||||
|
||||
try {
|
||||
await this.fork(backupName)
|
||||
backupCreated = true
|
||||
|
||||
// Tag the backup branch with metadata so we can identify it later
|
||||
if (this.storage.refManager) {
|
||||
await this.storage.refManager.updateRefMetadata(backupName, {
|
||||
type: 'system:backup',
|
||||
migrationVersion: version,
|
||||
author: 'brainy-migration'
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
// Fork may fail if COW not initialized (e.g., memory storage with no commits)
|
||||
// Continue without backup — migrations are still safe (user can re-import)
|
||||
}
|
||||
|
||||
// 2. Run migrations on current branch
|
||||
const runResult = await runner.run({ onProgress: options?.onProgress, maxErrors: options?.maxErrors })
|
||||
|
||||
// 3. Migrate all other user branches (branch-local entities only)
|
||||
// getNouns() uses listObjectsInBranch() which only lists branch-overlay files,
|
||||
// so each branch iteration only touches entities written directly to that branch.
|
||||
const branchResult = await this.migrateOtherBranches(
|
||||
runResult.migrationsApplied,
|
||||
backupCreated ? backupName : null,
|
||||
options
|
||||
)
|
||||
|
||||
// 4. Rebuild MetadataIndex if any entities were modified (on the current branch)
|
||||
const totalModified = runResult.entitiesModified + branchResult.entitiesModified
|
||||
if (totalModified > 0) {
|
||||
await this.metadataIndex.rebuild()
|
||||
}
|
||||
|
||||
// 5. Clear deferred runner
|
||||
this._pendingMigrationRunner = undefined
|
||||
|
||||
return {
|
||||
backupBranch: backupCreated ? backupName : null,
|
||||
migrationsApplied: runResult.migrationsApplied,
|
||||
entitiesProcessed: runResult.entitiesProcessed + branchResult.entitiesProcessed,
|
||||
entitiesModified: totalModified,
|
||||
errors: [...(runResult.errors || []), ...branchResult.errors]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate branch-local entities on all non-current branches.
|
||||
*
|
||||
* Why this is needed: getNouns() lists entities from the branch overlay only
|
||||
* (via listObjectsInBranch), not inherited entities. So migrating main doesn't
|
||||
* touch entities written directly to feature branches. We iterate each branch
|
||||
* and run the same transforms — they're idempotent, so already-migrated
|
||||
* inherited entities return null and are skipped.
|
||||
*
|
||||
* Lightweight: switches storage.currentBranch directly instead of full checkout()
|
||||
* (no index rebuild needed — migration uses storage-level methods only).
|
||||
*/
|
||||
private async migrateOtherBranches(
|
||||
migrationIds: string[],
|
||||
skipBranch: string | null,
|
||||
options?: MigrateOptions
|
||||
): Promise<{ entitiesProcessed: number; entitiesModified: number; errors: import('./migration/types.js').MigrationError[] }> {
|
||||
const empty = { entitiesProcessed: 0, entitiesModified: 0, errors: [] as import('./migration/types.js').MigrationError[] }
|
||||
if (migrationIds.length === 0) return empty
|
||||
|
||||
// Only if COW branching is available
|
||||
const refManager = this.storage.refManager
|
||||
if (!refManager) return empty
|
||||
|
||||
const currentBranch = this.storage.currentBranch || 'main'
|
||||
let totalProcessed = 0
|
||||
let totalModified = 0
|
||||
const allErrors: import('./migration/types.js').MigrationError[] = []
|
||||
|
||||
try {
|
||||
const { MIGRATIONS } = await import('./migration/migrations.js')
|
||||
const migrationsToRun = MIGRATIONS.filter(m => migrationIds.includes(m.id))
|
||||
if (migrationsToRun.length === 0) return empty
|
||||
|
||||
const refs = await refManager.listRefs()
|
||||
const branches = refs
|
||||
.filter(ref => ref.name.startsWith('refs/heads/'))
|
||||
.map(ref => ({
|
||||
name: ref.name.replace('refs/heads/', ''),
|
||||
metadata: ref.metadata
|
||||
}))
|
||||
|
||||
for (const branch of branches) {
|
||||
// Skip current branch (already migrated above)
|
||||
if (branch.name === currentBranch) continue
|
||||
// Skip the backup branch we just created
|
||||
if (branch.name === skipBranch) continue
|
||||
// Skip system backup branches
|
||||
if (branch.metadata?.type === 'system:backup') continue
|
||||
|
||||
// Switch storage to this branch (lightweight — no index rebuild)
|
||||
this.storage.currentBranch = branch.name
|
||||
|
||||
// Run transforms — idempotent, so inherited entities return null and are skipped.
|
||||
// Uses runMigrations() which bypasses the state check (state on main says "completed"
|
||||
// but branch-local entities haven't been touched yet).
|
||||
const branchRunner = new MigrationRunner(this.storage)
|
||||
const result = await branchRunner.runMigrations(migrationsToRun, {
|
||||
onProgress: options?.onProgress,
|
||||
maxErrors: options?.maxErrors
|
||||
})
|
||||
|
||||
totalProcessed += result.entitiesProcessed
|
||||
totalModified += result.entitiesModified
|
||||
if (result.errors.length > 0) {
|
||||
allErrors.push(...result.errors)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
// Always restore original branch
|
||||
this.storage.currentBranch = currentBranch
|
||||
}
|
||||
|
||||
return { entitiesProcessed: totalProcessed, entitiesModified: totalModified, errors: allErrors }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get commit history for current branch
|
||||
* @param options - History options (limit, offset, author)
|
||||
|
|
@ -6616,7 +6833,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Plugin configuration - undefined = auto-detect
|
||||
plugins: config?.plugins ?? undefined as any,
|
||||
// Integration Hub - undefined/false = disabled
|
||||
integrations: config?.integrations ?? undefined as any
|
||||
integrations: config?.integrations ?? undefined as any,
|
||||
// Migration — disabled by default, opt-in for automatic migration
|
||||
autoMigrate: config?.autoMigrate ?? false
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,10 @@ export { getBrainyVersion } from './utils/version.js'
|
|||
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'
|
||||
export { PluginRegistry } from './plugin.js'
|
||||
|
||||
// Export migration system
|
||||
export { MigrationRunner, MIGRATIONS } from './migration/index.js'
|
||||
export type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './migration/index.js'
|
||||
|
||||
// Export embedding functionality
|
||||
import {
|
||||
UniversalSentenceEncoder,
|
||||
|
|
|
|||
544
src/migration/MigrationRunner.ts
Normal file
544
src/migration/MigrationRunner.ts
Normal file
|
|
@ -0,0 +1,544 @@
|
|||
/**
|
||||
* MigrationRunner: Executes schema migrations on Brainy storage
|
||||
*
|
||||
* Handles paginated iteration, resume-safe batching, and dry-run previews.
|
||||
* Uses BaseStorage methods directly — no adapter-level changes needed.
|
||||
*/
|
||||
|
||||
import type { BaseStorage } from '../storage/baseStorage.js'
|
||||
import type { NounMetadata, VerbMetadata } from '../coreTypes.js'
|
||||
import type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './types.js'
|
||||
import { MIGRATIONS } from './migrations.js'
|
||||
|
||||
const MIGRATION_STATE_KEY = '__migration_state__'
|
||||
const PREVIEW_SAMPLE_SIZE = 5
|
||||
const DEFAULT_MAX_ERRORS = 100
|
||||
|
||||
export class MigrationRunner {
|
||||
private storage: BaseStorage
|
||||
private stateCache: MigrationState | null | undefined = undefined
|
||||
|
||||
constructor(storage: BaseStorage) {
|
||||
this.storage = storage
|
||||
MigrationRunner.validateMigrations(MIGRATIONS)
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate migration definitions.
|
||||
* Called automatically in constructor for the global MIGRATIONS array.
|
||||
* Also available as a static method for validating custom migration arrays.
|
||||
*/
|
||||
static validateMigrations(migrations: Migration[]): void {
|
||||
if (migrations.length === 0) return
|
||||
|
||||
const seenIds = new Set<string>()
|
||||
const validApplies = new Set(['nouns', 'verbs', 'both'])
|
||||
|
||||
for (const m of migrations) {
|
||||
if (!m.id || typeof m.id !== 'string') {
|
||||
throw new Error(`Migration has missing or invalid id`)
|
||||
}
|
||||
if (seenIds.has(m.id)) {
|
||||
throw new Error(`Duplicate migration id: "${m.id}"`)
|
||||
}
|
||||
seenIds.add(m.id)
|
||||
|
||||
if (!m.version || typeof m.version !== 'string') {
|
||||
throw new Error(`Migration "${m.id}" has missing or invalid version`)
|
||||
}
|
||||
if (!m.description || typeof m.description !== 'string') {
|
||||
throw new Error(`Migration "${m.id}" has missing or invalid description`)
|
||||
}
|
||||
if (!validApplies.has(m.applies)) {
|
||||
throw new Error(`Migration "${m.id}" has invalid applies value: "${m.applies}" (must be "nouns", "verbs", or "both")`)
|
||||
}
|
||||
if (typeof m.transform !== 'function') {
|
||||
throw new Error(`Migration "${m.id}" has non-function transform`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there are pending migrations to run.
|
||||
* Single getMetadata() call — ~0ms overhead when no migrations exist.
|
||||
*/
|
||||
async hasPendingMigrations(): Promise<boolean> {
|
||||
if (MIGRATIONS.length === 0) return false
|
||||
const state = await this.getState()
|
||||
return this.getPendingMigrations(state).length > 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the version string for the next pending migration.
|
||||
*/
|
||||
nextMigrationVersion(): string {
|
||||
const pending = this.getPendingMigrationsFromCache()
|
||||
return pending.length > 0 ? pending[pending.length - 1].version : 'unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of pending migrations (for log messages).
|
||||
*/
|
||||
async pendingCount(): Promise<number> {
|
||||
if (MIGRATIONS.length === 0) return 0
|
||||
const state = await this.getState()
|
||||
return this.getPendingMigrations(state).length
|
||||
}
|
||||
|
||||
/**
|
||||
* Preview what a migration would do without writing anything.
|
||||
* Scans entities, applies transforms in memory, reports counts + samples.
|
||||
*/
|
||||
async preview(): Promise<MigrationPreview> {
|
||||
const state = await this.getState()
|
||||
const pending = this.getPendingMigrations(state)
|
||||
|
||||
if (pending.length === 0) {
|
||||
return {
|
||||
pendingMigrations: [],
|
||||
affectedEntities: 0,
|
||||
totalEntities: 0,
|
||||
sampleChanges: [],
|
||||
estimatedTime: '0ms'
|
||||
}
|
||||
}
|
||||
|
||||
let totalEntities = 0
|
||||
let affectedEntities = 0
|
||||
const sampleChanges: MigrationPreview['sampleChanges'] = []
|
||||
const batchConfig = this.storage.getBatchConfig()
|
||||
const batchSize = batchConfig.maxBatchSize
|
||||
|
||||
// Scan nouns if any pending migration applies to nouns
|
||||
const nounMigrations = pending.filter(m => m.applies === 'nouns' || m.applies === 'both')
|
||||
if (nounMigrations.length > 0) {
|
||||
let offset = 0
|
||||
let hasMore = true
|
||||
|
||||
while (hasMore) {
|
||||
const batch = await this.storage.getNouns({ pagination: { offset, limit: batchSize } })
|
||||
const ids = batch.items.map(e => e.id)
|
||||
const metadataBatch = await this.storage.getNounMetadataBatch(ids)
|
||||
|
||||
for (const entity of batch.items) {
|
||||
totalEntities++
|
||||
const entityMeta = metadataBatch.get(entity.id)
|
||||
if (!entityMeta) continue
|
||||
|
||||
const metadata = entityMeta as Record<string, unknown>
|
||||
const result = this.applyTransforms(metadata, nounMigrations)
|
||||
if (result !== null) {
|
||||
affectedEntities++
|
||||
if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) {
|
||||
sampleChanges.push({
|
||||
id: entity.id,
|
||||
before: { ...metadata },
|
||||
after: result
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
hasMore = batch.hasMore
|
||||
offset += batch.items.length
|
||||
}
|
||||
}
|
||||
|
||||
// Scan verbs if any pending migration applies to verbs
|
||||
const verbMigrations = pending.filter(m => m.applies === 'verbs' || m.applies === 'both')
|
||||
if (verbMigrations.length > 0) {
|
||||
let offset = 0
|
||||
let hasMore = true
|
||||
|
||||
while (hasMore) {
|
||||
const batch = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } })
|
||||
|
||||
for (const verb of batch.items) {
|
||||
totalEntities++
|
||||
const verbMeta = await this.storage.getVerbMetadata(verb.id)
|
||||
if (!verbMeta) continue
|
||||
|
||||
const metadata = verbMeta as Record<string, unknown>
|
||||
const result = this.applyTransforms(metadata, verbMigrations)
|
||||
if (result !== null) {
|
||||
affectedEntities++
|
||||
if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) {
|
||||
sampleChanges.push({
|
||||
id: verb.id,
|
||||
before: { ...metadata },
|
||||
after: result
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
hasMore = batch.hasMore
|
||||
offset += batch.items.length
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
pendingMigrations: pending.map(m => ({ id: m.id, description: m.description })),
|
||||
affectedEntities,
|
||||
totalEntities,
|
||||
sampleChanges,
|
||||
estimatedTime: this.estimateTime(totalEntities)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all pending migrations.
|
||||
* Iterates entities in paginated batches, transforms metadata, saves changes.
|
||||
* Resume-safe: saves offset after each batch so interrupted migrations can continue.
|
||||
*
|
||||
* Entity-level errors are tracked (not thrown). If maxErrors is exceeded, migration
|
||||
* stops early and returns partial results with errors.
|
||||
*/
|
||||
async run(options?: Pick<MigrateOptions, 'onProgress' | 'maxErrors'>): Promise<Omit<MigrationResult, 'backupBranch'>> {
|
||||
const state = await this.getState()
|
||||
const pending = this.getPendingMigrations(state)
|
||||
|
||||
if (pending.length === 0) {
|
||||
return { migrationsApplied: [], entitiesProcessed: 0, entitiesModified: 0, errors: [] }
|
||||
}
|
||||
|
||||
let totalProcessed = 0
|
||||
let totalModified = 0
|
||||
const appliedMigrations: string[] = []
|
||||
const errors: MigrationError[] = []
|
||||
const maxErrors = options?.maxErrors ?? DEFAULT_MAX_ERRORS
|
||||
const batchConfig = this.storage.getBatchConfig()
|
||||
const batchSize = batchConfig.maxBatchSize
|
||||
const batchDelay = batchConfig.batchDelayMs
|
||||
|
||||
for (const migration of pending) {
|
||||
if (errors.length >= maxErrors) break
|
||||
|
||||
const resumeOffset = state?.resumeState?.migrationId === migration.id
|
||||
? state.resumeState.lastProcessedOffset
|
||||
: 0
|
||||
|
||||
let processed = 0
|
||||
let modified = 0
|
||||
|
||||
// Process nouns
|
||||
if (migration.applies === 'nouns' || migration.applies === 'both') {
|
||||
const result = await this.migrateNouns(migration, resumeOffset, batchSize, batchDelay, errors, maxErrors, options?.onProgress)
|
||||
processed += result.processed
|
||||
modified += result.modified
|
||||
}
|
||||
|
||||
// Process verbs
|
||||
if (migration.applies === 'verbs' || migration.applies === 'both') {
|
||||
if (errors.length < maxErrors) {
|
||||
const result = await this.migrateVerbs(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress)
|
||||
processed += result.processed
|
||||
modified += result.modified
|
||||
}
|
||||
}
|
||||
|
||||
totalProcessed += processed
|
||||
totalModified += modified
|
||||
appliedMigrations.push(migration.id)
|
||||
|
||||
// Save completed migration state
|
||||
await this.saveState({
|
||||
completedVersion: migration.version,
|
||||
completedAt: Date.now(),
|
||||
completedMigrations: [...(state?.completedMigrations || []), migration.id],
|
||||
resumeState: undefined
|
||||
})
|
||||
}
|
||||
|
||||
// Clear state cache so next check reads fresh
|
||||
this.stateCache = undefined
|
||||
|
||||
return {
|
||||
migrationsApplied: appliedMigrations,
|
||||
entitiesProcessed: totalProcessed,
|
||||
entitiesModified: totalModified,
|
||||
errors
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run specific migrations without checking completion state.
|
||||
* Used for branch iterations where the state on main says "completed"
|
||||
* but branch-local entities still need transforming.
|
||||
*
|
||||
* Safe because transforms are idempotent (return null when already applied).
|
||||
* Does NOT save migration state — the authoritative state lives on main.
|
||||
*/
|
||||
async runMigrations(
|
||||
migrations: Migration[],
|
||||
options?: Pick<MigrateOptions, 'onProgress' | 'maxErrors'>
|
||||
): Promise<Omit<MigrationResult, 'backupBranch'>> {
|
||||
if (migrations.length === 0) {
|
||||
return { migrationsApplied: [], entitiesProcessed: 0, entitiesModified: 0, errors: [] }
|
||||
}
|
||||
|
||||
let totalProcessed = 0
|
||||
let totalModified = 0
|
||||
const appliedMigrations: string[] = []
|
||||
const errors: MigrationError[] = []
|
||||
const maxErrors = options?.maxErrors ?? DEFAULT_MAX_ERRORS
|
||||
const batchConfig = this.storage.getBatchConfig()
|
||||
const batchSize = batchConfig.maxBatchSize
|
||||
const batchDelay = batchConfig.batchDelayMs
|
||||
|
||||
for (const migration of migrations) {
|
||||
if (errors.length >= maxErrors) break
|
||||
|
||||
let processed = 0
|
||||
let modified = 0
|
||||
|
||||
if (migration.applies === 'nouns' || migration.applies === 'both') {
|
||||
const result = await this.migrateNouns(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress)
|
||||
processed += result.processed
|
||||
modified += result.modified
|
||||
}
|
||||
|
||||
if (migration.applies === 'verbs' || migration.applies === 'both') {
|
||||
if (errors.length < maxErrors) {
|
||||
const result = await this.migrateVerbs(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress)
|
||||
processed += result.processed
|
||||
modified += result.modified
|
||||
}
|
||||
}
|
||||
|
||||
totalProcessed += processed
|
||||
totalModified += modified
|
||||
if (modified > 0) {
|
||||
appliedMigrations.push(migration.id)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
migrationsApplied: appliedMigrations,
|
||||
entitiesProcessed: totalProcessed,
|
||||
entitiesModified: totalModified,
|
||||
errors
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old system:backup branches created by previous migrations.
|
||||
* Identifies backups via ref metadata (not by name prefix).
|
||||
*/
|
||||
async cleanupOldBackups(): Promise<void> {
|
||||
const refManager = this.storage.refManager
|
||||
if (!refManager) return
|
||||
|
||||
const refs = await refManager.listRefs()
|
||||
const currentBranch = this.storage.currentBranch || 'main'
|
||||
|
||||
for (const ref of refs) {
|
||||
if (
|
||||
ref.metadata?.type === 'system:backup' &&
|
||||
ref.name.startsWith('refs/heads/') &&
|
||||
ref.name !== `refs/heads/${currentBranch}`
|
||||
) {
|
||||
const branchName = ref.name.replace('refs/heads/', '')
|
||||
try {
|
||||
await refManager.deleteRef(branchName)
|
||||
} catch {
|
||||
// Ignore — branch may be current or protected
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Private helpers ───────────────────────────────────────────────
|
||||
|
||||
private async migrateNouns(
|
||||
migration: Migration,
|
||||
startOffset: number,
|
||||
batchSize: number,
|
||||
batchDelay: number,
|
||||
errors: MigrationError[],
|
||||
maxErrors: number,
|
||||
onProgress?: MigrateOptions['onProgress']
|
||||
): Promise<{ processed: number; modified: number }> {
|
||||
let offset = startOffset
|
||||
let hasMore = true
|
||||
let processed = 0
|
||||
let modified = 0
|
||||
|
||||
while (hasMore) {
|
||||
const batch = await this.storage.getNouns({ pagination: { offset, limit: batchSize } })
|
||||
|
||||
for (const entity of batch.items) {
|
||||
if (errors.length >= maxErrors) {
|
||||
return { processed, modified }
|
||||
}
|
||||
|
||||
processed++
|
||||
const metadata = await this.storage.getNounMetadataBatch([entity.id])
|
||||
const entityMeta = metadata.get(entity.id)
|
||||
if (!entityMeta) continue
|
||||
|
||||
try {
|
||||
const transformed = migration.transform(entityMeta as Record<string, unknown>)
|
||||
if (transformed !== null) {
|
||||
await this.storage.saveNounMetadata(entity.id, transformed as NounMetadata)
|
||||
modified++
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push({
|
||||
entityId: entity.id,
|
||||
migrationId: migration.id,
|
||||
error: err instanceof Error ? err.message : String(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
hasMore = batch.hasMore
|
||||
offset += batch.items.length
|
||||
|
||||
// Save resume state after each batch
|
||||
if (hasMore) {
|
||||
await this.saveResumeState(migration.id, offset)
|
||||
}
|
||||
|
||||
// Report progress
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
migrationId: migration.id,
|
||||
processed,
|
||||
modified,
|
||||
hasMore
|
||||
})
|
||||
}
|
||||
|
||||
// Respect adapter rate limiting
|
||||
if (batchDelay > 0 && hasMore) {
|
||||
await new Promise(resolve => setTimeout(resolve, batchDelay))
|
||||
}
|
||||
}
|
||||
|
||||
return { processed, modified }
|
||||
}
|
||||
|
||||
private async migrateVerbs(
|
||||
migration: Migration,
|
||||
startOffset: number,
|
||||
batchSize: number,
|
||||
batchDelay: number,
|
||||
errors: MigrationError[],
|
||||
maxErrors: number,
|
||||
onProgress?: MigrateOptions['onProgress']
|
||||
): Promise<{ processed: number; modified: number }> {
|
||||
let offset = startOffset
|
||||
let hasMore = true
|
||||
let processed = 0
|
||||
let modified = 0
|
||||
|
||||
while (hasMore) {
|
||||
const batch = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } })
|
||||
|
||||
for (const verb of batch.items) {
|
||||
if (errors.length >= maxErrors) {
|
||||
return { processed, modified }
|
||||
}
|
||||
|
||||
processed++
|
||||
const metadata = await this.storage.getVerbMetadata(verb.id)
|
||||
if (!metadata) continue
|
||||
|
||||
try {
|
||||
const transformed = migration.transform(metadata as Record<string, unknown>)
|
||||
if (transformed !== null) {
|
||||
await this.storage.saveVerbMetadata(verb.id, transformed as VerbMetadata)
|
||||
modified++
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push({
|
||||
entityId: verb.id,
|
||||
migrationId: migration.id,
|
||||
error: err instanceof Error ? err.message : String(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
hasMore = batch.hasMore
|
||||
offset += batch.items.length
|
||||
|
||||
// Save resume state after each batch
|
||||
if (hasMore) {
|
||||
await this.saveResumeState(migration.id, offset)
|
||||
}
|
||||
|
||||
// Report progress
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
migrationId: migration.id,
|
||||
processed,
|
||||
modified,
|
||||
hasMore
|
||||
})
|
||||
}
|
||||
|
||||
// Respect adapter rate limiting
|
||||
if (batchDelay > 0 && hasMore) {
|
||||
await new Promise(resolve => setTimeout(resolve, batchDelay))
|
||||
}
|
||||
}
|
||||
|
||||
return { processed, modified }
|
||||
}
|
||||
|
||||
private applyTransforms(metadata: Record<string, unknown>, migrations: Migration[]): Record<string, unknown> | null {
|
||||
let current = metadata
|
||||
let anyChanged = false
|
||||
|
||||
for (const migration of migrations) {
|
||||
const result = migration.transform(current)
|
||||
if (result !== null) {
|
||||
current = result
|
||||
anyChanged = true
|
||||
}
|
||||
}
|
||||
|
||||
return anyChanged ? current : null
|
||||
}
|
||||
|
||||
private getPendingMigrations(state: MigrationState | null): Migration[] {
|
||||
const completed = new Set(state?.completedMigrations || [])
|
||||
return MIGRATIONS.filter(m => !completed.has(m.id))
|
||||
}
|
||||
|
||||
private getPendingMigrationsFromCache(): Migration[] {
|
||||
const state = this.stateCache === undefined ? null : this.stateCache
|
||||
return this.getPendingMigrations(state)
|
||||
}
|
||||
|
||||
private async getState(): Promise<MigrationState | null> {
|
||||
if (this.stateCache !== undefined) return this.stateCache
|
||||
const state = await this.storage.getMetadata(MIGRATION_STATE_KEY) as unknown as MigrationState | null
|
||||
this.stateCache = state
|
||||
return state
|
||||
}
|
||||
|
||||
private async saveState(state: MigrationState): Promise<void> {
|
||||
await this.storage.saveMetadata(MIGRATION_STATE_KEY, state as unknown as NounMetadata)
|
||||
this.stateCache = state
|
||||
}
|
||||
|
||||
private async saveResumeState(migrationId: string, offset: number): Promise<void> {
|
||||
const state = await this.getState()
|
||||
const branch = this.storage.currentBranch || 'main'
|
||||
await this.saveState({
|
||||
completedVersion: state?.completedVersion || '',
|
||||
completedAt: state?.completedAt || 0,
|
||||
completedMigrations: state?.completedMigrations || [],
|
||||
resumeState: { migrationId, lastProcessedOffset: offset, branch }
|
||||
})
|
||||
}
|
||||
|
||||
private estimateTime(entityCount: number): string {
|
||||
if (entityCount === 0) return '0ms'
|
||||
if (entityCount < 1000) return '<1s'
|
||||
if (entityCount < 10000) return '~1-5s'
|
||||
if (entityCount < 100000) return '~10s-1min'
|
||||
if (entityCount < 1000000) return '~1-5min'
|
||||
return '~5min+'
|
||||
}
|
||||
}
|
||||
7
src/migration/index.ts
Normal file
7
src/migration/index.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
/**
|
||||
* Migration system public API
|
||||
*/
|
||||
|
||||
export { MigrationRunner } from './MigrationRunner.js'
|
||||
export { MIGRATIONS } from './migrations.js'
|
||||
export type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './types.js'
|
||||
21
src/migration/migrations.ts
Normal file
21
src/migration/migrations.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/**
|
||||
* Migration registry
|
||||
*
|
||||
* Ordered array of migrations. Each migration runs exactly once per storage instance.
|
||||
* Add new migrations at the end — order matters.
|
||||
*/
|
||||
|
||||
import type { Migration } from './types.js'
|
||||
|
||||
export const MIGRATIONS: Migration[] = [
|
||||
// Empty for v7.16.0 — framework scaffolded, ready for future use.
|
||||
// Example of a future migration:
|
||||
//
|
||||
// {
|
||||
// id: '7.17.0-rename-status-field',
|
||||
// version: '7.17.0',
|
||||
// description: 'Rename metadata.state to metadata.status',
|
||||
// applies: 'nouns',
|
||||
// transform: (m) => 'state' in m ? { ...m, status: m.state, state: undefined } : null
|
||||
// }
|
||||
]
|
||||
83
src/migration/types.ts
Normal file
83
src/migration/types.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Migration system types for Brainy
|
||||
*
|
||||
* Defines the interfaces for schema migrations that transform
|
||||
* entity/verb metadata across storage versions.
|
||||
*/
|
||||
|
||||
export interface Migration {
|
||||
/** Unique migration identifier, e.g., "7.17.0-rename-field" */
|
||||
id: string
|
||||
/** Version that introduced this migration */
|
||||
version: string
|
||||
/** Human-readable description of what this migration does */
|
||||
description: string
|
||||
/** Which entity types this migration applies to */
|
||||
applies: 'nouns' | 'verbs' | 'both'
|
||||
/** Return transformed metadata, or null if no change needed */
|
||||
transform: (metadata: Record<string, unknown>) => Record<string, unknown> | null
|
||||
}
|
||||
|
||||
export interface MigrationState {
|
||||
/** Last completed migration version */
|
||||
completedVersion: string
|
||||
/** Timestamp of last completed migration */
|
||||
completedAt: number
|
||||
/** List of completed migration IDs */
|
||||
completedMigrations: string[]
|
||||
/** Resume state for crash recovery */
|
||||
resumeState?: {
|
||||
migrationId: string
|
||||
lastProcessedOffset: number
|
||||
branch: string
|
||||
}
|
||||
}
|
||||
|
||||
export interface MigrationPreview {
|
||||
/** Migrations that will be applied */
|
||||
pendingMigrations: { id: string; description: string }[]
|
||||
/** Number of entities that would be modified */
|
||||
affectedEntities: number
|
||||
/** Total number of entities scanned */
|
||||
totalEntities: number
|
||||
/** Sample before/after transformations (up to 5) */
|
||||
sampleChanges: { id: string; before: Record<string, unknown>; after: Record<string, unknown> }[]
|
||||
/** Rough time estimate */
|
||||
estimatedTime: string
|
||||
}
|
||||
|
||||
export interface MigrationError {
|
||||
/** ID of the entity that failed */
|
||||
entityId: string
|
||||
/** ID of the migration that caused the failure */
|
||||
migrationId: string
|
||||
/** Error message */
|
||||
error: string
|
||||
}
|
||||
|
||||
export interface MigrationResult {
|
||||
/** Backup branch name, or null if no changes were needed */
|
||||
backupBranch: string | null
|
||||
/** IDs of migrations that were applied */
|
||||
migrationsApplied: string[]
|
||||
/** Total entities processed (scanned) */
|
||||
entitiesProcessed: number
|
||||
/** Entities actually modified */
|
||||
entitiesModified: number
|
||||
/** Errors encountered during migration (entity-level, non-fatal) */
|
||||
errors: MigrationError[]
|
||||
}
|
||||
|
||||
export interface MigrateOptions {
|
||||
/** Preview what would change without writing */
|
||||
dryRun?: boolean
|
||||
/** Progress callback for long-running migrations */
|
||||
onProgress?: (progress: {
|
||||
migrationId: string
|
||||
processed: number
|
||||
modified: number
|
||||
hasMore: boolean
|
||||
}) => void
|
||||
/** Maximum entity-level errors before bailing out (default: 100) */
|
||||
maxErrors?: number
|
||||
}
|
||||
|
|
@ -433,6 +433,22 @@ export class RefManager {
|
|||
await this.setRef(name, newCommitHash, options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update metadata on an existing ref (merge semantics).
|
||||
*
|
||||
* @param name - Reference name
|
||||
* @param metadata - Metadata fields to merge into the ref
|
||||
*/
|
||||
async updateRefMetadata(name: string, metadata: Record<string, unknown>): Promise<void> {
|
||||
const fullName = this.normalizeRefName(name)
|
||||
const ref = await this.getRef(fullName)
|
||||
if (!ref) throw new Error(`Ref not found: ${fullName}`)
|
||||
ref.metadata = { ...ref.metadata, ...metadata }
|
||||
ref.updatedAt = Date.now()
|
||||
await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref)))
|
||||
this.cache.set(fullName, ref)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get commit hash for reference
|
||||
*
|
||||
|
|
|
|||
|
|
@ -837,6 +837,11 @@ export interface BrainyConfig {
|
|||
// - false/undefined: Disable integrations (default)
|
||||
// - IntegrationsConfig: Custom configuration
|
||||
integrations?: boolean | IntegrationsConfig
|
||||
|
||||
// Migration configuration
|
||||
// - false/undefined (default): Log warning if pending migrations exist, but don't auto-run
|
||||
// - true: Automatically run pending migrations during init() for small datasets (<10K entities)
|
||||
autoMigrate?: boolean
|
||||
}
|
||||
|
||||
// ============= Neural API Types =============
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
import { Vector } from '../coreTypes.js'
|
||||
import { AddParams, RelateParams, Result, Entity, FindParams, SimilarParams } from './brainy.types.js'
|
||||
import { NounType, VerbType } from './graphTypes.js'
|
||||
import type { MigrationPreview, MigrationResult, MigrateOptions } from '../migration/types.js'
|
||||
|
||||
export interface BrainyInterface<T = unknown> {
|
||||
/**
|
||||
|
|
@ -159,4 +160,23 @@ export interface BrainyInterface<T = unknown> {
|
|||
entities: Entity<any>[]
|
||||
centroid?: number[]
|
||||
}>>
|
||||
|
||||
/**
|
||||
* Run pending data migrations, or preview what would change.
|
||||
*
|
||||
* @param options - Pass { dryRun: true } to preview without writing
|
||||
* @returns Migration result or preview depending on options
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* // Preview
|
||||
* const preview = await brain.migrate({ dryRun: true })
|
||||
* console.log(preview.affectedEntities)
|
||||
*
|
||||
* // Apply
|
||||
* const result = await brain.migrate()
|
||||
* console.log(result.backupBranch) // 'pre-migration-7.17.0'
|
||||
* ```
|
||||
*/
|
||||
migrate(options?: MigrateOptions): Promise<MigrationResult | MigrationPreview>
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue