From 39b099cafc833dbee3c26e9e601931a0414b697e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 9 Feb 2026 16:13:14 -0800 Subject: [PATCH] feat: add migration system with error handling, validation, and enterprise hardening - MigrationRunner: per-entity error tracking (non-fatal), maxErrors bail-out, static validateMigrations() called in constructor, branch error propagation - RefManager: updateRefMetadata() method for clean metadata updates - brainy.ts: eliminate as-any casts in migration methods, use updateRefMetadata, forward maxErrors through full chain including branch migrations - Types: MigrationError interface, errors field on MigrationResult, maxErrors on MigrateOptions - Package exports: MigrationError type, migrate() on BrainyInterface, autoMigrate config - 31 integration tests covering error handling, validation, branch error propagation - Documentation: docs/guides/schema-migrations.md --- .gitignore | 3 + docs/guides/schema-migrations.md | 286 ++++++++++ src/brainy.ts | 221 +++++++- src/index.ts | 4 + src/migration/MigrationRunner.ts | 544 +++++++++++++++++++ src/migration/index.ts | 7 + src/migration/migrations.ts | 21 + src/migration/types.ts | 83 +++ src/storage/cow/RefManager.ts | 16 + src/types/brainy.types.ts | 5 + src/types/brainyInterface.ts | 20 + tests/integration/migration.test.ts | 813 ++++++++++++++++++++++++++++ 12 files changed, 2022 insertions(+), 1 deletion(-) create mode 100644 docs/guides/schema-migrations.md create mode 100644 src/migration/MigrationRunner.ts create mode 100644 src/migration/index.ts create mode 100644 src/migration/migrations.ts create mode 100644 src/migration/types.ts create mode 100644 tests/integration/migration.test.ts diff --git a/.gitignore b/.gitignore index d0a27628..f3dad91a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ coverage/ # Test results tests/results/ +# Filesystem test artifacts (created by integration tests) +test-*/ + # IDE files .vscode/ .idea/ diff --git a/docs/guides/schema-migrations.md b/docs/guides/schema-migrations.md new file mode 100644 index 00000000..3eb19e6c --- /dev/null +++ b/docs/guides/schema-migrations.md @@ -0,0 +1,286 @@ +# Schema Migrations + +Brainy includes a built-in migration system for transforming entity and verb metadata across storage versions. Migrations are pure functions that run once per storage instance, with automatic backup, resume support, and error tracking. + +--- + +## Quick Start + +### 1. Define a migration + +Add your migration to the `MIGRATIONS` array in `src/migration/migrations.ts`: + +```typescript +import type { Migration } from './types.js' + +export const MIGRATIONS: Migration[] = [ + { + id: '7.17.0-rename-status', + version: '7.17.0', + description: 'Rename "state" field to "status"', + applies: 'nouns', + transform: (m) => { + if ('state' in m) { + const { state, ...rest } = m + return { ...rest, status: state } + } + return null // already migrated or not applicable + } + } +] +``` + +### 2. Ship the new version + +That's it. Brainy detects pending migrations on `init()` and either runs them automatically or warns the user to call `brain.migrate()`. + +--- + +## How It Works + +When `brain.init()` runs: + +1. **Detection** — reads migration state from storage (one key lookup). Compares completed migration IDs against the `MIGRATIONS` array. If nothing pending, cost is ~0ms. + +2. **Small datasets** (`autoMigrate: true`, <10K entities) — migrates inline during `init()`. + +3. **Large datasets or manual mode** — logs a warning. User calls `brain.migrate()` when ready. + +When `brain.migrate()` runs: + +1. **Backup** — creates an instant COW branch (`pre-migration-7.17.0`) tagged with `system:backup` metadata. Rollback is possible by switching to this branch. + +2. **Transform main branch** — iterates all nouns/verbs in paginated batches. For each entity, calls the `transform` function. If it returns a new object, saves it. If it returns `null`, skips. Vectors are never touched. + +3. **Transform other branches** — switches to each user branch, runs the same transforms. Inherited (already-migrated) entities return `null` and are skipped automatically. + +4. **Save state** — records each completed migration ID so it never re-runs. + +5. **Rebuild indexes** — if any entities were modified, rebuilds the MetadataIndex. + +--- + +## Writing Migrations + +### The Migration interface + +```typescript +interface Migration { + id: string // Unique ID, e.g. "7.17.0-rename-field" + version: string // Version that introduced this migration + description: string // Human-readable description + applies: 'nouns' | 'verbs' | 'both' + transform: (metadata: Record) => Record | null +} +``` + +### Transform rules + +- **Return a new object** to modify the entity's metadata. +- **Return `null`** to skip (no change needed). +- **Must be idempotent** — running the same transform twice on the same data should produce the same result (or return `null` the second time). This is required because branch iterations may re-encounter inherited entities. +- **Must be pure** — no side effects, no async, no external state. +- Transforms only see metadata. Vectors, embeddings, and the `data` field stored inside metadata are available as properties on the metadata object. + +### Ordering + +Migrations run in array order. Add new migrations at the end of the `MIGRATIONS` array. Each migration runs independently per entity — migration 2 sees the output of migration 1. + +### Validation + +`MigrationRunner.validateMigrations()` checks migration definitions and will throw on: + +- Duplicate IDs +- Invalid `applies` values (must be `'nouns'`, `'verbs'`, or `'both'`) +- Non-function `transform` +- Missing or empty `id`, `version`, or `description` + +--- + +## API Reference + +### `brain.migrate(options?)` + +```typescript +// Dry-run: preview what would change without writing +const preview = await brain.migrate({ dryRun: true }) +// preview.pendingMigrations — array of { id, description } +// preview.affectedEntities — count of entities that would change +// preview.totalEntities — count of entities scanned +// preview.sampleChanges — up to 5 before/after samples +// preview.estimatedTime — rough time estimate string + +// Apply migrations +const result = await brain.migrate() +// result.backupBranch — name of COW backup branch, or null +// result.migrationsApplied — array of migration IDs that ran +// result.entitiesProcessed — total entities scanned +// result.entitiesModified — entities actually changed +// result.errors — array of entity-level errors (non-fatal) +``` + +### Options + +```typescript +interface MigrateOptions { + dryRun?: boolean // Preview without writing (default: false) + maxErrors?: number // Bail out after N entity errors (default: 100) + onProgress?: (progress: { + migrationId: string + processed: number + modified: number + hasMore: boolean + }) => void +} +``` + +--- + +## Error Handling + +If a transform function throws on a specific entity, the error is recorded and migration continues to the next entity. The failed entity's metadata is left unchanged. + +```typescript +const result = await brain.migrate() + +if (result.errors.length > 0) { + for (const err of result.errors) { + console.warn(`Entity ${err.entityId} failed in ${err.migrationId}: ${err.error}`) + } +} +``` + +If errors exceed `maxErrors` (default: 100), the migration stops early and returns partial results. Successfully migrated entities keep their changes; failed entities are unchanged. + +```typescript +// Strict mode: fail fast on any error +const result = await brain.migrate({ maxErrors: 1 }) + +// Lenient mode: tolerate many errors +const result = await brain.migrate({ maxErrors: 10000 }) +``` + +--- + +## Backup and Rollback + +Before modifying any data, `brain.migrate()` calls `brain.fork()` to create a COW snapshot. This is instant regardless of dataset size — it's a pointer copy, not a data copy. + +The backup branch is named `pre-migration-{version}` and tagged with metadata: +- `type: 'system:backup'` +- `migrationVersion: '7.17.0'` +- `author: 'brainy-migration'` + +To roll back, switch to the backup branch: + +```typescript +await brain.checkout('pre-migration-7.17.0') +``` + +Old backup branches from previous migrations are cleaned up automatically before each new migration run. + +--- + +## Progress Tracking + +For large datasets, use the `onProgress` callback: + +```typescript +await brain.migrate({ + onProgress: ({ migrationId, processed, modified, hasMore }) => { + console.log(`[${migrationId}] ${processed} scanned, ${modified} modified${hasMore ? '...' : ' (done)'}`) + } +}) +``` + +Progress is reported after each batch (batch size is determined by the storage adapter). + +--- + +## Examples + +### Rename a field + +```typescript +{ + id: '7.17.0-rename-state-to-status', + version: '7.17.0', + description: 'Rename metadata.state to metadata.status', + applies: 'nouns', + transform: (m) => { + if ('state' in m) { + const { state, ...rest } = m + return { ...rest, status: state } + } + return null + } +} +``` + +### Add a default value + +```typescript +{ + id: '7.18.0-add-priority-default', + version: '7.18.0', + description: 'Add priority field with default "normal"', + applies: 'both', + transform: (m) => { + if (!('priority' in m)) { + return { ...m, priority: 'normal' } + } + return null + } +} +``` + +### Remove a deprecated field + +```typescript +{ + id: '7.19.0-remove-legacy-flag', + version: '7.19.0', + description: 'Remove deprecated "legacy" field', + applies: 'nouns', + transform: (m) => { + if ('legacy' in m) { + const { legacy, ...rest } = m + return rest + } + return null + } +} +``` + +### Transform verb metadata + +```typescript +{ + id: '7.20.0-normalize-verb-weights', + version: '7.20.0', + description: 'Normalize verb weights from 0-100 to 0-1 scale', + applies: 'verbs', + transform: (m) => { + if (typeof m.weight === 'number' && m.weight > 1) { + return { ...m, weight: m.weight / 100 } + } + return null + } +} +``` + +--- + +## Storage Backend Compatibility + +Migrations work identically across all storage backends (Memory, FileSystem, S3, R2, GCS, OPFS). The system uses `BaseStorage` methods (`getNouns`, `saveNounMetadata`, `getVerbs`, `saveVerbMetadata`) which are implemented by every adapter. + +Batch size and rate limiting are automatically configured per adapter — no tuning required. + +--- + +## What Migrations Don't Do + +- **Re-embedding** — migrations transform metadata only. If you change your embedding model or dimensions, that requires re-vectorizing data, which is a separate concern (not part of this system). +- **Vector modification** — the `vectors.json` files are never touched by migrations. +- **Schema enforcement** — migrations are opt-in transforms, not schema validators. Brainy's metadata is schemaless by design. diff --git a/src/brainy.ts b/src/brainy.ts index 9ecf40ca..60dc6863 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -88,6 +88,8 @@ import { import { NounType, VerbType } from './types/graphTypes.js' import { BrainyInterface } from './types/brainyInterface.js' import type { IntegrationHub } from './integrations/core/IntegrationHub.js' +import { MigrationRunner } from './migration/MigrationRunner.js' +import type { MigrationPreview, MigrationResult, MigrateOptions } from './migration/types.js' /** * Stopwords for semantic highlighting @@ -169,6 +171,7 @@ export class Brainy implements BrainyInterface { private _vfs?: VirtualFileSystem private _vfsInitialized = false // Track VFS init completion separately private _hub?: IntegrationHub // Integration Hub for external tools + private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets // State private initialized = false @@ -351,6 +354,9 @@ export class Brainy implements BrainyInterface { // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() + // Check for pending data migrations + await this.checkMigrations() + // Connect distributed components to storage await this.connectDistributedStorage() @@ -3551,6 +3557,217 @@ export class Brainy implements BrainyInterface { await refManager.deleteRef(branch) } + // ─── Migration API ─────────────────────────────────────────────── + + /** + * Run pending data migrations, or preview what would change. + * + * @param options - Pass { dryRun: true } to preview without writing + * @returns Migration result (or preview if dryRun) + * + * @example + * ```typescript + * // Preview what would change + * const preview = await brain.migrate({ dryRun: true }) + * console.log(preview.affectedEntities) + * + * // Apply migrations (auto-forks a backup branch first) + * const result = await brain.migrate() + * console.log(result.backupBranch) // 'pre-migration-7.17.0' + * + * // Rollback if needed + * await brain.checkout('pre-migration-7.17.0') + * ``` + */ + async migrate(options?: MigrateOptions): Promise { + await this.ensureInitialized() + const runner = this._pendingMigrationRunner || new MigrationRunner(this.storage) + + if (options?.dryRun) { + return runner.preview() + } + + return this.migrateInternal(runner, options) + } + + /** + * Check for pending migrations during init(). + * Runs inline for small datasets if autoMigrate is enabled, + * otherwise logs a warning. + */ + private async checkMigrations(): Promise { + const runner = new MigrationRunner(this.storage) + + if (!(await runner.hasPendingMigrations())) { + return + } + + const count = await runner.pendingCount() + + if (this.config.autoMigrate) { + // Quick entity count check to decide inline vs deferred + const probe = await this.storage.getNouns({ pagination: { limit: 1 } }) + const totalEstimate = probe.totalCount ?? (probe.hasMore ? 10001 : probe.items.length) + + if (totalEstimate < 10000) { + // Small dataset — migrate inline during init + await this.migrateInternal(runner) + } else { + // Large dataset — defer to explicit brain.migrate() call + this._pendingMigrationRunner = runner + if (!this.config.silent) { + console.log(`[brainy] ${count} pending migration(s) detected. Call brain.migrate() to apply (dataset too large for inline migration).`) + } + } + } else { + if (!this.config.silent) { + console.log(`[brainy] ${count} pending migration(s) available. Set autoMigrate: true or call brain.migrate() to apply.`) + } + } + } + + /** + * Internal: fork backup, run migrations on current branch + all other branches. + * + * Branch strategy: + * - getNouns() uses listObjectsInBranch() which only returns branch-local entities + * - So migrating main only transforms main's entities; branch-local entities are untouched + * - After main, we iterate all other user branches and run the same transforms + * - Lightweight: just switches storage.currentBranch (no full checkout/index rebuild) + * - Transforms are idempotent (return null when already applied), so this is safe + */ + private async migrateInternal(runner: MigrationRunner, options?: MigrateOptions): Promise { + // 0. Clean up old migration backup branches (by metadata tag, not name) + await runner.cleanupOldBackups() + + // 1. Fork backup branch (uses existing COW — instant) + const version = runner.nextMigrationVersion() + const backupName = `pre-migration-${version}` + let backupCreated = false + + try { + await this.fork(backupName) + backupCreated = true + + // Tag the backup branch with metadata so we can identify it later + if (this.storage.refManager) { + await this.storage.refManager.updateRefMetadata(backupName, { + type: 'system:backup', + migrationVersion: version, + author: 'brainy-migration' + }) + } + } catch { + // Fork may fail if COW not initialized (e.g., memory storage with no commits) + // Continue without backup — migrations are still safe (user can re-import) + } + + // 2. Run migrations on current branch + const runResult = await runner.run({ onProgress: options?.onProgress, maxErrors: options?.maxErrors }) + + // 3. Migrate all other user branches (branch-local entities only) + // getNouns() uses listObjectsInBranch() which only lists branch-overlay files, + // so each branch iteration only touches entities written directly to that branch. + const branchResult = await this.migrateOtherBranches( + runResult.migrationsApplied, + backupCreated ? backupName : null, + options + ) + + // 4. Rebuild MetadataIndex if any entities were modified (on the current branch) + const totalModified = runResult.entitiesModified + branchResult.entitiesModified + if (totalModified > 0) { + await this.metadataIndex.rebuild() + } + + // 5. Clear deferred runner + this._pendingMigrationRunner = undefined + + return { + backupBranch: backupCreated ? backupName : null, + migrationsApplied: runResult.migrationsApplied, + entitiesProcessed: runResult.entitiesProcessed + branchResult.entitiesProcessed, + entitiesModified: totalModified, + errors: [...(runResult.errors || []), ...branchResult.errors] + } + } + + /** + * Migrate branch-local entities on all non-current branches. + * + * Why this is needed: getNouns() lists entities from the branch overlay only + * (via listObjectsInBranch), not inherited entities. So migrating main doesn't + * touch entities written directly to feature branches. We iterate each branch + * and run the same transforms — they're idempotent, so already-migrated + * inherited entities return null and are skipped. + * + * Lightweight: switches storage.currentBranch directly instead of full checkout() + * (no index rebuild needed — migration uses storage-level methods only). + */ + private async migrateOtherBranches( + migrationIds: string[], + skipBranch: string | null, + options?: MigrateOptions + ): Promise<{ entitiesProcessed: number; entitiesModified: number; errors: import('./migration/types.js').MigrationError[] }> { + const empty = { entitiesProcessed: 0, entitiesModified: 0, errors: [] as import('./migration/types.js').MigrationError[] } + if (migrationIds.length === 0) return empty + + // Only if COW branching is available + const refManager = this.storage.refManager + if (!refManager) return empty + + const currentBranch = this.storage.currentBranch || 'main' + let totalProcessed = 0 + let totalModified = 0 + const allErrors: import('./migration/types.js').MigrationError[] = [] + + try { + const { MIGRATIONS } = await import('./migration/migrations.js') + const migrationsToRun = MIGRATIONS.filter(m => migrationIds.includes(m.id)) + if (migrationsToRun.length === 0) return empty + + const refs = await refManager.listRefs() + const branches = refs + .filter(ref => ref.name.startsWith('refs/heads/')) + .map(ref => ({ + name: ref.name.replace('refs/heads/', ''), + metadata: ref.metadata + })) + + for (const branch of branches) { + // Skip current branch (already migrated above) + if (branch.name === currentBranch) continue + // Skip the backup branch we just created + if (branch.name === skipBranch) continue + // Skip system backup branches + if (branch.metadata?.type === 'system:backup') continue + + // Switch storage to this branch (lightweight — no index rebuild) + this.storage.currentBranch = branch.name + + // Run transforms — idempotent, so inherited entities return null and are skipped. + // Uses runMigrations() which bypasses the state check (state on main says "completed" + // but branch-local entities haven't been touched yet). + const branchRunner = new MigrationRunner(this.storage) + const result = await branchRunner.runMigrations(migrationsToRun, { + onProgress: options?.onProgress, + maxErrors: options?.maxErrors + }) + + totalProcessed += result.entitiesProcessed + totalModified += result.entitiesModified + if (result.errors.length > 0) { + allErrors.push(...result.errors) + } + } + } finally { + // Always restore original branch + this.storage.currentBranch = currentBranch + } + + return { entitiesProcessed: totalProcessed, entitiesModified: totalModified, errors: allErrors } + } + /** * Get commit history for current branch * @param options - History options (limit, offset, author) @@ -6616,7 +6833,9 @@ export class Brainy implements BrainyInterface { // Plugin configuration - undefined = auto-detect plugins: config?.plugins ?? undefined as any, // Integration Hub - undefined/false = disabled - integrations: config?.integrations ?? undefined as any + integrations: config?.integrations ?? undefined as any, + // Migration — disabled by default, opt-in for automatic migration + autoMigrate: config?.autoMigrate ?? false } } diff --git a/src/index.ts b/src/index.ts index cfa93062..74d80a6f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -106,6 +106,10 @@ export { getBrainyVersion } from './utils/version.js' export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' export { PluginRegistry } from './plugin.js' +// Export migration system +export { MigrationRunner, MIGRATIONS } from './migration/index.js' +export type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './migration/index.js' + // Export embedding functionality import { UniversalSentenceEncoder, diff --git a/src/migration/MigrationRunner.ts b/src/migration/MigrationRunner.ts new file mode 100644 index 00000000..c848673f --- /dev/null +++ b/src/migration/MigrationRunner.ts @@ -0,0 +1,544 @@ +/** + * MigrationRunner: Executes schema migrations on Brainy storage + * + * Handles paginated iteration, resume-safe batching, and dry-run previews. + * Uses BaseStorage methods directly — no adapter-level changes needed. + */ + +import type { BaseStorage } from '../storage/baseStorage.js' +import type { NounMetadata, VerbMetadata } from '../coreTypes.js' +import type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './types.js' +import { MIGRATIONS } from './migrations.js' + +const MIGRATION_STATE_KEY = '__migration_state__' +const PREVIEW_SAMPLE_SIZE = 5 +const DEFAULT_MAX_ERRORS = 100 + +export class MigrationRunner { + private storage: BaseStorage + private stateCache: MigrationState | null | undefined = undefined + + constructor(storage: BaseStorage) { + this.storage = storage + MigrationRunner.validateMigrations(MIGRATIONS) + } + + /** + * Validate migration definitions. + * Called automatically in constructor for the global MIGRATIONS array. + * Also available as a static method for validating custom migration arrays. + */ + static validateMigrations(migrations: Migration[]): void { + if (migrations.length === 0) return + + const seenIds = new Set() + const validApplies = new Set(['nouns', 'verbs', 'both']) + + for (const m of migrations) { + if (!m.id || typeof m.id !== 'string') { + throw new Error(`Migration has missing or invalid id`) + } + if (seenIds.has(m.id)) { + throw new Error(`Duplicate migration id: "${m.id}"`) + } + seenIds.add(m.id) + + if (!m.version || typeof m.version !== 'string') { + throw new Error(`Migration "${m.id}" has missing or invalid version`) + } + if (!m.description || typeof m.description !== 'string') { + throw new Error(`Migration "${m.id}" has missing or invalid description`) + } + if (!validApplies.has(m.applies)) { + throw new Error(`Migration "${m.id}" has invalid applies value: "${m.applies}" (must be "nouns", "verbs", or "both")`) + } + if (typeof m.transform !== 'function') { + throw new Error(`Migration "${m.id}" has non-function transform`) + } + } + } + + /** + * Check if there are pending migrations to run. + * Single getMetadata() call — ~0ms overhead when no migrations exist. + */ + async hasPendingMigrations(): Promise { + if (MIGRATIONS.length === 0) return false + const state = await this.getState() + return this.getPendingMigrations(state).length > 0 + } + + /** + * Get the version string for the next pending migration. + */ + nextMigrationVersion(): string { + const pending = this.getPendingMigrationsFromCache() + return pending.length > 0 ? pending[pending.length - 1].version : 'unknown' + } + + /** + * Get count of pending migrations (for log messages). + */ + async pendingCount(): Promise { + if (MIGRATIONS.length === 0) return 0 + const state = await this.getState() + return this.getPendingMigrations(state).length + } + + /** + * Preview what a migration would do without writing anything. + * Scans entities, applies transforms in memory, reports counts + samples. + */ + async preview(): Promise { + const state = await this.getState() + const pending = this.getPendingMigrations(state) + + if (pending.length === 0) { + return { + pendingMigrations: [], + affectedEntities: 0, + totalEntities: 0, + sampleChanges: [], + estimatedTime: '0ms' + } + } + + let totalEntities = 0 + let affectedEntities = 0 + const sampleChanges: MigrationPreview['sampleChanges'] = [] + const batchConfig = this.storage.getBatchConfig() + const batchSize = batchConfig.maxBatchSize + + // Scan nouns if any pending migration applies to nouns + const nounMigrations = pending.filter(m => m.applies === 'nouns' || m.applies === 'both') + if (nounMigrations.length > 0) { + let offset = 0 + let hasMore = true + + while (hasMore) { + const batch = await this.storage.getNouns({ pagination: { offset, limit: batchSize } }) + const ids = batch.items.map(e => e.id) + const metadataBatch = await this.storage.getNounMetadataBatch(ids) + + for (const entity of batch.items) { + totalEntities++ + const entityMeta = metadataBatch.get(entity.id) + if (!entityMeta) continue + + const metadata = entityMeta as Record + const result = this.applyTransforms(metadata, nounMigrations) + if (result !== null) { + affectedEntities++ + if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) { + sampleChanges.push({ + id: entity.id, + before: { ...metadata }, + after: result + }) + } + } + } + hasMore = batch.hasMore + offset += batch.items.length + } + } + + // Scan verbs if any pending migration applies to verbs + const verbMigrations = pending.filter(m => m.applies === 'verbs' || m.applies === 'both') + if (verbMigrations.length > 0) { + let offset = 0 + let hasMore = true + + while (hasMore) { + const batch = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } }) + + for (const verb of batch.items) { + totalEntities++ + const verbMeta = await this.storage.getVerbMetadata(verb.id) + if (!verbMeta) continue + + const metadata = verbMeta as Record + const result = this.applyTransforms(metadata, verbMigrations) + if (result !== null) { + affectedEntities++ + if (sampleChanges.length < PREVIEW_SAMPLE_SIZE) { + sampleChanges.push({ + id: verb.id, + before: { ...metadata }, + after: result + }) + } + } + } + hasMore = batch.hasMore + offset += batch.items.length + } + } + + return { + pendingMigrations: pending.map(m => ({ id: m.id, description: m.description })), + affectedEntities, + totalEntities, + sampleChanges, + estimatedTime: this.estimateTime(totalEntities) + } + } + + /** + * Run all pending migrations. + * Iterates entities in paginated batches, transforms metadata, saves changes. + * Resume-safe: saves offset after each batch so interrupted migrations can continue. + * + * Entity-level errors are tracked (not thrown). If maxErrors is exceeded, migration + * stops early and returns partial results with errors. + */ + async run(options?: Pick): Promise> { + const state = await this.getState() + const pending = this.getPendingMigrations(state) + + if (pending.length === 0) { + return { migrationsApplied: [], entitiesProcessed: 0, entitiesModified: 0, errors: [] } + } + + let totalProcessed = 0 + let totalModified = 0 + const appliedMigrations: string[] = [] + const errors: MigrationError[] = [] + const maxErrors = options?.maxErrors ?? DEFAULT_MAX_ERRORS + const batchConfig = this.storage.getBatchConfig() + const batchSize = batchConfig.maxBatchSize + const batchDelay = batchConfig.batchDelayMs + + for (const migration of pending) { + if (errors.length >= maxErrors) break + + const resumeOffset = state?.resumeState?.migrationId === migration.id + ? state.resumeState.lastProcessedOffset + : 0 + + let processed = 0 + let modified = 0 + + // Process nouns + if (migration.applies === 'nouns' || migration.applies === 'both') { + const result = await this.migrateNouns(migration, resumeOffset, batchSize, batchDelay, errors, maxErrors, options?.onProgress) + processed += result.processed + modified += result.modified + } + + // Process verbs + if (migration.applies === 'verbs' || migration.applies === 'both') { + if (errors.length < maxErrors) { + const result = await this.migrateVerbs(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress) + processed += result.processed + modified += result.modified + } + } + + totalProcessed += processed + totalModified += modified + appliedMigrations.push(migration.id) + + // Save completed migration state + await this.saveState({ + completedVersion: migration.version, + completedAt: Date.now(), + completedMigrations: [...(state?.completedMigrations || []), migration.id], + resumeState: undefined + }) + } + + // Clear state cache so next check reads fresh + this.stateCache = undefined + + return { + migrationsApplied: appliedMigrations, + entitiesProcessed: totalProcessed, + entitiesModified: totalModified, + errors + } + } + + /** + * Run specific migrations without checking completion state. + * Used for branch iterations where the state on main says "completed" + * but branch-local entities still need transforming. + * + * Safe because transforms are idempotent (return null when already applied). + * Does NOT save migration state — the authoritative state lives on main. + */ + async runMigrations( + migrations: Migration[], + options?: Pick + ): Promise> { + if (migrations.length === 0) { + return { migrationsApplied: [], entitiesProcessed: 0, entitiesModified: 0, errors: [] } + } + + let totalProcessed = 0 + let totalModified = 0 + const appliedMigrations: string[] = [] + const errors: MigrationError[] = [] + const maxErrors = options?.maxErrors ?? DEFAULT_MAX_ERRORS + const batchConfig = this.storage.getBatchConfig() + const batchSize = batchConfig.maxBatchSize + const batchDelay = batchConfig.batchDelayMs + + for (const migration of migrations) { + if (errors.length >= maxErrors) break + + let processed = 0 + let modified = 0 + + if (migration.applies === 'nouns' || migration.applies === 'both') { + const result = await this.migrateNouns(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress) + processed += result.processed + modified += result.modified + } + + if (migration.applies === 'verbs' || migration.applies === 'both') { + if (errors.length < maxErrors) { + const result = await this.migrateVerbs(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress) + processed += result.processed + modified += result.modified + } + } + + totalProcessed += processed + totalModified += modified + if (modified > 0) { + appliedMigrations.push(migration.id) + } + } + + return { + migrationsApplied: appliedMigrations, + entitiesProcessed: totalProcessed, + entitiesModified: totalModified, + errors + } + } + + /** + * Clean up old system:backup branches created by previous migrations. + * Identifies backups via ref metadata (not by name prefix). + */ + async cleanupOldBackups(): Promise { + const refManager = this.storage.refManager + if (!refManager) return + + const refs = await refManager.listRefs() + const currentBranch = this.storage.currentBranch || 'main' + + for (const ref of refs) { + if ( + ref.metadata?.type === 'system:backup' && + ref.name.startsWith('refs/heads/') && + ref.name !== `refs/heads/${currentBranch}` + ) { + const branchName = ref.name.replace('refs/heads/', '') + try { + await refManager.deleteRef(branchName) + } catch { + // Ignore — branch may be current or protected + } + } + } + } + + // ─── Private helpers ─────────────────────────────────────────────── + + private async migrateNouns( + migration: Migration, + startOffset: number, + batchSize: number, + batchDelay: number, + errors: MigrationError[], + maxErrors: number, + onProgress?: MigrateOptions['onProgress'] + ): Promise<{ processed: number; modified: number }> { + let offset = startOffset + let hasMore = true + let processed = 0 + let modified = 0 + + while (hasMore) { + const batch = await this.storage.getNouns({ pagination: { offset, limit: batchSize } }) + + for (const entity of batch.items) { + if (errors.length >= maxErrors) { + return { processed, modified } + } + + processed++ + const metadata = await this.storage.getNounMetadataBatch([entity.id]) + const entityMeta = metadata.get(entity.id) + if (!entityMeta) continue + + try { + const transformed = migration.transform(entityMeta as Record) + if (transformed !== null) { + await this.storage.saveNounMetadata(entity.id, transformed as NounMetadata) + modified++ + } + } catch (err) { + errors.push({ + entityId: entity.id, + migrationId: migration.id, + error: err instanceof Error ? err.message : String(err) + }) + } + } + + hasMore = batch.hasMore + offset += batch.items.length + + // Save resume state after each batch + if (hasMore) { + await this.saveResumeState(migration.id, offset) + } + + // Report progress + if (onProgress) { + onProgress({ + migrationId: migration.id, + processed, + modified, + hasMore + }) + } + + // Respect adapter rate limiting + if (batchDelay > 0 && hasMore) { + await new Promise(resolve => setTimeout(resolve, batchDelay)) + } + } + + return { processed, modified } + } + + private async migrateVerbs( + migration: Migration, + startOffset: number, + batchSize: number, + batchDelay: number, + errors: MigrationError[], + maxErrors: number, + onProgress?: MigrateOptions['onProgress'] + ): Promise<{ processed: number; modified: number }> { + let offset = startOffset + let hasMore = true + let processed = 0 + let modified = 0 + + while (hasMore) { + const batch = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } }) + + for (const verb of batch.items) { + if (errors.length >= maxErrors) { + return { processed, modified } + } + + processed++ + const metadata = await this.storage.getVerbMetadata(verb.id) + if (!metadata) continue + + try { + const transformed = migration.transform(metadata as Record) + if (transformed !== null) { + await this.storage.saveVerbMetadata(verb.id, transformed as VerbMetadata) + modified++ + } + } catch (err) { + errors.push({ + entityId: verb.id, + migrationId: migration.id, + error: err instanceof Error ? err.message : String(err) + }) + } + } + + hasMore = batch.hasMore + offset += batch.items.length + + // Save resume state after each batch + if (hasMore) { + await this.saveResumeState(migration.id, offset) + } + + // Report progress + if (onProgress) { + onProgress({ + migrationId: migration.id, + processed, + modified, + hasMore + }) + } + + // Respect adapter rate limiting + if (batchDelay > 0 && hasMore) { + await new Promise(resolve => setTimeout(resolve, batchDelay)) + } + } + + return { processed, modified } + } + + private applyTransforms(metadata: Record, migrations: Migration[]): Record | null { + let current = metadata + let anyChanged = false + + for (const migration of migrations) { + const result = migration.transform(current) + if (result !== null) { + current = result + anyChanged = true + } + } + + return anyChanged ? current : null + } + + private getPendingMigrations(state: MigrationState | null): Migration[] { + const completed = new Set(state?.completedMigrations || []) + return MIGRATIONS.filter(m => !completed.has(m.id)) + } + + private getPendingMigrationsFromCache(): Migration[] { + const state = this.stateCache === undefined ? null : this.stateCache + return this.getPendingMigrations(state) + } + + private async getState(): Promise { + if (this.stateCache !== undefined) return this.stateCache + const state = await this.storage.getMetadata(MIGRATION_STATE_KEY) as unknown as MigrationState | null + this.stateCache = state + return state + } + + private async saveState(state: MigrationState): Promise { + await this.storage.saveMetadata(MIGRATION_STATE_KEY, state as unknown as NounMetadata) + this.stateCache = state + } + + private async saveResumeState(migrationId: string, offset: number): Promise { + const state = await this.getState() + const branch = this.storage.currentBranch || 'main' + await this.saveState({ + completedVersion: state?.completedVersion || '', + completedAt: state?.completedAt || 0, + completedMigrations: state?.completedMigrations || [], + resumeState: { migrationId, lastProcessedOffset: offset, branch } + }) + } + + private estimateTime(entityCount: number): string { + if (entityCount === 0) return '0ms' + if (entityCount < 1000) return '<1s' + if (entityCount < 10000) return '~1-5s' + if (entityCount < 100000) return '~10s-1min' + if (entityCount < 1000000) return '~1-5min' + return '~5min+' + } +} diff --git a/src/migration/index.ts b/src/migration/index.ts new file mode 100644 index 00000000..e3f101d0 --- /dev/null +++ b/src/migration/index.ts @@ -0,0 +1,7 @@ +/** + * Migration system public API + */ + +export { MigrationRunner } from './MigrationRunner.js' +export { MIGRATIONS } from './migrations.js' +export type { Migration, MigrationState, MigrationPreview, MigrationResult, MigrateOptions, MigrationError } from './types.js' diff --git a/src/migration/migrations.ts b/src/migration/migrations.ts new file mode 100644 index 00000000..a02950ad --- /dev/null +++ b/src/migration/migrations.ts @@ -0,0 +1,21 @@ +/** + * Migration registry + * + * Ordered array of migrations. Each migration runs exactly once per storage instance. + * Add new migrations at the end — order matters. + */ + +import type { Migration } from './types.js' + +export const MIGRATIONS: Migration[] = [ + // Empty for v7.16.0 — framework scaffolded, ready for future use. + // Example of a future migration: + // + // { + // id: '7.17.0-rename-status-field', + // version: '7.17.0', + // description: 'Rename metadata.state to metadata.status', + // applies: 'nouns', + // transform: (m) => 'state' in m ? { ...m, status: m.state, state: undefined } : null + // } +] diff --git a/src/migration/types.ts b/src/migration/types.ts new file mode 100644 index 00000000..53a23930 --- /dev/null +++ b/src/migration/types.ts @@ -0,0 +1,83 @@ +/** + * Migration system types for Brainy + * + * Defines the interfaces for schema migrations that transform + * entity/verb metadata across storage versions. + */ + +export interface Migration { + /** Unique migration identifier, e.g., "7.17.0-rename-field" */ + id: string + /** Version that introduced this migration */ + version: string + /** Human-readable description of what this migration does */ + description: string + /** Which entity types this migration applies to */ + applies: 'nouns' | 'verbs' | 'both' + /** Return transformed metadata, or null if no change needed */ + transform: (metadata: Record) => Record | null +} + +export interface MigrationState { + /** Last completed migration version */ + completedVersion: string + /** Timestamp of last completed migration */ + completedAt: number + /** List of completed migration IDs */ + completedMigrations: string[] + /** Resume state for crash recovery */ + resumeState?: { + migrationId: string + lastProcessedOffset: number + branch: string + } +} + +export interface MigrationPreview { + /** Migrations that will be applied */ + pendingMigrations: { id: string; description: string }[] + /** Number of entities that would be modified */ + affectedEntities: number + /** Total number of entities scanned */ + totalEntities: number + /** Sample before/after transformations (up to 5) */ + sampleChanges: { id: string; before: Record; after: Record }[] + /** Rough time estimate */ + estimatedTime: string +} + +export interface MigrationError { + /** ID of the entity that failed */ + entityId: string + /** ID of the migration that caused the failure */ + migrationId: string + /** Error message */ + error: string +} + +export interface MigrationResult { + /** Backup branch name, or null if no changes were needed */ + backupBranch: string | null + /** IDs of migrations that were applied */ + migrationsApplied: string[] + /** Total entities processed (scanned) */ + entitiesProcessed: number + /** Entities actually modified */ + entitiesModified: number + /** Errors encountered during migration (entity-level, non-fatal) */ + errors: MigrationError[] +} + +export interface MigrateOptions { + /** Preview what would change without writing */ + dryRun?: boolean + /** Progress callback for long-running migrations */ + onProgress?: (progress: { + migrationId: string + processed: number + modified: number + hasMore: boolean + }) => void + /** Maximum entity-level errors before bailing out (default: 100) */ + maxErrors?: number +} diff --git a/src/storage/cow/RefManager.ts b/src/storage/cow/RefManager.ts index 453a07ba..69203029 100644 --- a/src/storage/cow/RefManager.ts +++ b/src/storage/cow/RefManager.ts @@ -433,6 +433,22 @@ export class RefManager { await this.setRef(name, newCommitHash, options) } + /** + * Update metadata on an existing ref (merge semantics). + * + * @param name - Reference name + * @param metadata - Metadata fields to merge into the ref + */ + async updateRefMetadata(name: string, metadata: Record): Promise { + const fullName = this.normalizeRefName(name) + const ref = await this.getRef(fullName) + if (!ref) throw new Error(`Ref not found: ${fullName}`) + ref.metadata = { ...ref.metadata, ...metadata } + ref.updatedAt = Date.now() + await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref))) + this.cache.set(fullName, ref) + } + /** * Get commit hash for reference * diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index a9562771..14e04190 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -837,6 +837,11 @@ export interface BrainyConfig { // - false/undefined: Disable integrations (default) // - IntegrationsConfig: Custom configuration integrations?: boolean | IntegrationsConfig + + // Migration configuration + // - false/undefined (default): Log warning if pending migrations exist, but don't auto-run + // - true: Automatically run pending migrations during init() for small datasets (<10K entities) + autoMigrate?: boolean } // ============= Neural API Types ============= diff --git a/src/types/brainyInterface.ts b/src/types/brainyInterface.ts index 642fc675..fe42c441 100644 --- a/src/types/brainyInterface.ts +++ b/src/types/brainyInterface.ts @@ -10,6 +10,7 @@ import { Vector } from '../coreTypes.js' import { AddParams, RelateParams, Result, Entity, FindParams, SimilarParams } from './brainy.types.js' import { NounType, VerbType } from './graphTypes.js' +import type { MigrationPreview, MigrationResult, MigrateOptions } from '../migration/types.js' export interface BrainyInterface { /** @@ -159,4 +160,23 @@ export interface BrainyInterface { entities: Entity[] centroid?: number[] }>> + + /** + * Run pending data migrations, or preview what would change. + * + * @param options - Pass { dryRun: true } to preview without writing + * @returns Migration result or preview depending on options + * + * @example + * ```typescript + * // Preview + * const preview = await brain.migrate({ dryRun: true }) + * console.log(preview.affectedEntities) + * + * // Apply + * const result = await brain.migrate() + * console.log(result.backupBranch) // 'pre-migration-7.17.0' + * ``` + */ + migrate(options?: MigrateOptions): Promise } \ No newline at end of file diff --git a/tests/integration/migration.test.ts b/tests/integration/migration.test.ts new file mode 100644 index 00000000..ff115a10 --- /dev/null +++ b/tests/integration/migration.test.ts @@ -0,0 +1,813 @@ +/** + * Migration System Integration Tests + * + * Verifies: + * - MigrationRunner with in-memory storage + * - brain.migrate() public API (dry-run and apply) + * - Backup branch creation with metadata tagging + * - No-op when MIGRATIONS array is empty + * - Warning log when autoMigrate is false + * - Multi-tenant independent migration state + * - Verb and noun transforms + */ + +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { MigrationRunner, MIGRATIONS } from '../../src/migration/index.js' +import type { Migration } from '../../src/migration/index.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +// Helper to temporarily inject migrations into the MIGRATIONS array +function withMigrations(migrations: Migration[], fn: () => Promise): Promise { + const original = MIGRATIONS.splice(0, MIGRATIONS.length) + MIGRATIONS.push(...migrations) + return fn().finally(() => { + MIGRATIONS.splice(0, MIGRATIONS.length) + MIGRATIONS.push(...original) + }) +} + +describe('Migration System', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ storage: { type: 'memory' }, silent: true }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + // ─── No-op tests ─────────────────────────────────────────────── + + describe('No-op with empty MIGRATIONS', () => { + it('should have zero pending migrations by default', async () => { + // MIGRATIONS is empty by default — init() should add ~0ms overhead + const preview = await brain.migrate({ dryRun: true }) + expect(preview).toHaveProperty('pendingMigrations') + const p = preview as any + expect(p.pendingMigrations).toHaveLength(0) + expect(p.affectedEntities).toBe(0) + }) + + it('should return no-op result when applying with no migrations', async () => { + const result = await brain.migrate() + expect(result).toHaveProperty('migrationsApplied') + const r = result as any + expect(r.migrationsApplied).toHaveLength(0) + expect(r.entitiesModified).toBe(0) + }) + }) + + // ─── Dry-run preview ────────────────────────────────────────── + + describe('Dry-run preview', () => { + it('should preview affected entities without writing', async () => { + // Add some entities with a unique marker field + const id1 = await brain.add({ type: NounType.Concept, data: { name: 'Alpha' }, metadata: { status: 'active' } }) + const id2 = await brain.add({ type: NounType.Concept, data: { name: 'Beta' }, metadata: { status: 'inactive' } }) + const id3 = await brain.add({ type: NounType.Concept, data: { name: 'Gamma' }, metadata: { status: 'pending' } }) + + const testMigration: Migration = { + id: 'test-1.0.0-add-version', + version: '1.0.0', + description: 'Add version field to entities with status', + applies: 'nouns', + transform: (m) => { + // Only transform entities that have our specific 'status' field + if ('status' in m && !('version' in m)) { + return { ...m, version: 1 } + } + return null + } + } + + await withMigrations([testMigration], async () => { + const preview = await brain.migrate({ dryRun: true }) + const p = preview as any + expect(p.pendingMigrations).toHaveLength(1) + expect(p.pendingMigrations[0].id).toBe('test-1.0.0-add-version') + // All 3 entities have 'status' metadata + expect(p.affectedEntities).toBeGreaterThanOrEqual(3) + expect(p.sampleChanges.length).toBeGreaterThan(0) + expect(p.sampleChanges[0].after.version).toBe(1) + + // Verify no data was modified (dry-run) + const entity = await brain.get(id1) + expect(entity?.metadata?.version).toBeUndefined() + }) + }) + + it('should show correct sample before/after', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { state: 'draft' } }) + + const renameMigration: Migration = { + id: 'test-rename', + version: '1.0.0', + description: 'Rename state to status', + applies: 'nouns', + transform: (m) => { + if ('state' in m) { + const { state, ...rest } = m + return { ...rest, status: state } + } + return null + } + } + + await withMigrations([renameMigration], async () => { + const preview = await brain.migrate({ dryRun: true }) + const p = preview as any + expect(p.sampleChanges.length).toBeGreaterThanOrEqual(1) + + // Find the sample for our entity (it has the 'state' field) + const sample = p.sampleChanges.find((s: any) => s.before.state === 'draft') + expect(sample).toBeDefined() + expect(sample.after.status).toBe('draft') + expect(sample.after.state).toBeUndefined() + }) + }) + }) + + // ─── Apply migrations ───────────────────────────────────────── + + describe('Apply migrations', () => { + it('should transform noun metadata and return result', async () => { + const id = await brain.add({ + type: NounType.Concept, + data: { name: 'Alice' }, + metadata: { priority: 'low' } + }) + + const addFieldMigration: Migration = { + id: 'test-add-field', + version: '1.0.0', + description: 'Add migrated flag to entities with priority', + applies: 'nouns', + transform: (m) => { + if ('priority' in m && !('migrated' in m)) { + return { ...m, migrated: true } + } + return null + } + } + + await withMigrations([addFieldMigration], async () => { + const result = await brain.migrate() + const r = result as any + expect(r.migrationsApplied).toContain('test-add-field') + expect(r.entitiesModified).toBeGreaterThanOrEqual(1) + + // Verify data was actually transformed + const entity = await brain.get(id) + expect(entity?.metadata?.migrated).toBe(true) + expect(entity?.metadata?.priority).toBe('low') + }) + }) + + it('should skip entities that return null from transform', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'Has status' }, metadata: { status: 'active' } }) + await brain.add({ type: NounType.Concept, data: { name: 'No status' }, metadata: { priority: 'high' } }) + + const conditionalMigration: Migration = { + id: 'test-conditional', + version: '1.0.0', + description: 'Uppercase status field only when present', + applies: 'nouns', + transform: (m) => { + if (typeof m.status === 'string') { + return { ...m, status: (m.status as string).toUpperCase() } + } + return null + } + } + + await withMigrations([conditionalMigration], async () => { + const result = await brain.migrate() + const r = result as any + // Only 1 entity has 'status', the other has 'priority' + expect(r.entitiesModified).toBeGreaterThanOrEqual(1) + expect(r.entitiesProcessed).toBeGreaterThanOrEqual(2) + }) + }) + + it('should run multiple migrations in order', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { count: 1 } }) + + const migration1: Migration = { + id: 'test-double', + version: '1.0.0', + description: 'Double count', + applies: 'nouns', + transform: (m) => typeof m.count === 'number' ? { ...m, count: (m.count as number) * 2 } : null + } + + const migration2: Migration = { + id: 'test-add-ten', + version: '1.1.0', + description: 'Add 10 to count', + applies: 'nouns', + transform: (m) => typeof m.count === 'number' ? { ...m, count: (m.count as number) + 10 } : null + } + + await withMigrations([migration1, migration2], async () => { + const result = await brain.migrate() + const r = result as any + expect(r.migrationsApplied).toEqual(['test-double', 'test-add-ten']) + }) + }) + + it('should not re-run completed migrations', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { v: 1 } }) + + const migration: Migration = { + id: 'test-increment', + version: '1.0.0', + description: 'Increment v', + applies: 'nouns', + transform: (m) => typeof m.v === 'number' ? { ...m, v: (m.v as number) + 1 } : null + } + + await withMigrations([migration], async () => { + // Run once + const result1 = await brain.migrate() + expect((result1 as any).migrationsApplied).toHaveLength(1) + + // Run again — should be no-op + const result2 = await brain.migrate() + expect((result2 as any).migrationsApplied).toHaveLength(0) + expect((result2 as any).entitiesModified).toBe(0) + }) + }) + }) + + // ─── Backup branch tests ────────────────────────────────────── + + describe('Backup branches', () => { + it('should create a backup branch before migrating', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { x: 1 } }) + // Need a commit for fork to work + await brain.commit({ message: 'test', author: 'test' }) + + const migration: Migration = { + id: 'test-backup', + version: '2.0.0', + description: 'Add y field to entities with x', + applies: 'nouns', + transform: (m) => 'x' in m && !('y' in m) ? { ...m, y: 2 } : null + } + + await withMigrations([migration], async () => { + const result = await brain.migrate() + const r = result as any + expect(r.backupBranch).toBe('pre-migration-2.0.0') + + const branches = await brain.listBranches() + expect(branches).toContain('pre-migration-2.0.0') + }) + }) + + it('should tag backup branch with system:backup metadata', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { z: 1 } }) + await brain.commit({ message: 'test', author: 'test' }) + + const migration: Migration = { + id: 'test-tag', + version: '4.0.0', + description: 'Tag test', + applies: 'nouns', + transform: (m) => 'z' in m ? { ...m, tagged: true } : null + } + + await withMigrations([migration], async () => { + await brain.migrate() + + // Verify metadata tag on the backup branch ref + const refManager = (brain as any).storage.refManager + if (refManager) { + const ref = await refManager.getRef('pre-migration-4.0.0') + expect(ref).toBeDefined() + expect(ref?.metadata?.type).toBe('system:backup') + expect(ref?.metadata?.migrationVersion).toBe('4.0.0') + expect(ref?.metadata?.author).toBe('brainy-migration') + } + }) + }) + + it('should keep backup branch as a named restore point', async () => { + const id = await brain.add({ type: NounType.Concept, data: { name: 'Restore Test' }, metadata: { original: true } }) + await brain.commit({ message: 'before migration', author: 'test' }) + + const migration: Migration = { + id: 'test-restore', + version: '3.0.0', + description: 'Replace original with migrated', + applies: 'nouns', + transform: (m) => { + if (m.original === true) { + return { ...m, original: false, migrated: true } + } + return null + } + } + + await withMigrations([migration], async () => { + const result = await brain.migrate() + const r = result as any + + // Verify migration was applied + const migrated = await brain.get(id) + expect(migrated?.metadata?.migrated).toBe(true) + + // Verify backup branch exists as a named restore point + expect(r.backupBranch).toBe('pre-migration-3.0.0') + const branches = await brain.listBranches() + expect(branches).toContain('pre-migration-3.0.0') + + // Verify the backup ref points to the pre-migration commit + const refManager = (brain as any).storage.refManager + if (refManager) { + const mainRef = await refManager.getRef('main') + const backupRef = await refManager.getRef('pre-migration-3.0.0') + expect(backupRef).toBeDefined() + // Backup was forked before migration, so both share the same commit + // (the pre-migration commit). After migration, main's overlay has new data, + // but the commit pointer is unchanged. + expect(backupRef.commitHash).toBe(mainRef.commitHash) + } + }) + }) + }) + + // ─── Warning log tests ──────────────────────────────────────── + + describe('autoMigrate: false (default)', () => { + it('should log warning when pending migrations exist', async () => { + const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + const migration: Migration = { + id: 'test-warning', + version: '1.0.0', + description: 'Test warning', + applies: 'nouns', + transform: () => null + } + + await withMigrations([migration], async () => { + const warnBrain = new Brainy({ storage: { type: 'memory' } }) + await warnBrain.init() + + const migrationLogs = consoleSpy.mock.calls + .filter(call => typeof call[0] === 'string' && call[0].includes('pending migration')) + expect(migrationLogs.length).toBeGreaterThan(0) + + await warnBrain.close() + }) + + consoleSpy.mockRestore() + }) + }) + + // ─── autoMigrate: true inline ───────────────────────────────── + + describe('autoMigrate: true', () => { + it('should auto-migrate small datasets during init', async () => { + // Create a brain with data, close it + const setupBrain = new Brainy({ storage: { type: 'memory' }, silent: true }) + await setupBrain.init() + await setupBrain.add({ type: NounType.Concept, data: { name: 'AutoTest' }, metadata: { legacy: true } }) + + // We can't easily share memory storage between instances, + // so test the migrate() path directly instead + const migration: Migration = { + id: 'test-auto', + version: '1.0.0', + description: 'Auto migrate test', + applies: 'nouns', + transform: (m) => 'legacy' in m ? { ...m, legacy: false, upgraded: true } : null + } + + await withMigrations([migration], async () => { + const result = await setupBrain.migrate() + const r = result as any + expect(r.migrationsApplied).toContain('test-auto') + expect(r.entitiesModified).toBeGreaterThanOrEqual(1) + }) + + await setupBrain.close() + }) + }) + + // ─── Progress callback ──────────────────────────────────────── + + describe('Progress callback', () => { + it('should call onProgress during migration', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'A' }, metadata: { x: 1 } }) + await brain.add({ type: NounType.Concept, data: { name: 'B' }, metadata: { x: 2 } }) + + const migration: Migration = { + id: 'test-progress', + version: '1.0.0', + description: 'Add y to entities with x', + applies: 'nouns', + transform: (m) => 'x' in m ? { ...m, y: true } : null + } + + const progressCalls: any[] = [] + + await withMigrations([migration], async () => { + await brain.migrate({ + onProgress: (p) => progressCalls.push(p) + }) + }) + + expect(progressCalls.length).toBeGreaterThan(0) + expect(progressCalls[0].migrationId).toBe('test-progress') + expect(progressCalls[0].processed).toBeGreaterThan(0) + }) + }) + + // ─── Multi-tenant independence ───────────────────────────────── + + describe('Multi-tenant independence', () => { + it('should have independent migration state per instance', async () => { + const brain1 = new Brainy({ storage: { type: 'memory' }, silent: true }) + const brain2 = new Brainy({ storage: { type: 'memory' }, silent: true }) + await brain1.init() + await brain2.init() + + await brain1.add({ type: NounType.Concept, data: { name: 'User1 data' }, metadata: { v: 1 } }) + await brain2.add({ type: NounType.Concept, data: { name: 'User2 data' }, metadata: { v: 1 } }) + + const migration: Migration = { + id: 'test-tenant', + version: '1.0.0', + description: 'Increment v on entities that have it', + applies: 'nouns', + transform: (m) => typeof m.v === 'number' ? { ...m, v: (m.v as number) + 1 } : null + } + + await withMigrations([migration], async () => { + // Migrate only brain1 + const r1 = await brain1.migrate() + expect((r1 as any).entitiesModified).toBeGreaterThanOrEqual(1) + + // brain2 should still have pending migrations + const preview = await brain2.migrate({ dryRun: true }) + expect((preview as any).pendingMigrations).toHaveLength(1) + expect((preview as any).affectedEntities).toBeGreaterThanOrEqual(1) + }) + + await brain1.close() + await brain2.close() + }) + }) + + // ─── Verb migration ─────────────────────────────────────────── + + describe('Verb migration', () => { + it('should transform verb metadata when applies is verbs', async () => { + const id1 = await brain.add({ type: NounType.Concept, data: { name: 'A' } }) + const id2 = await brain.add({ type: NounType.Concept, data: { name: 'B' } }) + await brain.relate({ from: id1, to: id2, type: VerbType.RelatedTo, metadata: { strength: 'weak' } }) + + const migration: Migration = { + id: 'test-verb-migration', + version: '1.0.0', + description: 'Rename strength to intensity', + applies: 'verbs', + transform: (m) => { + if ('strength' in m) { + const { strength, ...rest } = m + return { ...rest, intensity: strength } + } + return null + } + } + + await withMigrations([migration], async () => { + const result = await brain.migrate() + expect((result as any).migrationsApplied).toContain('test-verb-migration') + // At least the one verb with 'strength' should be modified + expect((result as any).entitiesModified).toBeGreaterThanOrEqual(1) + }) + }) + }) + + // ─── Both nouns and verbs ───────────────────────────────────── + + describe('applies: both', () => { + it('should transform both nouns and verbs when applies is both', async () => { + const id1 = await brain.add({ type: NounType.Concept, data: { name: 'A' }, metadata: { tag: 'old' } }) + const id2 = await brain.add({ type: NounType.Concept, data: { name: 'B' }, metadata: { tag: 'old' } }) + await brain.relate({ from: id1, to: id2, type: VerbType.RelatedTo, metadata: { tag: 'old' } }) + + const migration: Migration = { + id: 'test-both', + version: '1.0.0', + description: 'Update tag from old to new', + applies: 'both', + transform: (m) => m.tag === 'old' ? { ...m, tag: 'new' } : null + } + + await withMigrations([migration], async () => { + const result = await brain.migrate() + const r = result as any + // At least 2 nouns + 1 verb should be modified + expect(r.entitiesModified).toBeGreaterThanOrEqual(3) + }) + }) + }) + + // ─── Multi-branch migration ───────────────────────────────── + + describe('Multi-branch migration', () => { + it('should migrate entities on all branches including feature branches', async () => { + // Setup: create entities on main, fork a branch, add entities on the branch + const mainId = await brain.add({ + type: NounType.Concept, + data: { name: 'Main Entity' }, + metadata: { legacy: true, source: 'main' } + }) + await brain.commit({ message: 'initial', author: 'test' }) + + // Fork a feature branch and add branch-local entity + const fork = await brain.fork('feature-x') + const branchId = await fork.add({ + type: NounType.Concept, + data: { name: 'Branch Entity' }, + metadata: { legacy: true, source: 'branch' } + }) + + // Define migration that transforms the 'legacy' field + const migration: Migration = { + id: 'test-multi-branch', + version: '1.0.0', + description: 'Mark legacy entities as upgraded', + applies: 'nouns', + transform: (m) => { + if (m.legacy === true) { + return { ...m, legacy: false, upgraded: true } + } + return null // Already migrated or not applicable + } + } + + await withMigrations([migration], async () => { + // Migrate from main — should reach all branches + const result = await brain.migrate() + const r = result as any + + expect(r.migrationsApplied).toContain('test-multi-branch') + // Main entity + branch-local entity should both be modified + expect(r.entitiesModified).toBeGreaterThanOrEqual(2) + + // Verify main entity was transformed + const mainEntity = await brain.get(mainId) + expect(mainEntity?.metadata?.upgraded).toBe(true) + expect(mainEntity?.metadata?.legacy).toBe(false) + }) + + await fork.close() + }) + + it('should skip system:backup branches during multi-branch migration', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { v: 1 } }) + await brain.commit({ message: 'test', author: 'test' }) + + // Create two migrations — first creates a backup branch, second should skip it + const migration1: Migration = { + id: 'test-skip-backup-1', + version: '5.0.0', + description: 'First migration', + applies: 'nouns', + transform: (m) => typeof m.v === 'number' && !('m1' in m) ? { ...m, m1: true } : null + } + + await withMigrations([migration1], async () => { + const result = await brain.migrate() + const r = result as any + expect(r.backupBranch).toBe('pre-migration-5.0.0') + + // Verify backup branch exists + const branches = await brain.listBranches() + expect(branches).toContain('pre-migration-5.0.0') + + // Migration should not have errored from trying to migrate the backup + expect(r.migrationsApplied).toContain('test-skip-backup-1') + }) + }) + }) + + // ─── MigrationRunner unit-level tests ───────────────────────── + + describe('MigrationRunner', () => { + it('should report no pending migrations when array is empty', async () => { + const runner = new MigrationRunner((brain as any).storage) + expect(await runner.hasPendingMigrations()).toBe(false) + }) + + it('should report pending migrations when array has entries', async () => { + const migration: Migration = { + id: 'runner-test', + version: '1.0.0', + description: 'Test', + applies: 'nouns', + transform: () => null + } + + await withMigrations([migration], async () => { + const runner = new MigrationRunner((brain as any).storage) + expect(await runner.hasPendingMigrations()).toBe(true) + expect(await runner.pendingCount()).toBe(1) + expect(runner.nextMigrationVersion()).toBe('1.0.0') + }) + }) + + it('should estimate time correctly', async () => { + const migration: Migration = { + id: 'estimate-test', + version: '1.0.0', + description: 'Test', + applies: 'nouns', + transform: () => null + } + + await withMigrations([migration], async () => { + const runner = new MigrationRunner((brain as any).storage) + const preview = await runner.preview() + // With 0 entities, estimatedTime should be '0ms' or '<1s' + expect(preview.estimatedTime).toBeDefined() + }) + }) + }) + + // ─── Error handling ────────────────────────────────────────────── + + describe('Error handling', () => { + it('should track transform errors without crashing the whole migration', async () => { + // Add 3 entities — one will trigger a throw + await brain.add({ type: NounType.Concept, data: { name: 'Good1' }, metadata: { value: 1 } }) + await brain.add({ type: NounType.Concept, data: { name: 'Bad' }, metadata: { value: 'crash-me' } }) + await brain.add({ type: NounType.Concept, data: { name: 'Good2' }, metadata: { value: 3 } }) + + const migration: Migration = { + id: 'test-error-handling', + version: '1.0.0', + description: 'Transform that throws on non-number values', + applies: 'nouns', + transform: (m) => { + if ('value' in m) { + if (typeof m.value !== 'number') { + throw new Error('value must be a number') + } + return { ...m, value: (m.value as number) * 10 } + } + return null + } + } + + await withMigrations([migration], async () => { + const result = await brain.migrate() + const r = result as any + + // Migration should still complete for the good entities + expect(r.migrationsApplied).toContain('test-error-handling') + expect(r.entitiesModified).toBeGreaterThanOrEqual(2) + + // Errors should be tracked + expect(r.errors.length).toBeGreaterThanOrEqual(1) + expect(r.errors[0].migrationId).toBe('test-error-handling') + expect(r.errors[0].error).toContain('value must be a number') + expect(r.errors[0].entityId).toBeDefined() + }) + }) + + it('should propagate errors from branch migrations into the result', async () => { + // Create entity on main, commit, fork a branch, add a branch-local entity that will fail + await brain.add({ type: NounType.Concept, data: { name: 'MainOk' }, metadata: { branchTest: 1 } }) + await brain.commit({ message: 'initial', author: 'test' }) + + const fork = await brain.fork('branch-with-error') + await fork.add({ type: NounType.Concept, data: { name: 'BranchBad' }, metadata: { branchTest: 'crash' } }) + + const migration: Migration = { + id: 'test-branch-error-prop', + version: '1.0.0', + description: 'Throws on non-number branchTest', + applies: 'nouns', + transform: (m) => { + if ('branchTest' in m) { + if (typeof m.branchTest !== 'number') { + throw new Error('branchTest must be a number') + } + return { ...m, branchTest: (m.branchTest as number) * 10 } + } + return null + } + } + + await withMigrations([migration], async () => { + const result = await brain.migrate() + const r = result as any + + // Main entity should have migrated successfully + expect(r.migrationsApplied).toContain('test-branch-error-prop') + expect(r.entitiesModified).toBeGreaterThanOrEqual(1) + + // Branch error should appear in the combined result + expect(r.errors.length).toBeGreaterThanOrEqual(1) + const branchError = r.errors.find((e: any) => e.error.includes('branchTest must be a number')) + expect(branchError).toBeDefined() + }) + + await fork.close() + }) + + it('should stop early when maxErrors is exceeded', async () => { + // Add many entities that will all fail + for (let i = 0; i < 5; i++) { + await brain.add({ type: NounType.Concept, data: { name: `Fail${i}` }, metadata: { boom: true } }) + } + + const migration: Migration = { + id: 'test-max-errors', + version: '1.0.0', + description: 'Always throws', + applies: 'nouns', + transform: (m) => { + if ('boom' in m) { + throw new Error('deliberate failure') + } + return null + } + } + + await withMigrations([migration], async () => { + const result = await brain.migrate({ maxErrors: 2 }) + const r = result as any + + // Should have stopped at 2 errors + expect(r.errors.length).toBe(2) + // Not all entities should have been processed + expect(r.entitiesProcessed).toBeLessThanOrEqual(5) + }) + }) + }) + + // ─── Validation ───────────────────────────────────────────────── + + describe('Validation (MigrationRunner.validateMigrations)', () => { + it('should accept empty migrations array', () => { + expect(() => MigrationRunner.validateMigrations([])).not.toThrow() + }) + + it('should accept valid migrations', () => { + const valid: Migration[] = [ + { id: 'v1', version: '1.0.0', description: 'Test', applies: 'nouns', transform: () => null }, + { id: 'v2', version: '1.1.0', description: 'Test 2', applies: 'verbs', transform: () => null } + ] + expect(() => MigrationRunner.validateMigrations(valid)).not.toThrow() + }) + + it('should reject duplicate migration IDs', () => { + const dupes: Migration[] = [ + { id: 'same-id', version: '1.0.0', description: 'First', applies: 'nouns', transform: () => null }, + { id: 'same-id', version: '1.1.0', description: 'Second', applies: 'nouns', transform: () => null } + ] + expect(() => MigrationRunner.validateMigrations(dupes)).toThrow(/Duplicate migration id.*same-id/) + }) + + it('should reject invalid applies value', () => { + const bad = [ + { id: 'bad-applies', version: '1.0.0', description: 'Bad', applies: 'widgets' as any, transform: () => null } + ] + expect(() => MigrationRunner.validateMigrations(bad)).toThrow(/invalid applies value.*widgets/) + }) + + it('should reject non-function transform', () => { + const bad = [ + { id: 'bad-transform', version: '1.0.0', description: 'Bad', applies: 'nouns' as const, transform: 'not a function' as any } + ] + expect(() => MigrationRunner.validateMigrations(bad)).toThrow(/non-function transform/) + }) + + it('should reject missing required fields', () => { + const noId = [ + { id: '', version: '1.0.0', description: 'Bad', applies: 'nouns' as const, transform: () => null } + ] + expect(() => MigrationRunner.validateMigrations(noId)).toThrow(/missing or invalid id/) + + const noVersion = [ + { id: 'ok', version: '', description: 'Bad', applies: 'nouns' as const, transform: () => null } + ] + expect(() => MigrationRunner.validateMigrations(noVersion)).toThrow(/missing or invalid version/) + + const noDescription = [ + { id: 'ok', version: '1.0.0', description: '', applies: 'nouns' as const, transform: () => null } + ] + expect(() => MigrationRunner.validateMigrations(noDescription)).toThrow(/missing or invalid description/) + }) + }) +})