From 8f93add7052fc0d57f2da9fed4219c220328638e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 15:22:47 -0700 Subject: [PATCH] =?UTF-8?q?feat(8.0)!:=20delete=20fork/branch/commit/histo?= =?UTF-8?q?ry/versions=20=E2=80=94=20superseded=20by=20the=20Db=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The COW version-control surface (fork, branches, checkout, commit, getHistory/streamHistory, asOfCommit, brain.versions) is gone, along with its subsystems: src/versioning/, the COW object store (CommitLog, CommitObject, RefManager, TreeObject), HistoricalStorageAdapter, and the TypeAwareHNSWIndex path it kept alive. The Db API (now/transact/asOf/ with/persist/restore) is the one versioning model in 8.0. Survivors and replacements: - BlobStorage survives (the VFS stores file content through it), relocated to src/storage/blobStorage.ts with binaryDataCodec.ts; its adapter interface is now BlobStoreAdapter, slimmed to the consumed surface (write/read/has/delete/getMetadata + MIME-aware compression policy). - brain.migrate() backup branches are replaced by persist-before-migrate: MigrateOptions.backupTo persists a hard-link snapshot of the current generation before any transform runs; MigrationResult.backupPath reports it, and brain.restore(path) brings it back wholesale. - CLI: cow.ts (fork/branch/checkout/history/migrate) is replaced by snapshot.ts — snapshot , restore , history (tx-log), generation. - New public read API: brain.transactionLog({limit}) exposes the reified tx-log (generation/timestamp/meta, newest first) that backs the CLI history command; TxLogEntry is exported. Tests: superseded suites deleted; fork/commit blocks excised from shared suites; BlobStorage tests relocated + reworked against the slimmed store; migration tests now prove the backupTo snapshot/restore round trip; new transactionLog coverage in db-mvcc. --- src/brainy.ts | 1089 ++--------------- src/cli/commands/cow.ts | 465 ------- src/cli/commands/snapshot.ts | 227 ++++ src/cli/index.ts | 58 +- src/db/generationStore.ts | 27 +- src/hnsw/hnswIndex.ts | 107 -- src/hnsw/typeAwareHNSWIndex.ts | 763 ------------ src/index.ts | 18 +- src/indexes/columnStore/ColumnManifest.ts | 5 +- src/migration/MigrationRunner.ts | 92 +- src/migration/types.ts | 12 +- src/storage/adapters/baseStorageAdapter.ts | 5 +- src/storage/adapters/fileSystemStorage.ts | 46 +- .../adapters/historicalStorageAdapter.ts | 571 --------- src/storage/adapters/memoryStorage.ts | 18 +- src/storage/baseStorage.ts | 600 +++------ src/storage/binaryDataCodec.ts | 77 ++ src/storage/blobStorage.ts | 498 ++++++++ src/storage/cow/BlobStorage.ts | 763 ------------ src/storage/cow/CommitLog.ts | 531 -------- src/storage/cow/CommitObject.ts | 563 --------- src/storage/cow/RefManager.ts | 547 --------- src/storage/cow/TreeObject.ts | 359 ------ src/storage/cow/binaryDataCodec.ts | 100 -- src/storage/cow/constants.ts | 59 - src/storage/storageFactory.ts | 24 +- src/transaction/operations/IndexOperations.ts | 3 +- .../operations/StorageOperations.ts | 3 +- src/types/brainy.types.ts | 1 - src/types/brainyInterface.ts | 9 +- src/versioning/VersionDiff.ts | 459 ------- src/versioning/VersionIndex.ts | 310 ----- src/versioning/VersionManager.ts | 667 ---------- src/versioning/VersionStorage.ts | 250 ---- src/versioning/VersioningAPI.ts | 520 -------- src/vfs/VirtualFileSystem.ts | 25 +- tests/helpers/TestWrappingAdapter.ts | 10 +- tests/integration/commit-ref-bug.test.ts | 104 -- .../integration/commit-state-capture.test.ts | 573 --------- tests/integration/cow-commit-storage.test.ts | 191 --- .../integration/cow-full-integration.test.ts | 574 --------- tests/integration/db-mvcc.test.ts | 38 + tests/integration/empty-tree-bug.test.ts | 144 --- tests/integration/fork-persistence.test.ts | 267 ---- .../history-ref-resolution-bug.test.ts | 142 --- tests/integration/hybrid-search-cow.test.ts | 150 --- tests/integration/hybrid-search-vfs.test.ts | 80 ++ .../initial-commit-null-parent.test.ts | 143 --- .../memory-enhancements-v5.11.0.test.ts | 153 --- .../metadata-only-comprehensive.test.ts | 34 - tests/integration/migration.test.ts | 226 +--- .../phase3TypeFirstQuery.integration.test.ts | 229 ---- .../storage-batch-operations.test.ts | 83 +- .../typeAwareHNSW.integration.test.ts | 527 -------- tests/integration/versioning.test.ts | 511 -------- .../integration/vfs-historical-reads.test.ts | 431 ------- tests/integration/vfs-versioning.test.ts | 307 ----- .../integration/cow-transactions.test.ts | 265 ---- tests/typeAwareHNSWIndex.test.ts | 467 ------- ...=> blobStorage.compression-policy.test.ts} | 30 +- ...lobStorage.test.ts => blobStorage.test.ts} | 392 ++---- tests/unit/versioning/VersionDiff.test.ts | 538 -------- tests/unit/versioning/VersionManager.test.ts | 710 ----------- tests/unit/versioning/VersionStorage.test.ts | 567 --------- 64 files changed, 1444 insertions(+), 16313 deletions(-) delete mode 100644 src/cli/commands/cow.ts create mode 100644 src/cli/commands/snapshot.ts delete mode 100644 src/hnsw/typeAwareHNSWIndex.ts delete mode 100644 src/storage/adapters/historicalStorageAdapter.ts create mode 100644 src/storage/binaryDataCodec.ts create mode 100644 src/storage/blobStorage.ts delete mode 100644 src/storage/cow/BlobStorage.ts delete mode 100644 src/storage/cow/CommitLog.ts delete mode 100644 src/storage/cow/CommitObject.ts delete mode 100644 src/storage/cow/RefManager.ts delete mode 100644 src/storage/cow/TreeObject.ts delete mode 100644 src/storage/cow/binaryDataCodec.ts delete mode 100644 src/storage/cow/constants.ts delete mode 100644 src/versioning/VersionDiff.ts delete mode 100644 src/versioning/VersionIndex.ts delete mode 100644 src/versioning/VersionManager.ts delete mode 100644 src/versioning/VersionStorage.ts delete mode 100644 src/versioning/VersioningAPI.ts delete mode 100644 tests/integration/commit-ref-bug.test.ts delete mode 100644 tests/integration/commit-state-capture.test.ts delete mode 100644 tests/integration/cow-commit-storage.test.ts delete mode 100644 tests/integration/cow-full-integration.test.ts delete mode 100644 tests/integration/empty-tree-bug.test.ts delete mode 100644 tests/integration/fork-persistence.test.ts delete mode 100644 tests/integration/history-ref-resolution-bug.test.ts delete mode 100644 tests/integration/hybrid-search-cow.test.ts create mode 100644 tests/integration/hybrid-search-vfs.test.ts delete mode 100644 tests/integration/initial-commit-null-parent.test.ts delete mode 100644 tests/integration/phase3TypeFirstQuery.integration.test.ts delete mode 100644 tests/integration/typeAwareHNSW.integration.test.ts delete mode 100644 tests/integration/versioning.test.ts delete mode 100644 tests/integration/vfs-historical-reads.test.ts delete mode 100644 tests/integration/vfs-versioning.test.ts delete mode 100644 tests/transaction/integration/cow-transactions.test.ts delete mode 100644 tests/typeAwareHNSWIndex.test.ts rename tests/unit/storage/{cow/BlobStorage.compression-policy.test.ts => blobStorage.compression-policy.test.ts} (88%) rename tests/unit/storage/{cow/BlobStorage.test.ts => blobStorage.test.ts} (53%) delete mode 100644 tests/unit/versioning/VersionDiff.test.ts delete mode 100644 tests/unit/versioning/VersionManager.test.ts delete mode 100644 tests/unit/versioning/VersionStorage.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 1e229e1a..dcc54db2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -24,13 +24,9 @@ import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js' import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js' import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js' import { VirtualFileSystem } from './vfs/VirtualFileSystem.js' -import { VersioningAPI } from './versioning/VersioningAPI.js' import { MetadataIndexManager } from './utils/metadataIndex.js' import { detectContentType, extractForHighlighting } from './utils/contentExtractor.js' import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js' -import { CommitBuilder } from './storage/cow/CommitObject.js' -import { BlobStorage } from './storage/cow/BlobStorage.js' -import { NULL_HASH } from './storage/cow/constants.js' import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel, prodLog } from './utils/logger.js' import { setGlobalCache } from './utils/unifiedCache.js' @@ -121,6 +117,7 @@ import type { CompactHistoryResult, TransactOptions, TransactReceipt, + TxLogEntry, TxOperation } from './db/types.js' import type { VersionedIndexProvider } from './plugin.js' @@ -275,7 +272,6 @@ export class Brainy implements BrainyInterface { private _nlp?: NaturalLanguageProcessor private _extractor?: NeuralEntityExtractor private _tripleIntelligence?: TripleIntelligenceSystem - private _versions?: VersioningAPI private _vfs?: VirtualFileSystem private _vfsInitialized = false // Track VFS init completion separately private _hub?: IntegrationHub // Integration Hub for external tools @@ -382,7 +378,7 @@ export class Brainy implements BrainyInterface { * and with any number of other readers on the same data directory. * - Throws `Cannot mutate a read-only Brainy instance` from every mutation * method (`add`, `addMany`, `update`, `delete`, `deleteMany`, `relate`, - * `unrelate`, `commit`, `branch`, `merge`, `deleteBranch`). + * `unrelate`, `transact`, `restore`). * - Reflects the state of the writer's last successful `flush()`. To force * the writer to flush before opening, call `requestFlush()` on a separate * handle first, or pass `--fresh` to the `brainy inspect` CLI. @@ -584,13 +580,6 @@ export class Brainy implements BrainyInterface { } } - // Enable COW immediately after storage init - // This ensures ALL data is stored in branch-scoped paths from the start - // Lightweight: just sets cowEnabled=true and currentBranch, no RefManager/BlobStorage yet - if (typeof (this.storage as any).enableCOWLightweight === 'function') { - (this.storage as any).enableCOWLightweight((this.config.storage as any)?.branch || 'main') - } - // 8.0 generational MVCC: open the record layer BEFORE any index is // created or loaded. Crash recovery may rewrite canonical entity files // (restoring before-images of an uncommitted transaction), and every @@ -769,13 +758,10 @@ export class Brainy implements BrainyInterface { Brainy.shutdownHooksRegisteredGlobally = true } - // Initialize COW (BlobStorage) before VFS - // VFS now requires BlobStorage for unified file storage - if (typeof (this.storage as any).initializeCOW === 'function') { - await (this.storage as any).initializeCOW({ - branch: (this.config.storage as any)?.branch || 'main', - enableCompression: true - }) + // Initialize the content-addressed blob store before VFS — the VFS + // stores all file content through it. + if (typeof (this.storage as any).initializeBlobStorage === 'function') { + await (this.storage as any).initializeBlobStorage() } // Log provider summary after all wiring is complete @@ -4336,20 +4322,15 @@ export class Brainy implements BrainyInterface { this._nlp = undefined this._tripleIntelligence = undefined - // Re-initialize COW (BlobStorage) after storage.clear() - // storage.clear() sets blobStorage=undefined for FileSystem/cloud adapters - // VFS depends on blobStorage being available (unified blob storage for all files) - // Must be done BEFORE VFS reinitialization - if (typeof (this.storage as any).initializeCOW === 'function') { - await (this.storage as any).initializeCOW({ - branch: (this.config.storage as any)?.branch || 'main', - enableCompression: true - }) + // Re-initialize the content-addressed blob store after storage.clear() + // (clear() drops the BlobStorage instance). The VFS stores all file + // content through it, so this must happen BEFORE VFS reinitialization. + if (typeof (this.storage as any).initializeBlobStorage === 'function') { + await (this.storage as any).initializeBlobStorage() } // Reset VFS state - root entity was deleted by storage.clear() // Bug: VFS instance remained in memory pointing to deleted root entity - // Following checkout() pattern exactly (see lines 2907-2914) if (this._vfs) { // Clear PathResolver caches (including UnifiedCache VFS entries) if ((this._vfs as any).pathResolver?.invalidateAllCaches) { @@ -4365,615 +4346,30 @@ export class Brainy implements BrainyInterface { } } - // ============= COW (COPY-ON-WRITE) API ============= - - /** - * Fork the brain (instant clone via Snowflake-style COW) - * - * Creates a shallow copy in <100ms using copy-on-write (COW) technology. - * Fork shares storage and HNSW data structures with parent, copying only - * when modified (lazy deep copy). - * - * **How It Works**: - * 1. HNSW Index: Shallow copy via `enableCOW()` (~10ms for 1M+ nodes) - * 2. Metadata Index: Fast rebuild from shared storage (<100ms) - * 3. Graph Index: Fast rebuild from shared storage (<500ms) - * - * **Performance**: - * - Fork time: <100ms @ 10K entities (MEASURED) - * - Memory overhead: 10-20% (shared HNSW nodes) - * - Storage overhead: 10-20% (shared blobs) - * - * **Write Isolation**: Changes in fork don't affect parent, and vice versa. - * - * @param branch - Optional branch name (auto-generated if not provided) - * @param options - Optional fork metadata (author, message) - * @returns New Brainy instance (forked, fully independent) - * - * @example - * ```typescript - * const brain = new Brainy() - * await brain.init() - * - * // Add data to parent - * await brain.add({ type: 'user', data: { name: 'Alice' } }) - * - * // Fork instantly (<100ms) - * const experiment = await brain.fork('test-migration') - * - * // Make changes safely in fork - * await experiment.add({ type: 'user', data: { name: 'Bob' } }) - * - * // Original untouched - * console.log((await brain.find({})).length) // 1 (Alice) - * console.log((await experiment.find({})).length) // 2 (Alice + Bob) - * ``` - * - */ - async fork(branch?: string, options?: { - author?: string - message?: string - metadata?: Record - }): Promise> { - this.assertWritable('fork') - await this.ensureInitialized() - - const branchName = branch || `fork-${Date.now()}` - - // Lazy COW initialization - enable automatically on first fork() - // This is zero-config and transparent to users - if (!('refManager' in this.storage) || !(this.storage as any).refManager) { - // Storage supports COW but isn't initialized yet - initialize now - if (typeof (this.storage as any).initializeCOW === 'function') { - await (this.storage as any).initializeCOW({ - branch: (this.config.storage as any)?.branch || 'main', - enableCompression: true - }) - } else { - // Storage adapter doesn't support COW at all - throw new Error( - 'Fork requires COW-enabled storage. ' + - 'This storage adapter does not support branching. ' + - 'Please use compatible storage adapters.' - ) - } - } - - const refManager = (this.storage as any).refManager - const currentBranch = (this.storage as any).currentBranch || 'main' - - // Step 1: Ensure initial commit exists (required for fork) - const currentRef = await refManager.getRef(currentBranch) - if (!currentRef) { - // Auto-create initial commit if none exists - await this.commit({ - message: `Initial commit on ${currentBranch}`, - author: options?.author || 'Brainy', - metadata: { timestamp: Date.now() } - }) - if (!this.config.silent) { - console.log(`📝 Auto-created initial commit on ${currentBranch} (required for fork)`) - } - } - - // Step 2: Copy storage ref (COW layer - instant!) - await refManager.copyRef(currentBranch, branchName) - - // CRITICAL FIX: Verify branch was actually created to prevent silent failures. - // Without this check, fork() could complete successfully but the branch wouldn't - // exist on disk, causing subsequent checkout() calls to fail with a - // "Branch does not exist" error. - const verifyBranch = await refManager.getRef(branchName) - if (!verifyBranch) { - throw new Error( - `Fork failed: Branch '${branchName}' was not created. ` + - `This indicates a storage adapter configuration issue. Please report this bug.` - ) - } - - // Step 3: Create new Brainy instance pointing to fork branch - const forkConfig = { - ...this.config, - storage: { - ...(this.config.storage || { type: 'memory' as any }), - branch: branchName - } - } - - const clone = new Brainy(forkConfig) - - // Step 3: Clone storage with separate currentBranch - // Share RefManager/BlobStorage/CommitLog but maintain separate branch context - clone.storage = Object.create(this.storage) - clone.storage.currentBranch = branchName - // isInitialized inherited from prototype - - // Create HNSW index (uses plugin factory when available) - clone.index = this.createIndex() - - // Enable COW - if ('enableCOW' in clone.index && typeof clone.index.enableCOW === 'function') { - (clone.index as any).enableCOW(this.index) - } - - // Fast rebuild for small indexes from COW storage (Metadata/Graph are fast) - const cloneMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') - clone.metadataIndex = cloneMetadataFactory - ? cloneMetadataFactory(clone.storage) - : new MetadataIndexManager(clone.storage) - await clone.metadataIndex.init() - - // GraphAdjacencyIndex SINGLETON pattern for fork() - // Object.create() causes prototype inheritance, so clone.storage.graphIndex - // would point to parent's graphIndex. We must break this inheritance and - // create a fresh instance for the clone's branch. - ;(clone.storage as any).graphIndex = undefined - ;(clone.storage as any).graphIndexPromise = undefined - const cloneGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') - if (cloneGraphFactory) { - clone.graphIndex = cloneGraphFactory(clone.storage) - clone.storage.setGraphIndex(clone.graphIndex) - } else { - clone.graphIndex = await (clone.storage as any).getGraphIndex() - } - // getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex) - - // 8.0 u64 contract: the clone has its own metadata index (and idMapper), - // so thread the clone's resolver into its graph index + storage. - clone.wireGraphIdResolver() - - // Mark as initialized - clone.initialized = true - clone.dimensions = this.dimensions - - return clone - } - - /** - * List all branches/forks - * @returns Array of branch names - * - * @example - * ```typescript - * const branches = await brain.listBranches() - * console.log(branches) // ['main', 'experiment', 'backup'] - * ``` - */ - async listBranches(): Promise { - await this.ensureInitialized() - - if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage') - } - - const refManager = (this.storage as any).refManager - const refs = await refManager.listRefs() - - // Filter to branches only (exclude tags) - return refs - .filter((ref: any) => ref.name.startsWith('refs/heads/')) - .map((ref: any) => ref.name.replace('refs/heads/', '')) - } - - /** - * Get current branch name - * @returns Current branch name - * - * @example - * ```typescript - * const current = await brain.getCurrentBranch() - * console.log(current) // 'main' - * ``` - */ - async getCurrentBranch(): Promise { - await this.ensureInitialized() - - if (!('currentBranch' in this.storage)) { - return 'main' // Default branch - } - - return (this.storage as any).currentBranch || 'main' - } - - /** - * Switch to a different branch - * @param branch - Branch name to switch to - * - * @example - * ```typescript - * await brain.checkout('experiment') - * console.log(await brain.getCurrentBranch()) // 'experiment' - * ``` - */ - async checkout(branch: string): Promise { - await this.ensureInitialized() - - if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage') - } - - // Verify branch exists - const branches = await this.listBranches() - if (!branches.includes(branch)) { - throw new Error(`Branch '${branch}' does not exist`) - } - - // Update storage currentBranch - (this.storage as any).currentBranch = branch - - // Fix: Reload indexes from new branch WITHOUT recreating storage - // Previous implementation called init() which recreated storage, losing currentBranch - this.index = this.createIndex() - const checkoutMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') - this.metadataIndex = checkoutMetadataFactory - ? checkoutMetadataFactory(this.storage) - : new MetadataIndexManager(this.storage) - await this.metadataIndex.init() - - // GraphAdjacencyIndex SINGLETON pattern for checkout() - // Invalidate the old graphIndex (it has data from the old branch) - // and get a fresh instance for the new branch - ;(this.storage as any).invalidateGraphIndex() - const checkoutGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') - if (checkoutGraphFactory) { - this.graphIndex = checkoutGraphFactory(this.storage) - this.storage.setGraphIndex(this.graphIndex) - } else { - this.graphIndex = await (this.storage as any).getGraphIndex() - } - - // 8.0 u64 contract: the branch switch recreated the metadata index (and - // idMapper) plus the graph index — re-thread the shared resolver. - this.wireGraphIdResolver() - - // Reset lazy loading state when switching branches - // Indexes contain data from previous branch, must rebuild for new branch - this.lazyRebuildCompleted = false - - // Rebuild indexes from new branch data (force=true to override disableAutoRebuild) - await this.rebuildIndexesIfNeeded(true) - - // Clear VFS caches before recreating VFS for new branch - // UnifiedCache is global, so old branch's VFS path cache entries would persist - if (this._vfs) { - // Clear old PathResolver's caches including UnifiedCache entries - if ((this._vfs as any).pathResolver?.invalidateAllCaches) { - (this._vfs as any).pathResolver.invalidateAllCaches() - } - // Recreate VFS for new branch - this._vfs = new VirtualFileSystem(this) - await this._vfs.init() - } - } - - /** - * Create a commit with current state - * - * Flushes all buffered data to disk, captures a full state snapshot - * (entities + relationships) into a content-addressed tree, then creates - * an immutable commit object pointing to that tree. - * - * By default, every commit captures a complete snapshot that can be - * restored via time-travel. Pass `captureState: false` for a lightweight - * metadata-only commit (tree hash will be NULL_HASH). - * - * @param options - Commit options - * @param options.message - Human-readable commit message - * @param options.author - Author identifier (email, username, or 'system') - * @param options.metadata - Arbitrary key-value metadata stored with the commit - * @param options.captureState - Capture entity snapshots for time-travel (default: true) - * @returns Commit hash (64-char hex SHA-256) - * - * @example - * ```typescript - * await brain.add({ noun: 'user', data: { name: 'Alice' } }) - * const commitHash = await brain.commit({ - * message: 'Add Alice user', - * author: 'dev@example.com' - * }) - * ``` - */ - async commit(options?: { - message?: string - author?: string - metadata?: Record - captureState?: boolean - }): Promise { - this.assertWritable('commit') - await this.ensureInitialized() - - if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) { - throw new Error('Commit requires COW-enabled storage') - } - - // Flush all buffered data to disk before creating the snapshot. - // Without this, deferred HNSW nodes, pending count batches, and - // unflushed index state could be missing from the commit tree. - await this.flush() - - const refManager = (this.storage as any).refManager - const blobStorage = (this.storage as any).blobStorage - const currentBranch = await this.getCurrentBranch() - - // Get current HEAD commit (parent) - const currentCommitHash = await refManager.resolveRef(currentBranch) - - // Get current state statistics - const entityCount = await this.getNounCount() - const relationshipCount = await this.getVerbCount() - - // Import NULL_HASH constant - const { NULL_HASH } = await import('./storage/cow/constants.js') - - // Capture entity state to tree (default: true). - // Only skip when caller explicitly passes captureState: false - // for a lightweight metadata-only commit. - const shouldCapture = options?.captureState !== false - let treeHash = NULL_HASH - if (shouldCapture) { - treeHash = await this.captureStateToTree() - } - - // Build commit object using builder pattern - const builder = CommitBuilder.create(blobStorage) - .tree(treeHash) - .message(options?.message || 'Snapshot commit') - .author(options?.author || 'unknown') - .timestamp(Date.now()) - .entityCount(entityCount) - .relationshipCount(relationshipCount) - - // Set parent if this is not the first commit - if (currentCommitHash) { - builder.parent(currentCommitHash) - } - - // Add custom metadata - if (options?.metadata) { - Object.entries(options.metadata).forEach(([key, value]) => { - builder.meta(key, value) - }) - } - - // Build and persist commit (returns hash directly) - const commitHash = await builder.build() - - // Update branch ref to point to new commit - await refManager.setRef(currentBranch, commitHash, { - author: options?.author || 'unknown', - message: options?.message || 'Snapshot commit' - }) - - return commitHash - } - - /** - * Capture current entity and relationship state to tree object - * Called by commit() by default for full state snapshots - * - * Serializes ALL entities + relationships to blobs and builds a tree. - * BlobStorage automatically deduplicates unchanged data. - * - * Handles all storage adapters including sharded/distributed setups. - * Storage adapter is responsible for aggregating data from all shards. - * - * Performance: O(n+m) where n = entity count, m = relationship count - * - 1K entities + 500 relations: ~150ms - * - 100K entities + 50K relations: ~1.5s - * - 1M entities + 500K relations: ~8s - * - * @returns Tree hash containing all entities and relationships - * @private - */ - private async captureStateToTree(): Promise { - const blobStorage = (this.storage as any).blobStorage as BlobStorage - const { TreeBuilder } = await import('./storage/cow/TreeObject.js') - - // Query ALL entities (excludeVFS: false to capture VFS files too - default behavior) - const entityResults = await this.find({ excludeVFS: false }) - - // Query ALL relationships with pagination (handles sharding via storage adapter) - const allRelations: any[] = [] - let hasMore = true - let offset = 0 - const limit = 1000 // Fetch in batches - - while (hasMore) { - const relationResults = await this.storage.getVerbs({ - pagination: { offset, limit } - }) - - allRelations.push(...relationResults.items) - hasMore = relationResults.hasMore - offset += limit - } - - // Return NULL_HASH for empty workspace (no data to capture) - if (entityResults.length === 0 && allRelations.length === 0) { - console.log(`[captureStateToTree] Empty workspace - returning NULL_HASH`) - return NULL_HASH - } - - console.log(`[captureStateToTree] Capturing ${entityResults.length} entities + ${allRelations.length} relationships to tree`) - - // Build tree with TreeBuilder - const builder = TreeBuilder.create(blobStorage) - - // Serialize each entity to blob and add to tree - for (const result of entityResults) { - const entity = result.entity - - // Serialize entity to JSON - const entityJson = JSON.stringify(entity) - const entityBlob = Buffer.from(entityJson) - - // Write to BlobStorage (auto-deduplicates by content hash) - const blobHash = await blobStorage.write(entityBlob, { - type: 'blob', - compression: 'auto' // Compress large entities (>10KB) - }) - - // Add to tree: entities/entity-id → blob-hash - await builder.addBlob(`entities/${entity.id}`, blobHash, entityBlob.length) - } - - // Serialize each relationship to blob and add to tree - for (const relation of allRelations) { - // Serialize relationship to JSON - const relationJson = JSON.stringify(relation) - const relationBlob = Buffer.from(relationJson) - - // Write to BlobStorage (auto-deduplicates by content hash) - const blobHash = await blobStorage.write(relationBlob, { - type: 'blob', - compression: 'auto' - }) - - // Add to tree: relations/sourceId-targetId-verb → blob-hash - // Use sourceId-targetId-verb as unique identifier for each relationship - const relationKey = `relations/${relation.sourceId}-${relation.targetId}-${relation.verb}` - await builder.addBlob(relationKey, blobHash, relationBlob.length) - } - - // Build and persist tree, return hash - const treeHash = await builder.build() - - console.log(`[captureStateToTree] Tree created: ${treeHash.slice(0, 8)} with ${entityResults.length} entities + ${allRelations.length} relationships`) - - return treeHash - } - - /** - * Create a read-only snapshot of the workspace at a specific commit - * (pre-8.0 commit-based time travel — the generational `asOf()` is the - * 8.0 replacement; this method is retired with the COW surface). - * - * Time-travel API for historical queries. Returns a new Brainy instance that: - * - Contains all entities and relationships from that commit - * - Has all indexes rebuilt (HNSW, MetadataIndex, GraphAdjacencyIndex) - * - Supports full triple intelligence (vector + graph + metadata queries) - * - Is read-only (throws errors on add/update/delete/commit/relate) - * - Must be closed when done to free memory - * - * Performance characteristics: - * - Initial snapshot: O(n+m) where n = entities, m = relationships - * - Subsequent queries: Same as normal Brainy (uses rebuilt indexes) - * - Memory overhead: Snapshot has separate in-memory indexes - * - * Use case: rendering a file tree (or any indexed view) at a historical commit - * - * @param commitId - Commit hash to snapshot from - * @returns Read-only Brainy instance with historical state - * - * @example - * ```typescript - * // Create snapshot at specific commit - * const snapshot = await brain.asOfCommit(commitId) - * - * // Query historical state (full triple intelligence works!) - * const files = await snapshot.find({ - * query: 'AI research', - * where: { 'metadata.vfsType': 'file' } - * }) - * - * // Get historical relationships - * const related = await snapshot.getRelated(entityId, { depth: 2 }) - * - * // MUST close when done to free memory - * await snapshot.close() - * ``` - */ - async asOfCommit(commitId: string, options?: { - cacheSize?: number // LRU cache size (default: 10000) - }): Promise { - await this.ensureInitialized() - - // Lazy-loading historical adapter with bounded memory - // No eager loading of entire commit state! - const { HistoricalStorageAdapter } = await import('./storage/adapters/historicalStorageAdapter.js') - const { BaseStorage } = await import('./storage/baseStorage.js') - - // Create lazy-loading historical storage adapter - const historicalStorage = new HistoricalStorageAdapter({ - underlyingStorage: this.storage as BaseStorage, - commitId, - cacheSize: options?.cacheSize || 10000, - branch: await this.getCurrentBranch() || 'main' - }) - - // Initialize historical adapter (loads commit metadata, NOT entities) - await historicalStorage.init() - - console.log(`[asOf] Historical storage adapter created for commit ${commitId.slice(0, 8)}`) - - // Create Brainy instance wrapping historical storage - // All queries will lazy-load from historical state on-demand - // mode: 'reader' wires ReaderMode so every mutation throws. - const snapshotBrain = new Brainy({ - ...this.config, - // Use the historical adapter directly (no need for separate storage type) - storage: historicalStorage as any, - mode: 'reader' - }) - - // Initialize the snapshot (creates indexes, but they'll be populated lazily) - await snapshotBrain.init() - - // Track which commit this snapshot reflects - ;(snapshotBrain as any).snapshotCommitId = commitId - - console.log(`[asOf] Snapshot ready (lazy-loading, cache size: ${options?.cacheSize || 10000})`) - - return snapshotBrain - } - - - /** - * Delete a branch/fork - * @param branch - Branch name to delete - * - * @example - * ```typescript - * await brain.deleteBranch('old-experiment') - * ``` - */ - async deleteBranch(branch: string): Promise { - this.assertWritable('deleteBranch') - await this.ensureInitialized() - - if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage') - } - - const currentBranch = await this.getCurrentBranch() - if (branch === currentBranch) { - throw new Error('Cannot delete current branch') - } - - const refManager = (this.storage as any).refManager - await refManager.deleteRef(branch) - } - // ─── Migration API ─────────────────────────────────────────────── /** * Run pending data migrations, or preview what would change. * - * @param options - Pass { dryRun: true } to preview without writing + * @param options - Pass `{ dryRun: true }` to preview without writing; + * pass `{ backupTo: path }` to persist a pre-migration snapshot. * @returns Migration result (or preview if dryRun) * + * Safety: with `backupTo`, a hard-link snapshot of the current generation + * is persisted BEFORE any transform runs (the Db API's `persist()`), and + * `brain.restore(path, { confirm: true })` brings it back wholesale. + * Transforms are also idempotent (they return `null` when already + * applied), so re-running migrations is safe. + * * @example * ```typescript * // Preview what would change * const preview = await brain.migrate({ dryRun: true }) * console.log(preview.affectedEntities) * - * // Apply migrations (auto-forks a backup branch first) - * const result = await brain.migrate() - * console.log(result.backupBranch) // 'pre-migration-7.17.0' - * - * // Rollback if needed - * await brain.checkout('pre-migration-7.17.0') + * // Apply migrations with a restore point + * const result = await brain.migrate({ backupTo: '/backups/pre-migration' }) + * console.log(result.entitiesModified, result.backupPath) * ``` */ async migrate(options?: MigrateOptions): Promise { @@ -5024,274 +4420,35 @@ export class Brainy implements BrainyInterface { } /** - * Internal: fork backup, run migrations on current branch + all other branches. - * - * Branch strategy: - * - getNouns() uses listObjectsInBranch() which only returns branch-local entities - * - So migrating main only transforms main's entities; branch-local entities are untouched - * - After main, we iterate all other user branches and run the same transforms - * - Lightweight: just switches storage.currentBranch (no full checkout/index rebuild) - * - Transforms are idempotent (return null when already applied), so this is safe + * Internal: run pending migrations and rebuild the metadata index when + * anything changed. With `backupTo`, a hard-link snapshot of the current + * generation is persisted before any transform runs — the Db-API + * replacement for the pre-8.0 automatic backup branches. Transforms are + * idempotent (return `null` when already applied), so re-runs are safe. */ private async migrateInternal(runner: MigrationRunner, options?: MigrateOptions): Promise { - // 0. Clean up old migration backup branches (by metadata tag, not name) - await runner.cleanupOldBackups() - - // 1. Fork backup branch (uses existing COW — instant) - const version = runner.nextMigrationVersion() - const backupName = `pre-migration-${version}` - let backupCreated = false - - try { - await this.fork(backupName) - backupCreated = true - - // Tag the backup branch with metadata so we can identify it later - if (this.storage.refManager) { - await this.storage.refManager.updateRefMetadata(backupName, { - type: 'system:backup', - migrationVersion: version, - author: 'brainy-migration' - }) + let backupPath: string | null = null + if (options?.backupTo) { + const pin = this.now() + try { + await pin.persist(options.backupTo) + backupPath = options.backupTo + } finally { + await pin.release() } - } catch { - // Fork may fail if COW not initialized (e.g., memory storage with no commits) - // Continue without backup — migrations are still safe (user can re-import) } - // 2. Run migrations on current branch const runResult = await runner.run({ onProgress: options?.onProgress, maxErrors: options?.maxErrors }) - // 3. Migrate all other user branches (branch-local entities only) - // getNouns() uses listObjectsInBranch() which only lists branch-overlay files, - // so each branch iteration only touches entities written directly to that branch. - const branchResult = await this.migrateOtherBranches( - runResult.migrationsApplied, - backupCreated ? backupName : null, - options - ) - - // 4. Rebuild MetadataIndex if any entities were modified (on the current branch) - const totalModified = runResult.entitiesModified + branchResult.entitiesModified - if (totalModified > 0) { + // Rebuild MetadataIndex if any entities were modified + if (runResult.entitiesModified > 0) { await this.metadataIndex.rebuild() } - // 5. Clear deferred runner + // Clear deferred runner this._pendingMigrationRunner = undefined - return { - backupBranch: backupCreated ? backupName : null, - migrationsApplied: runResult.migrationsApplied, - entitiesProcessed: runResult.entitiesProcessed + branchResult.entitiesProcessed, - entitiesModified: totalModified, - errors: [...(runResult.errors || []), ...branchResult.errors] - } - } - - /** - * Migrate branch-local entities on all non-current branches. - * - * Why this is needed: getNouns() lists entities from the branch overlay only - * (via listObjectsInBranch), not inherited entities. So migrating main doesn't - * touch entities written directly to feature branches. We iterate each branch - * and run the same transforms — they're idempotent, so already-migrated - * inherited entities return null and are skipped. - * - * Lightweight: switches storage.currentBranch directly instead of full checkout() - * (no index rebuild needed — migration uses storage-level methods only). - */ - private async migrateOtherBranches( - migrationIds: string[], - skipBranch: string | null, - options?: MigrateOptions - ): Promise<{ entitiesProcessed: number; entitiesModified: number; errors: import('./migration/types.js').MigrationError[] }> { - const empty = { entitiesProcessed: 0, entitiesModified: 0, errors: [] as import('./migration/types.js').MigrationError[] } - if (migrationIds.length === 0) return empty - - // Only if COW branching is available - const refManager = this.storage.refManager - if (!refManager) return empty - - const currentBranch = this.storage.currentBranch || 'main' - let totalProcessed = 0 - let totalModified = 0 - const allErrors: import('./migration/types.js').MigrationError[] = [] - - try { - const { MIGRATIONS } = await import('./migration/migrations.js') - const migrationsToRun = MIGRATIONS.filter(m => migrationIds.includes(m.id)) - if (migrationsToRun.length === 0) return empty - - const refs = await refManager.listRefs() - const branches = refs - .filter(ref => ref.name.startsWith('refs/heads/')) - .map(ref => ({ - name: ref.name.replace('refs/heads/', ''), - metadata: ref.metadata - })) - - for (const branch of branches) { - // Skip current branch (already migrated above) - if (branch.name === currentBranch) continue - // Skip the backup branch we just created - if (branch.name === skipBranch) continue - // Skip system backup branches - if (branch.metadata?.type === 'system:backup') continue - - // Switch storage to this branch (lightweight — no index rebuild) - this.storage.currentBranch = branch.name - - // Run transforms — idempotent, so inherited entities return null and are skipped. - // Uses runMigrations() which bypasses the state check (state on main says "completed" - // but branch-local entities haven't been touched yet). - const branchRunner = new MigrationRunner(this.storage) - const result = await branchRunner.runMigrations(migrationsToRun, { - onProgress: options?.onProgress, - maxErrors: options?.maxErrors - }) - - totalProcessed += result.entitiesProcessed - totalModified += result.entitiesModified - if (result.errors.length > 0) { - allErrors.push(...result.errors) - } - } - } finally { - // Always restore original branch - this.storage.currentBranch = currentBranch - } - - return { entitiesProcessed: totalProcessed, entitiesModified: totalModified, errors: allErrors } - } - - /** - * Get commit history for current branch - * @param options - History options (limit, offset, author) - * @returns Array of commits - * - * @example - * ```typescript - * const history = await brain.getHistory({ limit: 10 }) - * history.forEach(commit => { - * console.log(`${commit.hash}: ${commit.message}`) - * }) - * ``` - */ - async getHistory(options?: { - limit?: number - offset?: number - author?: string - }): Promise - }>> { - await this.ensureInitialized() - - if (!('commitLog' in this.storage) || !('refManager' in this.storage)) { - throw new Error('History requires COW-enabled storage') - } - - const commitLog = (this.storage as any).commitLog - const currentBranch = await this.getCurrentBranch() - - // Get commit history for current branch - const commits = await commitLog.getHistory(currentBranch, { - maxCount: options?.limit || 10 - }) - - // Map to expected format (compute hash for each commit) - return commits.map((commit: any) => ({ - hash: (this.storage as any).blobStorage.constructor.hash( - Buffer.from(JSON.stringify(commit)) - ), - message: commit.message, - author: commit.author, - timestamp: commit.timestamp, - metadata: commit.metadata - })) - } - - /** - * Stream commit history (memory-efficient) - * - * Use this for large commit histories (1000s of snapshots) where memory - * efficiency is critical. Yields commits one at a time without accumulating - * them in memory. - * - * For small histories (< 100 commits), use getHistory() for simpler API. - * - * @param options - History options - * @param options.limit - Maximum number of commits to stream - * @param options.since - Only include commits after this timestamp - * @param options.until - Only include commits before this timestamp - * @param options.author - Filter by author name - * - * @yields Commit metadata in reverse chronological order (newest first) - * - * @example - * ```typescript - * // Stream all commits without memory accumulation - * for await (const commit of brain.streamHistory({ limit: 10000 })) { - * console.log(`${commit.timestamp}: ${commit.message}`) - * } - * - * // Stream with filtering - * for await (const commit of brain.streamHistory({ - * author: 'alice', - * since: Date.now() - 86400000 // Last 24 hours - * })) { - * // Process commit - * } - * ``` - */ - async *streamHistory(options?: { - limit?: number - since?: number - until?: number - author?: string - }): AsyncIterableIterator<{ - hash: string - message: string - author: string - timestamp: number - metadata?: Record - }> { - await this.ensureInitialized() - - if (!('commitLog' in this.storage) || !('refManager' in this.storage)) { - throw new Error('History streaming requires COW-enabled storage') - } - - const commitLog = (this.storage as any).commitLog - const currentBranch = await this.getCurrentBranch() - const blobStorage = (this.storage as any).blobStorage - - // Stream commits from CommitLog - for await (const commit of commitLog.streamHistory(currentBranch, { - maxCount: options?.limit, - since: options?.since, - until: options?.until - })) { - // Filter by author if specified - if (options?.author && commit.author !== options.author) { - continue - } - - // Map to expected format (compute hash for commit) - yield { - hash: blobStorage.constructor.hash( - Buffer.from(JSON.stringify(commit)) - ), - message: commit.message, - author: commit.author, - timestamp: commit.timestamp, - metadata: commit.metadata - } - } + return { backupPath, ...runResult } } // ============= 8.0 DB API — GENERATIONAL MVCC ============= @@ -5324,6 +4481,31 @@ export class Brainy implements BrainyInterface { return this.generationStore.generation() } + /** + * @description Read the reified transaction log — one entry per committed + * `transact()` batch, carrying the committed generation, the commit + * timestamp, and the `meta` the transaction was submitted with. Entries + * are returned newest first. Single-operation writes + * (`add`/`update`/`delete`/`relate`/…) advance the generation counter but + * do not append log entries (the documented 8.0 history granularity), and + * `compactHistory()` never rewrites the log — entries may reference + * generations whose record-sets were already reclaimed. + * + * @param options - `limit` caps the number of entries returned (newest + * first; default: all). + * @returns Committed transaction entries, newest first. + * @example + * await brain.transact([{ op: 'update', id, metadata: { v: 2 } }], { meta: { author: 'sync-job' } }) + * const [latest] = await brain.transactionLog({ limit: 1 }) + * latest.meta // { author: 'sync-job' } + */ + async transactionLog(options?: { limit?: number }): Promise { + await this.ensureInitialized() + const entries = await this.generationStore.txLog() + entries.reverse() + return options?.limit !== undefined ? entries.slice(0, options.limit) : entries + } + /** * @description Pin the CURRENT generation and return an immutable `Db` * view of it — O(1), no I/O. The returned view keeps reading exactly this @@ -5455,8 +4637,7 @@ export class Brainy implements BrainyInterface { * (equivalent to {@link Brainy.load}) with the FULL query surface, * including vector search. * - * Release the returned `Db` when done. (For pre-8.0 commit-hash time - * travel, see `asOfCommit()` — retired with the COW surface.) + * Release the returned `Db` when done. * * @param target - Generation number, `Date`, or snapshot directory path. * @returns A `Db` pinned at the resolved state. @@ -6525,38 +5706,6 @@ export class Brainy implements BrainyInterface { return this._neural } - /** - * Versioning API - Entity version control - * - * Provides entity-level versioning with: - * - save() - Create version of entity - * - restore() - Restore entity to specific version - * - list() - List all versions of entity - * - compare() - Deep diff between versions - * - prune() - Remove old versions (retention policies) - * - * @example - * ```typescript - * // Save current state - * const version = await brain.versions.save('user-123', { tag: 'v1.0' }) - * - * // List versions - * const versions = await brain.versions.list('user-123') - * - * // Restore to previous version - * await brain.versions.restore('user-123', 5) - * - * // Compare versions - * const diff = await brain.versions.compare('user-123', 2, 5) - * ``` - */ - get versions(): VersioningAPI { - if (!this._versions) { - this._versions = new VersioningAPI(this as any) - } - return this._versions - } - /** * Natural Language Processing API */ @@ -6841,18 +5990,6 @@ export class Brainy implements BrainyInterface { * const content = await brain.vfs.readFile('/docs/readme.md') * ``` * - * @example With fork (COW isolation) - * ```typescript - * await brain.init() - * await brain.vfs.writeFile('/config.json', '{"v": 1}') - * - * const fork = await brain.fork('experiment') - * // Fork inherits parent's files - * const config = await fork.vfs.readFile('/config.json') - * // Fork modifications are isolated - * await fork.vfs.writeFile('/test.txt', 'Fork only') - * ``` - * * **Pattern:** The VFS instance is cached, so multiple calls to brain.vfs * return the same instance. This ensures import and user code share state. * @@ -9393,95 +8530,11 @@ export class Brainy implements BrainyInterface { return clusters } - // ============= INTERNAL VERSIONING API ============= - // These methods are used by the versioning system (brain.versions.*) - // They expose internal storage and index operations needed for entity versioning - - /** - * Search entities by metadata filters (internal API) - * Used by versioning system for querying version metadata - * - * @param filters - Metadata filter object (same format as `where` in find()) - * @returns Array of matching entities - * @internal - */ - async searchByMetadata(filters: Record): Promise { - await this.ensureInitialized() - - const ids = await this.metadataIndex.getIdsForFilter(filters) - if (ids.length === 0) return [] - - const results: any[] = [] - const entitiesMap = await this.batchGet(ids) - for (const id of ids) { - const entity = entitiesMap.get(id) - if (entity) { - results.push(entity) - } - } - return results - } - - /** - * Get raw noun metadata (internal API) - * Used by versioning system for reading entity state - * - * @param id - Entity ID - * @returns Noun metadata or null if not found - * @internal - */ - async getNounMetadata(id: string): Promise { - await this.ensureInitialized() - return this.storage.getNounMetadata(id) - } - - /** - * Save raw noun metadata (internal API) - * Used by versioning system for storing version index entries - * - * @param id - Entity ID - * @param data - Metadata to save - * @internal - */ - async saveNounMetadata(id: string, data: any): Promise { - await this.ensureInitialized() - await this.storage.saveNounMetadata(id, data) - } - - /** - * Delete noun metadata (internal API) - * Used by versioning system for removing version index entries - * - * @param id - Entity ID - * @internal - */ - async deleteNounMetadata(id: string): Promise { - await this.ensureInitialized() - await this.storage.deleteNounMetadata(id) - } - - /** - * Current branch name (internal API) - * Used by versioning system for branch-aware operations - * @internal - */ - get currentBranch(): string { - return (this.storage as any).currentBranch || 'main' - } - - /** - * Reference manager for COW commits (internal API) - * Used by versioning system for commit operations - * @internal - */ - get refManager(): any { - return (this.storage as any).refManager - } - /** * Storage adapter (internal API) - * Used by versioning system for direct storage access - * Returns BaseStorage which has saveMetadata/getMetadata for key-value storage + * Direct access to the underlying BaseStorage for diagnostics and tests + * (e.g. reading the transaction log). Not part of the supported public + * surface. * @internal */ get storageAdapter(): BaseStorage { @@ -10398,7 +9451,7 @@ export class Brainy implements BrainyInterface { /** * Create the vector index, using the plugin-provided engine when available. - * Shared by init(), fork(), checkout(), and clear() to avoid duplication. + * Shared by init() and clear() to avoid duplication. * * Selection order (first match wins): * 1. `'vector'` provider if registered — the canonical 8.0 provider key. @@ -10582,8 +9635,8 @@ export class Brainy implements BrainyInterface { /** * @description Wire the shared UUID ↔ int resolver (the metadata index's * idMapper) into the JS graph index and the storage layer (8.0 u64 - * contract). Called after the graph index is resolved on init, fork, and - * checkout so every consumer of the BigInt boundary shares one int + * contract). Called after the graph index is resolved on init so every + * consumer of the BigInt boundary shares one int * universe. Native graph-index providers carry their own mapper and don't * expose `setEntityIdMapper` — the storage-level wiring still applies so * its verb read paths can convert UUIDs before provider calls. diff --git a/src/cli/commands/cow.ts b/src/cli/commands/cow.ts deleted file mode 100644 index ff197032..00000000 --- a/src/cli/commands/cow.ts +++ /dev/null @@ -1,465 +0,0 @@ -/** - * COW CLI Commands - Copy-on-Write Operations - * - * Fork, branch, merge, and migration operations for instant cloning - */ - -import chalk from 'chalk' -import ora from 'ora' -import inquirer from 'inquirer' -import { Brainy } from '../../brainy.js' -import { readFileSync, writeFileSync, existsSync } from 'node:fs' -import { resolve } from 'node:path' - -interface CoreOptions { - verbose?: boolean - json?: boolean - pretty?: boolean -} - -interface ForkOptions extends CoreOptions { - name?: string - message?: string - author?: string -} - -interface MergeOptions extends CoreOptions { - force?: boolean - strategy?: 'last-write-wins' | 'custom' -} - -interface MigrateOptions extends CoreOptions { - from?: string - to?: string - backup?: boolean - dryRun?: boolean -} - -let brainyInstance: Brainy | null = null - -const getBrainy = (): Brainy => { - if (!brainyInstance) { - brainyInstance = new Brainy() - } - return brainyInstance -} - -const formatOutput = (data: any, options: CoreOptions): void => { - if (options.json) { - console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data)) - } -} - -export const cowCommands = { - /** - * Fork the current brain (instant clone) - */ - async fork(name: string | undefined, options: ForkOptions) { - let spinner: any = null - try { - const brain = getBrainy() - await brain.init() - - // Interactive mode if no name provided - if (!name) { - const answers = await inquirer.prompt([ - { - type: 'input', - name: 'branchName', - message: 'Enter fork/branch name:', - default: `fork-${Date.now()}`, - validate: (input: string) => - input.trim().length > 0 || 'Branch name cannot be empty' - }, - { - type: 'input', - name: 'message', - message: 'Commit message (optional):', - default: 'Fork from main' - } - ]) - name = answers.branchName - options.message = answers.message - } - - spinner = ora(`Forking brain to ${chalk.cyan(name)}...`).start() - - const startTime = Date.now() - const fork = await brain.fork(name) - const elapsed = Date.now() - startTime - - spinner.succeed( - `Fork created: ${chalk.green(name)} ${chalk.dim(`(${elapsed}ms)`)}` - ) - - // Show stats - const stats = await fork.getStats() - console.log(` -${chalk.cyan('Fork Statistics:')} - ${chalk.dim('Entities:')} ${stats.entities.total || 0} - ${chalk.dim('Relationships:')} ${stats.relationships.totalRelationships || 0} - ${chalk.dim('Time:')} ${elapsed}ms - ${chalk.dim('Storage overhead:')} ~10-20% - `.trim()) - - if (options.json) { - formatOutput({ - branch: name, - time: elapsed, - stats - }, options) - } - - } catch (error: any) { - if (spinner) spinner.fail('Fork failed') - console.error(chalk.red('Error:'), error.message) - if (options.verbose) console.error(error) - process.exit(1) - } - }, - - /** - * List all branches/forks - */ - async branchList(options: CoreOptions) { - try { - const brain = getBrainy() - await brain.init() - - const branches = await brain.listBranches() - const currentBranch = await brain.getCurrentBranch() - - console.log(chalk.cyan('\nBranches:')) - - for (const branch of branches) { - const isCurrent = branch === currentBranch - const marker = isCurrent ? chalk.green('*') : ' ' - const name = isCurrent ? chalk.green(branch) : branch - - // Branch-info enrichment (last-commit timestamp, etc.) requires a - // refManager accessor that brainy hasn't surfaced on the public - // storage adapter yet. CLI shows the bare branch name for now. - console.log(` ${marker} ${name}`) - } - - console.log() - - if (options.json) { - formatOutput({ - branches, - currentBranch - }, options) - } - - } catch (error: any) { - console.error(chalk.red('Error:'), error.message) - if (options.verbose) console.error(error) - process.exit(1) - } - }, - - /** - * Switch to a different branch - */ - async checkout(branch: string | undefined, options: CoreOptions) { - let spinner: any = null - try { - const brain = getBrainy() - await brain.init() - - // Interactive mode if no branch provided - if (!branch) { - const branches = await brain.listBranches() - const currentBranch = await brain.getCurrentBranch() - - const { selected } = await inquirer.prompt([ - { - type: 'list', - name: 'selected', - message: 'Select branch:', - choices: branches.map(b => ({ - name: b === currentBranch ? `${b} (current)` : b, - value: b - })) - } - ]) - branch = selected - } - - const currentBranch = await brain.getCurrentBranch() - - if (branch === currentBranch) { - console.log(chalk.yellow(`Already on branch '${branch}'`)) - return - } - - spinner = ora(`Switching to ${chalk.cyan(branch)}...`).start() - - await brain.checkout(branch!) - - spinner.succeed(`Switched to branch ${chalk.green(branch)}`) - - if (options.json) { - formatOutput({ branch }, options) - } - - } catch (error: any) { - if (spinner) spinner.fail('Checkout failed') - console.error(chalk.red('Error:'), error.message) - if (options.verbose) console.error(error) - process.exit(1) - } - }, - - /** - * Delete a branch/fork - */ - async branchDelete(branch: string | undefined, options: CoreOptions & { force?: boolean }) { - try { - const brain = getBrainy() - await brain.init() - - // Interactive mode if no branch provided - if (!branch) { - const branches = await brain.listBranches() - const currentBranch = await brain.getCurrentBranch() - - const { selected } = await inquirer.prompt([ - { - type: 'list', - name: 'selected', - message: 'Select branch to delete:', - choices: branches - .filter(b => b !== currentBranch) // Can't delete current - .map(b => ({ name: b, value: b })) - } - ]) - branch = selected - } - - // Confirm deletion - if (!options.force) { - const { confirm } = await inquirer.prompt([ - { - type: 'confirm', - name: 'confirm', - message: `Delete branch '${branch}'? This cannot be undone.`, - default: false - } - ]) - - if (!confirm) { - console.log(chalk.yellow('Deletion cancelled')) - return - } - } - - const spinner = ora(`Deleting branch ${chalk.red(branch)}...`).start() - - await brain.deleteBranch(branch!) - - spinner.succeed(`Deleted branch ${chalk.red(branch)}`) - - if (options.json) { - formatOutput({ deleted: branch }, options) - } - - } catch (error: any) { - console.error(chalk.red('Error:'), error.message) - if (options.verbose) console.error(error) - process.exit(1) - } - }, - - /** - * View commit history - */ - async history(options: CoreOptions & { limit?: string }) { - try { - const brain = getBrainy() - await brain.init() - - const limit = options.limit ? parseInt(options.limit) : 10 - - const history = await brain.getHistory({ limit }) - - console.log(chalk.cyan(`\nCommit History (last ${limit}):\n`)) - - for (const commit of history) { - const date = new Date(commit.timestamp) - const age = formatAge(Date.now() - commit.timestamp) - - console.log( - `${chalk.yellow(commit.hash.substring(0, 8))} ` + - `${chalk.dim(commit.message)} ` + - `${chalk.dim(`by ${commit.author} (${age} ago)`)}` - ) - } - - console.log() - - if (options.json) { - formatOutput(history, options) - } - - } catch (error: any) { - console.error(chalk.red('Error:'), error.message) - if (options.verbose) console.error(error) - process.exit(1) - } - }, - - /** - * Migrate storage format (one-time) - */ - async migrate(options: MigrateOptions) { - let spinner: any = null - try { - // Interactive mode if paths not provided - let fromPath = options.from - let toPath = options.to - - if (!fromPath || !toPath) { - const answers = await inquirer.prompt([ - { - type: 'input', - name: 'from', - message: 'Old Brainy data path (v4.x):', - default: './brainy-data', - when: !fromPath - }, - { - type: 'input', - name: 'to', - message: 'New Brainy data path (v5.0.0):', - default: './brainy-data-v5', - when: !toPath - }, - { - type: 'confirm', - name: 'backup', - message: 'Create backup before migration?', - default: true, - when: options.backup === undefined - } - ]) - - fromPath = fromPath || answers.from - toPath = toPath || answers.to - options.backup = options.backup ?? answers.backup - } - - // Verify old data exists - if (!existsSync(resolve(fromPath!))) { - throw new Error(`Old data path not found: ${fromPath}`) - } - - // Create backup if requested - if (options.backup) { - const backupPath = `${fromPath}.backup-${Date.now()}` - spinner = ora(`Creating backup: ${backupPath}...`).start() - const fs = await import('node:fs/promises') - await fs.cp(fromPath, backupPath, { recursive: true, force: false }) - spinner.succeed(`Backup created: ${chalk.green(backupPath)}`) - } - - if (options.dryRun) { - console.log(chalk.yellow('\n[DRY RUN] Migration plan:')) - console.log(` From: ${fromPath}`) - console.log(` To: ${toPath}`) - console.log(` Backup: ${options.backup ? 'Yes' : 'No'}`) - console.log() - return - } - - spinner = ora('Migrating to v5.0.0 COW format...').start() - - // Load old brain (v4.x) - const oldBrain = new Brainy({ - storage: { - type: 'filesystem', - options: { path: fromPath } - } - }) - - await oldBrain.init() - - // Create new brain - const newBrain = new Brainy({ - storage: { - type: 'filesystem', - options: { path: toPath } - } - }) - - await newBrain.init() - - // Migrate all entities - const entities = await oldBrain.find({}) - let migrated = 0 - - spinner.text = `Migrating entities (0/${entities.length})...` - - for (const result of entities) { - // Add entity with proper params - await newBrain.add({ - type: result.entity.type as any, - data: result.entity.data - }) - migrated++ - - if (migrated % 100 === 0) { - spinner.text = `Migrating entities (${migrated}/${entities.length})...` - } - } - - // Create initial commit (will be available after COW integration) - // await newBrain.commit({ - // message: `Migrated from v4.x (${entities.length} entities)`, - // author: 'migration-tool' - // }) - - spinner.succeed(`Migration complete: ${chalk.green(migrated)} entities`) - - console.log(` -${chalk.cyan('Migration Summary:')} - ${chalk.dim('Old path:')} ${fromPath} - ${chalk.dim('New path:')} ${toPath} - ${chalk.dim('Entities:')} ${migrated} - ${chalk.dim('Format:')} v5.0.0 COW - `.trim()) - - if (options.json) { - formatOutput({ - from: fromPath, - to: toPath, - migrated - }, options) - } - - await oldBrain.close() - await newBrain.close() - - } catch (error: any) { - if (spinner) spinner.fail('Migration failed') - console.error(chalk.red('Error:'), error.message) - if (options.verbose) console.error(error) - process.exit(1) - } - } -} - -/** - * Format timestamp age - */ -function formatAge(ms: number): string { - const seconds = Math.floor(ms / 1000) - const minutes = Math.floor(seconds / 60) - const hours = Math.floor(minutes / 60) - const days = Math.floor(hours / 24) - - if (days > 0) return `${days}d` - if (hours > 0) return `${hours}h` - if (minutes > 0) return `${minutes}m` - return `${seconds}s` -} diff --git a/src/cli/commands/snapshot.ts b/src/cli/commands/snapshot.ts new file mode 100644 index 00000000..d30e4d23 --- /dev/null +++ b/src/cli/commands/snapshot.ts @@ -0,0 +1,227 @@ +/** + * @module cli/commands/snapshot + * @description Snapshot & time-travel CLI commands over the 8.0 Db API. + * + * - `snapshot ` — persist the current generation as a self-contained, + * hard-link snapshot directory (`brain.now().persist(path)`). + * - `restore ` — replace the store's ENTIRE state from a snapshot + * (`brain.restore(path, { confirm: true })`, after an interactive confirm). + * - `history` — the reified transaction log: one line per committed + * `transact()` batch (generation, commit time, metadata). + * - `generation` — the store's current generation watermark. + */ + +import chalk from 'chalk' +import ora from 'ora' +import inquirer from 'inquirer' +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { Brainy } from '../../brainy.js' + +/** + * @description Shared CLI flags every snapshot command accepts (`--verbose`, + * `--json`, `--pretty`), mirroring the other command modules. + */ +interface SnapshotCliOptions { + verbose?: boolean + json?: boolean + pretty?: boolean +} + +let brainyInstance: Brainy | null = null + +const getBrainy = (): Brainy => { + if (!brainyInstance) { + brainyInstance = new Brainy() + } + return brainyInstance +} + +const formatOutput = (data: unknown, options: SnapshotCliOptions): void => { + if (options.json) { + console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data)) + } +} + +/** + * @description The `snapshot`/`restore`/`history`/`generation` command + * handlers registered in `src/cli/index.ts`. + */ +export const snapshotCommands = { + /** + * @description Persist the current generation as a self-contained snapshot + * directory. Instant on same-device targets (hard links); the result opens + * with `Brainy.load(path)` or restores wholesale via `brainy restore`. + * @param path - Target directory (created; must be empty or absent). + * @param options - Shared CLI flags. + */ + async snapshot(path: string, options: SnapshotCliOptions) { + let spinner: ReturnType | null = null + try { + const brain = getBrainy() + await brain.init() + + const target = resolve(path) + spinner = ora(`Persisting snapshot to ${chalk.cyan(target)}...`).start() + + const startTime = Date.now() + const db = brain.now() + try { + await db.persist(target) + } finally { + await db.release() + } + const elapsed = Date.now() - startTime + + spinner.succeed( + `Snapshot persisted: ${chalk.green(target)} ${chalk.dim(`(generation ${db.generation}, ${elapsed}ms)`)}` + ) + console.log(` +${chalk.cyan('Snapshot:')} + ${chalk.dim('Path:')} ${target} + ${chalk.dim('Generation:')} ${db.generation} + ${chalk.dim('Open with:')} Brainy.load('${target}') + ${chalk.dim('Restore with:')} brainy restore ${target} + `.trim()) + + formatOutput({ path: target, generation: db.generation, time: elapsed }, options) + } catch (error: any) { + if (spinner) spinner.fail('Snapshot failed') + console.error(chalk.red('Error:'), error.message) + if (options.verbose) console.error(error) + process.exit(1) + } + }, + + /** + * @description Replace the store's ENTIRE current state from a snapshot + * directory. Destructive — asks for interactive confirmation first + * (skipped with `--force`). + * @param path - Snapshot directory produced by `brainy snapshot `. + * @param options - Shared CLI flags plus `--force`. + */ + async restore(path: string, options: SnapshotCliOptions & { force?: boolean }) { + let spinner: ReturnType | null = null + try { + const target = resolve(path) + if (!existsSync(target)) { + throw new Error(`Snapshot directory not found: ${target}`) + } + + if (!options.force) { + const { confirm } = await inquirer.prompt([ + { + type: 'confirm', + name: 'confirm', + message: `Replace the store's ENTIRE current state with the snapshot at '${target}'? This cannot be undone.`, + default: false + } + ]) + if (!confirm) { + console.log(chalk.yellow('Restore cancelled')) + return + } + } + + const brain = getBrainy() + await brain.init() + + spinner = ora(`Restoring from ${chalk.cyan(target)}...`).start() + const startTime = Date.now() + await brain.restore(target, { confirm: true }) + const elapsed = Date.now() - startTime + + spinner.succeed( + `Restored from ${chalk.green(target)} ${chalk.dim(`(now at generation ${brain.generation()}, ${elapsed}ms)`)}` + ) + + formatOutput({ path: target, generation: brain.generation(), time: elapsed }, options) + } catch (error: any) { + if (spinner) spinner.fail('Restore failed') + console.error(chalk.red('Error:'), error.message) + if (options.verbose) console.error(error) + process.exit(1) + } + }, + + /** + * @description Show the reified transaction log, newest first: one entry + * per committed `transact()` batch with its generation, commit time, and + * the `meta` the transaction was submitted with. + * @param options - Shared CLI flags plus `--limit`. + */ + async history(options: SnapshotCliOptions & { limit?: string }) { + try { + const brain = getBrainy() + await brain.init() + + const limit = options.limit ? parseInt(options.limit, 10) : 10 + const entries = await brain.transactionLog({ limit }) + + if (entries.length === 0) { + console.log(chalk.yellow('\nNo committed transactions yet (the log records transact() batches).\n')) + formatOutput([], options) + return + } + + console.log(chalk.cyan(`\nTransaction Log (last ${entries.length}):\n`)) + for (const entry of entries) { + const age = formatAge(Date.now() - entry.timestamp) + const meta = entry.meta ? ` ${chalk.dim(JSON.stringify(entry.meta))}` : '' + console.log( + `${chalk.yellow(`g${entry.generation}`)} ` + + `${chalk.dim(new Date(entry.timestamp).toISOString())} ` + + `${chalk.dim(`(${age} ago)`)}` + + meta + ) + } + console.log() + + formatOutput(entries, options) + } catch (error: any) { + console.error(chalk.red('Error:'), error.message) + if (options.verbose) console.error(error) + process.exit(1) + } + }, + + /** + * @description Print the store's current generation watermark (bumped once + * per committed `transact()` batch and once per single-operation write + * batch). + * @param options - Shared CLI flags. + */ + async generation(options: SnapshotCliOptions) { + try { + const brain = getBrainy() + await brain.init() + + const generation = brain.generation() + console.log(`${chalk.cyan('Generation:')} ${chalk.green(String(generation))}`) + + formatOutput({ generation }, options) + } catch (error: any) { + console.error(chalk.red('Error:'), error.message) + if (options.verbose) console.error(error) + process.exit(1) + } + } +} + +/** + * @description Format a millisecond duration as a compact age string + * (`3d` / `5h` / `12m` / `42s`). + * @param ms - Elapsed milliseconds. + * @returns The compact age string. + */ +function formatAge(ms: number): string { + const seconds = Math.floor(ms / 1000) + const minutes = Math.floor(seconds / 60) + const hours = Math.floor(minutes / 60) + const days = Math.floor(hours / 24) + + if (days > 0) return `${days}d` + if (hours > 0) return `${hours}h` + if (minutes > 0) return `${minutes}m` + return `${seconds}s` +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 912199ef..a9f10262 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -17,7 +17,7 @@ import { storageCommands } from './commands/storage.js' import { nlpCommands } from './commands/nlp.js' import { insightsCommands } from './commands/insights.js' import { importCommands } from './commands/import.js' -import { cowCommands } from './commands/cow.js' +import { snapshotCommands } from './commands/snapshot.js' import { inspectCommands } from './commands/inspect.js' import { readFileSync } from 'fs' import { fileURLToPath } from 'url' @@ -760,58 +760,30 @@ program .option('--iterations ', 'Number of iterations', '100') .action(utilityCommands.benchmark) -// ===== COW Commands - Instant Fork & Branching ===== +// ===== Snapshot & Time-Travel Commands - Db API ===== program - .command('fork [name]') - .description('🚀 Fork the brain (instant clone in 1-2 seconds)') - .option('--message ', 'Commit message') - .option('--author ', 'Author name') - .action(cowCommands.fork) + .command('snapshot ') + .description('📸 Persist the current generation as a self-contained snapshot (instant hard links)') + .action(snapshotCommands.snapshot) program - .command('branch') - .description('🌿 Branch management') - .addCommand( - new Command('list') - .alias('ls') - .description('List all branches/forks') - .action((options) => { - cowCommands.branchList(options) - }) - ) - .addCommand( - new Command('delete') - .alias('rm') - .argument('[name]', 'Branch name to delete') - .description('Delete a branch/fork') - .option('-f, --force', 'Skip confirmation') - .action((name, options) => { - cowCommands.branchDelete(name, options) - }) - ) - -program - .command('checkout [branch]') - .alias('co') - .description('Switch to a different branch') - .action(cowCommands.checkout) + .command('restore ') + .description('Replace the ENTIRE store state from a snapshot (asks for confirmation)') + .option('-f, --force', 'Skip confirmation') + .action(snapshotCommands.restore) program .command('history') .alias('log') - .description('Show commit history') - .option('-l, --limit ', 'Number of commits to show', '10') - .action(cowCommands.history) + .description('Show the transaction log (one entry per committed transact() batch)') + .option('-l, --limit ', 'Number of entries to show', '10') + .action(snapshotCommands.history) program - .command('migrate') - .description('🔄 Migrate from v4.x to v5.0.0 (one-time)') - .option('--from ', 'Old Brainy data path (v4.x)') - .option('--to ', 'New Brainy data path (v5.0.0)') - .option('--backup', 'Create backup before migration') - .option('--dry-run', 'Show migration plan without executing') - .action(cowCommands.migrate) + .command('generation') + .description('Show the store\'s current generation watermark') + .action(snapshotCommands.generation) // ===== Interactive Mode ===== diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index e56d43c7..78dfff13 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -621,8 +621,27 @@ export class GenerationStore { async resolveTimestamp( timestampMs: number ): Promise<{ generation: number; entry: TxLogEntry | null }> { - const lines = await this.storage.readTxLogLines() let best: TxLogEntry | null = null + for (const entry of await this.txLog()) { + if (entry.timestamp <= timestampMs) { + if (best === null || entry.generation > best.generation) best = entry + } + } + return { generation: best?.generation ?? 0, entry: best } + } + + /** + * @description Read all committed tx-log entries, oldest first. Torn + * trailing lines from a crashed append are tolerated and skipped, and + * entries beyond the committed watermark are excluded — the tx-log is + * advisory metadata; the manifest rename is the source of commit truth. + * Compaction never rewrites the log, so entries may reference generations + * whose record-sets were already reclaimed. + * @returns Every committed {@link TxLogEntry}, in commit order. + */ + async txLog(): Promise { + const lines = await this.storage.readTxLogLines() + const entries: TxLogEntry[] = [] for (const line of lines) { let entry: TxLogEntry try { @@ -630,11 +649,9 @@ export class GenerationStore { } catch { continue // tolerate a torn trailing line from a crashed append } - if (entry.timestamp <= timestampMs && entry.generation <= this.committed) { - if (best === null || entry.generation > best.generation) best = entry - } + if (entry.generation <= this.committed) entries.push(entry) } - return { generation: best?.generation ?? 0, entry: best } + return entries } /** diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index f4d76876..bf09ee8c 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -59,10 +59,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { // save, so the migration converges under live traffic with no big-bang. private connectionsCodec: ConnectionsCodec | null = null - // COW (Copy-on-Write) support - private cowEnabled: boolean = false - private cowModifiedNodes: Set = new Set() - private cowParent: JsHnswVectorIndex | null = null // Deferred HNSW persistence for cloud storage performance // In deferred mode, HNSW connections are only persisted on flush/close @@ -301,99 +297,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return this.persistMode } - /** - * Enable COW (Copy-on-Write) mode - Instant fork via shallow copy - * - * Snowflake-style instant fork: O(1) shallow copy of Maps, lazy deep copy on write. - * - * @param parent - Parent HNSW index to copy from - * - * Performance: - * - Fork time: <10ms for 1M+ nodes (just copies Map references) - * - Memory: Shared reads, only modified nodes duplicated (~10-20% overhead) - * - Reads: Same speed as parent (shared data structures) - * - * @example - * ```typescript - * const parent = new JsHnswVectorIndex(config) - * // ... parent has 1M nodes ... - * - * const fork = new JsHnswVectorIndex(config) - * fork.enableCOW(parent) // <10ms - instant! - * - * // Reads share data - * await fork.search(query) // Fast, uses parent's data - * - * // Writes trigger COW - * await fork.addItem(newItem) // Deep copies only modified nodes - * ``` - */ - public enableCOW(parent: JsHnswVectorIndex): void { - this.cowEnabled = true - this.cowParent = parent - - // Shallow copy Maps - O(1) per Map, just copies references - // All nodes/connections are shared until first write - this.nouns = new Map(parent.nouns) - this.highLevelNodes = new Map() - for (const [level, nodeSet] of parent.highLevelNodes.entries()) { - this.highLevelNodes.set(level, new Set(nodeSet)) - } - - // Copy scalar values - this.entryPointId = parent.entryPointId - this.maxLevel = parent.maxLevel - this.dimension = parent.dimension - - // Share cache (COW at cache level) - this.unifiedCache = parent.unifiedCache - - // Share config and distance function - this.config = parent.config - this.distanceFunction = parent.distanceFunction - this.useParallelization = parent.useParallelization - - prodLog.info(`HNSW COW enabled: ${parent.nouns.size} nodes shallow copied`) - } - - /** - * Ensure node is copied before modification (lazy COW) - * - * Deep copies a node only when first modified. Subsequent modifications - * use the already-copied node. - * - * @param nodeId - Node ID to ensure is copied - * @private - */ - private ensureCOW(nodeId: string): void { - if (!this.cowEnabled) return - if (this.cowModifiedNodes.has(nodeId)) return // Already copied - - const original = this.nouns.get(nodeId) - if (!original) return - - // Deep copy connections Map (separate Map + Sets for each level) - const connectionsCopy = new Map>() - for (const [level, ids] of original.connections.entries()) { - connectionsCopy.set(level, new Set(ids)) - } - - // Deep copy node - const nodeCopy: HNSWNoun = { - id: original.id, - vector: [...original.vector], // Deep copy vector array - connections: connectionsCopy, - level: original.level, - // Copy SQ8 quantized data if present - quantizedVector: original.quantizedVector ? new Uint8Array(original.quantizedVector) : undefined, - codebookMin: original.codebookMin, - codebookMax: original.codebookMax - } - - this.nouns.set(nodeId, nodeCopy) - this.cowModifiedNodes.add(nodeId) - } - /** * Calculate distances between a query vector and multiple vectors in parallel * This is used to optimize performance for search operations @@ -602,8 +505,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { continue } - // COW: Ensure neighbor is copied before modification - this.ensureCOW(neighborId) noun.connections.get(level)!.add(neighborId) @@ -965,16 +866,12 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return false } - // COW: Ensure node is copied before modification - this.ensureCOW(id) const noun = this.nouns.get(id)! // Remove connections to this noun from all neighbors for (const [level, connections] of noun.connections.entries()) { for (const neighborId of connections) { - // COW: Ensure neighbor is copied before modification - this.ensureCOW(neighborId) const neighbor = this.nouns.get(neighborId) if (!neighbor) { @@ -994,8 +891,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { for (const [nounId, otherNoun] of this.nouns.entries()) { if (nounId === id) continue // Skip the noun being removed - // COW: Ensure noun is copied before modification - this.ensureCOW(nounId) for (const [level, connections] of otherNoun.connections.entries()) { if (connections.has(id)) { @@ -1881,8 +1776,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { * Ensure a noun doesn't have too many connections at a given level */ private async pruneConnections(noun: HNSWNoun, level: number): Promise { - // COW: Ensure noun is copied before modification - this.ensureCOW(noun.id) const connections = noun.connections.get(level)! if (connections.size <= this.config.M) { diff --git a/src/hnsw/typeAwareHNSWIndex.ts b/src/hnsw/typeAwareHNSWIndex.ts deleted file mode 100644 index 4c0e8b36..00000000 --- a/src/hnsw/typeAwareHNSWIndex.ts +++ /dev/null @@ -1,763 +0,0 @@ -/** - * Type-Aware HNSW Index - Phase 2 Billion-Scale Optimization - * - * Maintains separate HNSW graphs per entity type for massive memory savings: - * - Memory @ 1B scale: PROJECTED 384GB → 50GB (-87% from architectural analysis, not yet benchmarked) - * - Query speed: PROJECTED 10x faster for single-type queries (not yet benchmarked) - * - Storage: Already type-first from Phase 1a - * - * Architecture: - * - One JsHnswVectorIndex per NounType (42 total) - * - Lazy initialization (indexes created on first use) - * - Type routing for optimal performance - * - Falls back to multi-type search when type unknown - */ - -import { JsHnswVectorIndex } from './hnswIndex.js' -import { - DistanceFunction, - HNSWConfig, - Vector, - VectorDocument -} from '../coreTypes.js' -import { NounType, NOUN_TYPE_COUNT, TypeUtils } from '../types/graphTypes.js' -import { euclideanDistance } from '../utils/index.js' -import type { BaseStorage } from '../storage/baseStorage.js' -import { prodLog } from '../utils/logger.js' - -// Default HNSW parameters (same as JsHnswVectorIndex) -const DEFAULT_CONFIG: HNSWConfig = { - M: 16, - efConstruction: 200, - efSearch: 50, - ml: 16 -} - -/** - * Type-aware HNSW statistics - */ -export interface TypeAwareHNSWStats { - totalNodes: number - totalMemoryMB: number - typeCount: number - typeStats: Map - memoryReductionPercent: number - estimatedMonolithicMemoryMB: number -} - -/** - * TypeAwareHNSWIndex - Separate HNSW graphs per entity type - * - * Phase 2 of billion-scale optimization roadmap. - * PROJECTED: Reduces HNSW memory by 87% @ billion scale (calculated from architecture, not yet benchmarked). - */ -export class TypeAwareHNSWIndex { - // One HNSW index per noun type (lazy initialization) - private indexes: Map = new Map() - - // Configuration - private config: HNSWConfig - private distanceFunction: DistanceFunction - private storage: BaseStorage | null - private useParallelization: boolean - private persistMode: 'immediate' | 'deferred' - - /** - * Create a new TypeAwareHNSWIndex - * - * @param config HNSW configuration (M, efConstruction, efSearch, ml) - * @param distanceFunction Distance function (default: euclidean) - * @param options Additional options (storage, parallelization, persistMode) - */ - constructor( - config: Partial = {}, - distanceFunction: DistanceFunction = euclideanDistance, - options: { useParallelization?: boolean; storage?: BaseStorage; persistMode?: 'immediate' | 'deferred' } = {} - ) { - this.config = { ...DEFAULT_CONFIG, ...config } - this.distanceFunction = distanceFunction - this.storage = options.storage || null - this.useParallelization = - options.useParallelization !== undefined - ? options.useParallelization - : true - this.persistMode = options.persistMode || 'immediate' - - prodLog.info('TypeAwareHNSWIndex initialized (Phase 2: Type-Aware HNSW)') - } - - /** - * Enable COW (Copy-on-Write) mode - Instant fork via shallow copy - * - * Propagates enableCOW() to all underlying type-specific HNSW indexes. - * Each index performs O(1) shallow copy of its own data structures. - * - * @param parent - Parent TypeAwareHNSWIndex to copy from - */ - public enableCOW(parent: TypeAwareHNSWIndex): void { - // Shallow copy indexes Map - this.indexes = new Map(parent.indexes) - - // Enable COW on each underlying type-specific index - for (const [type, parentIndex] of parent.indexes.entries()) { - const childIndex = new JsHnswVectorIndex(this.config, this.distanceFunction, { - useParallelization: this.useParallelization, - storage: this.storage || undefined, - persistMode: this.persistMode - }) - childIndex.enableCOW(parentIndex) - this.indexes.set(type, childIndex) - } - - prodLog.info(`TypeAwareHNSWIndex COW enabled: ${parent.indexes.size} type-specific indexes shallow copied`) - } - - /** - * Flush dirty HNSW data to storage for all type-specific indexes - * - * In deferred persistence mode, HNSW connections are tracked as dirty but not - * immediately persisted. Call flush() to persist all pending changes across - * all type-specific indexes. - * - * @returns Total number of nodes flushed across all indexes - */ - public async flush(): Promise { - if (this.indexes.size === 0) { - return 0 - } - - const flushPromises = Array.from(this.indexes.values()).map(index => - index.flush() - ) - - const results = await Promise.all(flushPromises) - const totalFlushed = results.reduce((sum, count) => sum + count, 0) - - if (totalFlushed > 0) { - prodLog.info(`[TypeAwareHNSW] Flushed ${totalFlushed} dirty nodes across ${this.indexes.size} type indexes`) - } - - return totalFlushed - } - - /** - * Get the total number of dirty (unpersisted) nodes across all type-specific indexes - */ - public getDirtyNodeCount(): number { - let total = 0 - for (const index of this.indexes.values()) { - total += index.getDirtyNodeCount() - } - return total - } - - /** - * Get the current persist mode - */ - public getPersistMode(): 'immediate' | 'deferred' { - return this.persistMode - } - - /** - * Switch persist mode at runtime. Propagates to all existing type-specific indexes. - * Used by addMany() to defer persistence during batch operations. - */ - public setPersistMode(mode: 'immediate' | 'deferred'): void { - this.persistMode = mode - for (const index of this.indexes.values()) { - if (typeof (index as any).setPersistMode === 'function') { - (index as any).setPersistMode(mode) - } - } - } - - /** - * Get or create HNSW index for a specific type (lazy initialization) - * - * Indexes are created on-demand to save memory. - * Only types with entities get an index. - * - * @param type The noun type - * @returns JsHnswVectorIndex for this type - */ - private getIndexForType(type: NounType): JsHnswVectorIndex { - // Validate type is a valid NounType - const typeIndex = TypeUtils.getNounIndex(type) - if (typeIndex === undefined || typeIndex === null || typeIndex < 0) { - throw new Error( - `Invalid NounType: ${type}. Must be one of the 42 defined types.` - ) - } - - if (!this.indexes.has(type)) { - prodLog.info(`Creating HNSW index for type: ${type}`) - - const index = new JsHnswVectorIndex(this.config, this.distanceFunction, { - useParallelization: this.useParallelization, - storage: this.storage || undefined, - persistMode: this.persistMode - }) - - this.indexes.set(type, index) - } - - const index = this.indexes.get(type) - if (!index) { - throw new Error( - `Unexpected: Index for type ${type} not found after creation` - ) - } - return index - } - - /** - * Add a vector to the type-aware index - * - * Routes to the correct type's HNSW graph. - * - * @param item Vector document to add - * @param type The noun type (required for routing) - * @returns The item ID - */ - public async addItem(item: VectorDocument, type: NounType): Promise { - if (!item || !item.vector) { - throw new Error( - 'Invalid VectorDocument: item or vector is null/undefined' - ) - } - if (!type) { - throw new Error('Type is required for type-aware indexing') - } - - const index = this.getIndexForType(type) - return await index.addItem(item) - } - - /** - * Search for nearest neighbors (type-aware) - * - * **Single-type search** (fast path): - * ```typescript - * await index.search(queryVector, 10, 'person') - * // Searches only person graph (100M nodes instead of 1B) - * ``` - * - * **Multi-type search**: - * ```typescript - * await index.search(queryVector, 10, ['person', 'organization']) - * // Searches person + organization, merges results - * ``` - * - * **All-types search** (fallback): - * ```typescript - * await index.search(queryVector, 10) - * // Searches all 42 graphs (slower but comprehensive) - * ``` - * - * @param queryVector Query vector - * @param k Number of results - * @param type Type or types to search (undefined = all types) - * @param filter Optional filter function - * @returns Array of [id, distance] tuples sorted by distance - */ - public async search( - queryVector: Vector, - k: number = 10, - type?: NounType | NounType[], - filter?: (id: string) => Promise, - options?: { rerank?: { multiplier: number }; candidateIds?: string[] } - ): Promise> { - // Single-type search (fast path) - if (type && typeof type === 'string') { - const index = this.getIndexForType(type) - return await index.search(queryVector, k, filter, options) - } - - // Multi-type search (handle empty array edge case) - if (type && Array.isArray(type) && type.length > 0) { - return await this.searchMultipleTypes(queryVector, k, type, filter, options) - } - - // All-types search (slowest path + empty array fallback) - return await this.searchAllTypes(queryVector, k, filter, options) - } - - /** - * Search across multiple specific types - * - * @param queryVector Query vector - * @param k Number of results - * @param types Array of types to search - * @param filter Optional filter function - * @returns Merged and sorted results - */ - private async searchMultipleTypes( - queryVector: Vector, - k: number, - types: NounType[], - filter?: (id: string) => Promise, - options?: { rerank?: { multiplier: number }; candidateIds?: string[] } - ): Promise> { - const allResults: Array<[string, number]> = [] - - // Search each specified type - for (const type of types) { - if (this.indexes.has(type)) { - const index = this.indexes.get(type)! - const results = await index.search(queryVector, k, filter, options) - allResults.push(...results) - } - } - - // Merge and sort by distance - allResults.sort((a, b) => a[1] - b[1]) - - // Return top k - return allResults.slice(0, k) - } - - /** - * Search across all types (fallback for type-agnostic queries) - * - * This is the slowest path, but provides comprehensive results. - * Used when type cannot be inferred from query. - * - * @param queryVector Query vector - * @param k Number of results - * @param filter Optional filter function - * @returns Merged and sorted results from all types - */ - private async searchAllTypes( - queryVector: Vector, - k: number, - filter?: (id: string) => Promise, - options?: { rerank?: { multiplier: number }; candidateIds?: string[] } - ): Promise> { - const allResults: Array<[string, number]> = [] - - // Search each type's graph - for (const [type, index] of this.indexes.entries()) { - const results = await index.search(queryVector, k, filter, options) - allResults.push(...results) - } - - // Merge and sort by distance - allResults.sort((a, b) => a[1] - b[1]) - - // Return top k - return allResults.slice(0, k) - } - - /** - * Remove an item from the index - * - * @param id Item ID to remove - * @param type The noun type (required for routing) - * @returns True if item was removed, false if not found - */ - public async removeItem(id: string, type: NounType): Promise { - const index = this.indexes.get(type) - if (!index) { - return false // Type has no index (no items ever added) - } - - return await index.removeItem(id) - } - - /** - * Get total number of items across all types - * - * @returns Total item count - */ - public size(): number { - let total = 0 - for (const index of this.indexes.values()) { - total += index.size() - } - return total - } - - /** - * Get number of items for a specific type - * - * @param type The noun type - * @returns Item count for this type - */ - public sizeForType(type: NounType): number { - const index = this.indexes.get(type) - return index ? index.size() : 0 - } - - /** - * Clear all indexes - */ - public clear(): void { - for (const index of this.indexes.values()) { - index.clear() - } - this.indexes.clear() - } - - /** - * Clear index for a specific type - * - * @param type The noun type to clear - */ - public clearType(type: NounType): void { - const index = this.indexes.get(type) - if (index) { - index.clear() - this.indexes.delete(type) - } - } - - /** - * Get configuration - * - * @returns HNSW configuration - */ - public getConfig(): HNSWConfig { - return { ...this.config } - } - - /** - * Get distance function - * - * @returns Distance function - */ - public getDistanceFunction(): DistanceFunction { - return this.distanceFunction - } - - /** - * Set parallelization (applies to all indexes) - * - * @param useParallelization Whether to use parallelization - */ - public setUseParallelization(useParallelization: boolean): void { - this.useParallelization = useParallelization - for (const index of this.indexes.values()) { - index.setUseParallelization(useParallelization) - } - } - - /** - * Get parallelization setting - * - * @returns Whether parallelization is enabled - */ - public getUseParallelization(): boolean { - return this.useParallelization - } - - /** - * Rebuild HNSW indexes from storage (type-aware) - * - * CRITICAL: This implementation uses type-filtered pagination to avoid - * loading ALL entities for each type (which would be 42 billion reads @ 1B scale). - * - * Can rebuild all types or specific types. - * Much faster than rebuilding a monolithic index. - * - * @param options Rebuild options - */ - public async rebuild( - options: { - types?: NounType[] // Rebuild specific types (undefined = all types) - batchSize?: number // Entities per batch - onProgress?: (type: NounType, loaded: number, total: number) => void - } = {} - ): Promise { - if (!this.storage) { - prodLog.warn('TypeAwareHNSW rebuild skipped: no storage adapter') - return - } - - const batchSize = options.batchSize || 1000 - - // Determine which types to rebuild - const typesToRebuild = options.types || this.getAllNounTypes() - - prodLog.info( - `Rebuilding ${typesToRebuild.length} type-aware HNSW indexes from persisted data...` - ) - - // Clear all indexes we're rebuilding - for (const type of typesToRebuild) { - const index = this.getIndexForType(type) - ;(index as any).nouns.clear() - } - - // Determine preloading strategy (adaptive caching) for entire dataset - const stats = await this.storage.getStatistics() - const entityCount = stats?.totalNodes || 0 - const vectorMemory = entityCount * 1536 // 384 dims × 4 bytes - - // Use first index's cache (they all share the same UnifiedCache) - const firstIndex = this.getIndexForType(typesToRebuild[0]) - const cacheStats = (firstIndex as any).unifiedCache.getStats() - const availableCache = cacheStats.maxSize * 0.80 - const shouldPreload = vectorMemory < availableCache - - if (shouldPreload) { - prodLog.info( - `HNSW: Preloading ${entityCount.toLocaleString()} vectors at init ` + - `(${(vectorMemory / 1024 / 1024).toFixed(1)}MB < ${(availableCache / 1024 / 1024).toFixed(1)}MB cache)` - ) - } else { - prodLog.info( - `HNSW: Adaptive caching for ${entityCount.toLocaleString()} vectors ` + - `(${(vectorMemory / 1024 / 1024).toFixed(1)}MB > ${(availableCache / 1024 / 1024).toFixed(1)}MB cache) - loading on-demand` - ) - } - - // Load ALL nouns ONCE and route to correct type indexes - // This is O(N) instead of O(42*N) from the previous parallel approach - let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop) - let hasMore = true - let totalLoaded = 0 - const loadedByType = new Map() - - while (hasMore) { - const result: { - items: Array<{ id: string; vector: number[]; type?: NounType }> - hasMore: boolean - nextCursor?: string - totalCount?: number - } = await (this.storage as any).getNounsWithPagination({ - limit: batchSize, - offset // Pass offset for proper pagination (previously passed cursor which was ignored) - }) - - // Route each noun to its type index - for (const nounData of result.items) { - try { - // Use 'type' property from HNSWNounWithMetadata (not 'nounType') - // Previously accessed wrong property, causing N+1 metadata fetches - // getNounsWithPagination already includes type in response - const nounType = nounData.type - - // Skip if type not in rebuild list - if (!nounType || !typesToRebuild.includes(nounType as NounType)) { - continue - } - - // Get the index for this type - const index = this.getIndexForType(nounType as NounType) - - // Load HNSW graph data - const hnswData = await (this.storage as any).getVectorIndexData(nounData.id) - if (!hnswData) { - continue // No HNSW data - } - - // Create noun with restored connections - const noun = { - id: nounData.id, - vector: shouldPreload ? nounData.vector : [], - connections: new Map(), - level: hnswData.level - } - - // Restore connections from storage - for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { - const level = parseInt(levelStr, 10) - noun.connections.set(level, new Set(nounIds as string[])) - } - - // Add to type-specific index - ;(index as any).nouns.set(nounData.id, noun) - - // Track high-level nodes - if (noun.level >= 2 && noun.level <= (index as any).MAX_TRACKED_LEVELS) { - if (!(index as any).highLevelNodes.has(noun.level)) { - ;(index as any).highLevelNodes.set(noun.level, new Set()) - } - ;(index as any).highLevelNodes.get(noun.level).add(nounData.id) - } - - // Track progress - loadedByType.set(nounType as NounType, (loadedByType.get(nounType as NounType) || 0) + 1) - totalLoaded++ - - if (options.onProgress && totalLoaded % 100 === 0) { - options.onProgress(nounType as NounType, loadedByType.get(nounType as NounType) || 0, totalLoaded) - } - } catch (error) { - prodLog.error(`Failed to restore HNSW data for ${nounData.id}:`, error) - } - } - - hasMore = result.hasMore - offset += batchSize // Increment offset for next page - - // Progress logging - if (totalLoaded % 1000 === 0) { - prodLog.info(`Progress: ${totalLoaded.toLocaleString()} entities loaded...`) - } - } - - // Restore entry points for each type - for (const type of typesToRebuild) { - const index = this.getIndexForType(type) - let maxLevel = 0 - let entryPointId: string | null = null - - for (const [id, noun] of (index as any).nouns.entries()) { - if (entryPointId === null || noun.level > maxLevel) { - maxLevel = noun.level - entryPointId = id - } - } - - // Recovery: if still null after loop but nouns exist - if ((index as any).nouns.size > 0 && !entryPointId) { - const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1() - entryPointId = recoveredId - maxLevel = recoveredLevel - } - - // Validation: if entry point doesn't exist in loaded nouns - if (entryPointId && !(index as any).nouns.has(entryPointId)) { - const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1() - entryPointId = recoveredId - maxLevel = recoveredLevel - } - - ;(index as any).entryPointId = entryPointId - ;(index as any).maxLevel = maxLevel - - const loaded = loadedByType.get(type) || 0 - const cacheInfo = shouldPreload ? ' (vectors preloaded)' : ' (adaptive caching)' - - prodLog.info( - `✅ Rebuilt ${type} index: ${loaded.toLocaleString()} entities, ` + - `${maxLevel + 1} levels, entry point: ${entryPointId || 'none'}${cacheInfo}` - ) - } - - prodLog.info( - `✅ TypeAwareHNSW rebuild complete: ${this.size().toLocaleString()} total entities across ${this.indexes.size} types (loaded from persisted graph structure)` - ) - } - - /** - * Get comprehensive statistics - * - * Shows memory reduction compared to monolithic approach. - * - * @returns Type-aware HNSW statistics - */ - public getStats(): TypeAwareHNSWStats { - const typeStats = new Map< - NounType, - { - nodeCount: number - memoryMB: number - maxLevel: number - entryPointId: string | null - } - >() - - let totalNodes = 0 - let totalMemoryMB = 0 - - // Collect stats from each type's index - for (const [type, index] of this.indexes.entries()) { - const cacheStats = index.getCacheStats() - const nodeCount = index.size() - const memoryMB = cacheStats.hnswCache.estimatedMemoryMB - - typeStats.set(type, { - nodeCount, - memoryMB, - maxLevel: index.getMaxLevel(), - entryPointId: index.getEntryPointId() - }) - - totalNodes += nodeCount - totalMemoryMB += memoryMB - } - - // Estimate monolithic memory (for comparison) - // Monolithic would use ~384 bytes per entity @ 1B scale - const estimatedMonolithicMemoryMB = (totalNodes * 384) / (1024 * 1024) - - // Calculate memory reduction - const memoryReductionPercent = - estimatedMonolithicMemoryMB > 0 - ? ((estimatedMonolithicMemoryMB - totalMemoryMB) / - estimatedMonolithicMemoryMB) * - 100 - : 0 - - return { - totalNodes, - totalMemoryMB: parseFloat(totalMemoryMB.toFixed(2)), - typeCount: this.indexes.size, - typeStats, - memoryReductionPercent: parseFloat(memoryReductionPercent.toFixed(2)), - estimatedMonolithicMemoryMB: parseFloat( - estimatedMonolithicMemoryMB.toFixed(2) - ) - } - } - - /** - * Get statistics for a specific type - * - * @param type The noun type - * @returns Statistics for this type's index (null if no index) - */ - public getStatsForType( - type: NounType - ): { - nodeCount: number - memoryMB: number - maxLevel: number - entryPointId: string | null - cacheStats: any - } | null { - const index = this.indexes.get(type) - if (!index) { - return null - } - - const cacheStats = index.getCacheStats() - - return { - nodeCount: index.size(), - memoryMB: cacheStats.hnswCache.estimatedMemoryMB, - maxLevel: index.getMaxLevel(), - entryPointId: index.getEntryPointId(), - cacheStats - } - } - - /** - * Get all noun types (for iteration) - * - * @returns Array of all noun types - */ - private getAllNounTypes(): NounType[] { - const types: NounType[] = [] - for (let i = 0; i < NOUN_TYPE_COUNT; i++) { - types.push(TypeUtils.getNounFromIndex(i)) - } - return types - } - - /** - * Get list of types that have indexes (have entities) - * - * @returns Array of types with indexes - */ - public getActiveTypes(): NounType[] { - return Array.from(this.indexes.keys()) - } -} diff --git a/src/index.ts b/src/index.ts index 0416f283..e7270a82 100644 --- a/src/index.ts +++ b/src/index.ts @@ -146,6 +146,7 @@ export type { TxUnrelateOperation, TransactOptions, TransactReceipt, + TxLogEntry, CompactHistoryOptions, CompactHistoryResult, ChangedIds @@ -221,23 +222,6 @@ export { MemoryStorage, createStorage } // FileSystemStorage is exported separately to avoid browser build issues. export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js' -// Export COW (Copy-on-Write) infrastructure -import { CommitLog } from './storage/cow/CommitLog.js' -import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js' -import { BlobStorage } from './storage/cow/BlobStorage.js' -import { RefManager } from './storage/cow/RefManager.js' -import { TreeObject } from './storage/cow/TreeObject.js' - -export { - // COW infrastructure - CommitLog, - CommitObject, - CommitBuilder, - BlobStorage, - RefManager, - TreeObject -} - // Export types import type { Vector, diff --git a/src/indexes/columnStore/ColumnManifest.ts b/src/indexes/columnStore/ColumnManifest.ts index 659a6c2e..c77098f9 100644 --- a/src/indexes/columnStore/ColumnManifest.ts +++ b/src/indexes/columnStore/ColumnManifest.ts @@ -2,9 +2,8 @@ * @module columnStore/ColumnManifest * @description Per-field manifest tracking segment files and field metadata. * - * Stored as JSON at `_column_index/{field}/MANIFEST.json` using workspace-global - * storage paths (not branch-scoped, same as `_cow/` metadata). Rewritten - * atomically on every flush and compaction. + * Stored as JSON at `_column_index/{field}/MANIFEST.json` using storage-root- + * relative paths. Rewritten atomically on every flush and compaction. * * The manifest is the single source of truth for which segments exist for a * field. On startup, the column store loads each field's manifest and diff --git a/src/migration/MigrationRunner.ts b/src/migration/MigrationRunner.ts index c848673f..d2e251a2 100644 --- a/src/migration/MigrationRunner.ts +++ b/src/migration/MigrationRunner.ts @@ -192,7 +192,7 @@ export class MigrationRunner { * Entity-level errors are tracked (not thrown). If maxErrors is exceeded, migration * stops early and returns partial results with errors. */ - async run(options?: Pick): Promise> { + async run(options?: Pick): Promise> { const state = await this.getState() const pending = this.getPendingMigrations(state) @@ -259,93 +259,6 @@ export class MigrationRunner { } } - /** - * Run specific migrations without checking completion state. - * Used for branch iterations where the state on main says "completed" - * but branch-local entities still need transforming. - * - * Safe because transforms are idempotent (return null when already applied). - * Does NOT save migration state — the authoritative state lives on main. - */ - async runMigrations( - migrations: Migration[], - options?: Pick - ): Promise> { - if (migrations.length === 0) { - return { migrationsApplied: [], entitiesProcessed: 0, entitiesModified: 0, errors: [] } - } - - let totalProcessed = 0 - let totalModified = 0 - const appliedMigrations: string[] = [] - const errors: MigrationError[] = [] - const maxErrors = options?.maxErrors ?? DEFAULT_MAX_ERRORS - const batchConfig = this.storage.getBatchConfig() - const batchSize = batchConfig.maxBatchSize - const batchDelay = batchConfig.batchDelayMs - - for (const migration of migrations) { - if (errors.length >= maxErrors) break - - let processed = 0 - let modified = 0 - - if (migration.applies === 'nouns' || migration.applies === 'both') { - const result = await this.migrateNouns(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress) - processed += result.processed - modified += result.modified - } - - if (migration.applies === 'verbs' || migration.applies === 'both') { - if (errors.length < maxErrors) { - const result = await this.migrateVerbs(migration, 0, batchSize, batchDelay, errors, maxErrors, options?.onProgress) - processed += result.processed - modified += result.modified - } - } - - totalProcessed += processed - totalModified += modified - if (modified > 0) { - appliedMigrations.push(migration.id) - } - } - - return { - migrationsApplied: appliedMigrations, - entitiesProcessed: totalProcessed, - entitiesModified: totalModified, - errors - } - } - - /** - * Clean up old system:backup branches created by previous migrations. - * Identifies backups via ref metadata (not by name prefix). - */ - async cleanupOldBackups(): Promise { - const refManager = this.storage.refManager - if (!refManager) return - - const refs = await refManager.listRefs() - const currentBranch = this.storage.currentBranch || 'main' - - for (const ref of refs) { - if ( - ref.metadata?.type === 'system:backup' && - ref.name.startsWith('refs/heads/') && - ref.name !== `refs/heads/${currentBranch}` - ) { - const branchName = ref.name.replace('refs/heads/', '') - try { - await refManager.deleteRef(branchName) - } catch { - // Ignore — branch may be current or protected - } - } - } - } - // ─── Private helpers ─────────────────────────────────────────────── private async migrateNouns( @@ -524,12 +437,11 @@ export class MigrationRunner { private async saveResumeState(migrationId: string, offset: number): Promise { const state = await this.getState() - const branch = this.storage.currentBranch || 'main' await this.saveState({ completedVersion: state?.completedVersion || '', completedAt: state?.completedAt || 0, completedMigrations: state?.completedMigrations || [], - resumeState: { migrationId, lastProcessedOffset: offset, branch } + resumeState: { migrationId, lastProcessedOffset: offset } }) } diff --git a/src/migration/types.ts b/src/migration/types.ts index 53a23930..2dcc1d1a 100644 --- a/src/migration/types.ts +++ b/src/migration/types.ts @@ -29,7 +29,6 @@ export interface MigrationState { resumeState?: { migrationId: string lastProcessedOffset: number - branch: string } } @@ -56,8 +55,8 @@ export interface MigrationError { } export interface MigrationResult { - /** Backup branch name, or null if no changes were needed */ - backupBranch: string | null + /** Path of the pre-migration snapshot, or null when no `backupTo` was supplied */ + backupPath: string | null /** IDs of migrations that were applied */ migrationsApplied: string[] /** Total entities processed (scanned) */ @@ -71,6 +70,13 @@ export interface MigrationResult { export interface MigrateOptions { /** Preview what would change without writing */ dryRun?: boolean + /** + * Directory to persist a pre-migration snapshot into (created; must be + * empty or absent). The snapshot is cut via the Db API (hard-link + * snapshot of the current generation) BEFORE any transform runs; restore + * it wholesale with `brain.restore(path, { confirm: true })`. + */ + backupTo?: string /** Progress callback for long-running migrations */ onProgress?: (progress: { migrationId: string diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 4e97a0f9..692baa99 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -152,9 +152,8 @@ export abstract class BaseStorageAdapter implements StorageAdapter { // "graph-lsm/source/sstable-123" "/_blobs/graph-lsm/source/sstable-123.bin" // // i.e. the key's "/"-separated segments are nested under a `_blobs/` prefix and - // suffixed with `.bin`. Blobs are deliberately NOT branch-scoped (COW): they - // are immutable, content-addressed segments managed by their producer, - // mirroring how COW metadata under `_cow/` is also kept global. + // suffixed with `.bin`. Blobs are immutable, content-addressed segments + // managed by their producer. /** * Persist a raw binary blob under `key`, writing the bytes verbatim (no JSON diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 535c9e77..f2732fe0 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -912,7 +912,7 @@ export class FileSystemStorage extends BaseStorage { if (entry.isFile()) { // Handle multiple compression formats for broad compatibility // - .json.gz: Standard entity/metadata files (JSON compressed) - // - .gz: COW files (refs, blobs, commits - raw compressed) + // - .gz: Raw compressed payloads (e.g. blob-store binary values) // - .json: Uncompressed JSON files if (entry.name.endsWith('.json.gz')) { // Strip .gz extension and add the .json path @@ -923,7 +923,7 @@ export class FileSystemStorage extends BaseStorage { seen.add(normalizedPath) } } else if (entry.name.endsWith('.gz')) { - // COW files stored as .gz (not .json.gz) + // Raw payloads stored as .gz (not .json.gz) // Strip .gz extension and return path const normalizedName = entry.name.slice(0, -3) // Remove .gz const normalizedPath = path.join(prefix, normalizedName) @@ -1387,12 +1387,10 @@ export class FileSystemStorage extends BaseStorage { } } - // Clear the entire branches/ directory (branch-based storage) - // Bug fix: Data is stored in branches/main/entities/, not just entities/ - // The branch-based structure was introduced for COW support - const branchesDir = path.join(this.rootDir, 'branches') - if (await this.directoryExists(branchesDir)) { - await removeDirectoryContents(branchesDir) + // Clear the canonical entity/verb data area + const entitiesDir = path.join(this.rootDir, 'entities') + if (await this.directoryExists(entitiesDir)) { + await removeDirectoryContents(entitiesDir) } // Remove all files in both system directories @@ -1401,19 +1399,16 @@ export class FileSystemStorage extends BaseStorage { await removeDirectoryContents(this.indexDir) } - // Remove COW (copy-on-write) version control data - // This directory stores all git-like versioning data (commits, trees, blobs, refs) - // Must be deleted to fully clear all data including version history - const cowDir = path.join(this.rootDir, '_cow') - if (await this.directoryExists(cowDir)) { - // Delete the entire _cow/ directory (not just contents) - await fs.promises.rm(cowDir, { recursive: true, force: true }) + // Remove the content-addressed blob store (VFS file content) + const casDir = path.join(this.rootDir, '_cas') + if (await this.directoryExists(casDir)) { + // Delete the entire _cas/ directory (not just contents) + await fs.promises.rm(casDir, { recursive: true, force: true }) - // Reset COW managers (but don't disable COW - it's always enabled) - // COW will re-initialize automatically on next use - this.refManager = undefined + // Drop the BlobStorage instance — its LRU cache holds deleted blobs. + // initializeBlobStorage() re-creates it (brain.clear() does this before + // the VFS is rebuilt). this.blobStorage = undefined - this.commitLog = undefined } // Clear the statistics cache @@ -1426,7 +1421,7 @@ export class FileSystemStorage extends BaseStorage { ;(this as any).totalVerbCount = 0 // Clear write-through cache (inherited from BaseStorage) - // Without this, readWithInheritance() would return stale cached data + // Without this, readCanonicalObject() would return stale cached data // after clear(), causing "ghost" entities to appear this.clearWriteCache() } @@ -1457,17 +1452,6 @@ export class FileSystemStorage extends BaseStorage { return result } - /** - * Check if COW has been explicitly disabled via clear() - * Fixes bug where clear() doesn't persist across instance restarts - * @returns true if marker file exists, false otherwise - * @protected - */ - /** - * Removed checkClearMarker() and createClearMarker() methods - * COW is now always enabled - marker files are no longer used - */ - /** * Get information about storage usage and capacity */ diff --git a/src/storage/adapters/historicalStorageAdapter.ts b/src/storage/adapters/historicalStorageAdapter.ts deleted file mode 100644 index 6ff29710..00000000 --- a/src/storage/adapters/historicalStorageAdapter.ts +++ /dev/null @@ -1,571 +0,0 @@ -/** - * Historical Storage Adapter - * - * Provides lazy-loading read-only access to a historical commit state. - * Uses LRU cache to bound memory usage and prevent eager-loading of entire history. - * - * Architecture: - * - Extends BaseStorage to inherit all storage infrastructure - * - Wraps an underlying storage adapter to access commit state - * - Implements lazy-loading with LRU cache (bounded memory) - * - All writes throw read-only errors - * - All reads load from historical commit state on-demand - * - * Usage: - * const historical = new HistoricalStorageAdapter({ - * underlyingStorage: brain.storage as BaseStorage, - * commitId: 'abc123...', - * cacheSize: 10000 // LRU cache size - * }) - * await historical.init() - * - * Performance: - * - O(1) cache lookups for frequently accessed entities - * - Bounded memory: max cacheSize entities in memory - * - Lazy loading: only loads entities when accessed - * - No eager-loading of entire commit state - * - * Production-ready, billion-scale historical queries - */ - -import { BaseStorage } from '../baseStorage.js' -import { CommitLog } from '../cow/CommitLog.js' -import { BlobStorage } from '../cow/BlobStorage.js' -import { - HNSWNoun, - HNSWVerb, - NounMetadata, - VerbMetadata, - StatisticsData -} from '../../coreTypes.js' - -/** - * Simple LRU Cache implementation - * Bounds memory usage by evicting least-recently-used items - */ -class LRUCache { - private cache = new Map() - private accessOrder: string[] = [] - private maxSize: number - - constructor(maxSize: number = 10000) { - this.maxSize = maxSize - } - - get(key: string): T | undefined { - const value = this.cache.get(key) - if (value !== undefined) { - // Move to end (most recently used) - this.accessOrder = this.accessOrder.filter(k => k !== key) - this.accessOrder.push(key) - } - return value - } - - set(key: string, value: T): void { - // Remove if already exists - if (this.cache.has(key)) { - this.accessOrder = this.accessOrder.filter(k => k !== key) - } - - // Add to cache - this.cache.set(key, value) - this.accessOrder.push(key) - - // Evict oldest if over capacity - if (this.cache.size > this.maxSize) { - const oldest = this.accessOrder.shift() - if (oldest) { - this.cache.delete(oldest) - } - } - } - - has(key: string): boolean { - return this.cache.has(key) - } - - clear(): void { - this.cache.clear() - this.accessOrder = [] - } - - get size(): number { - return this.cache.size - } -} - -export interface HistoricalStorageAdapterOptions { - /** Underlying storage to access commit state from */ - underlyingStorage: BaseStorage - - /** Commit ID to load historical state from */ - commitId: string - - /** Max number of entities to cache (default: 10000) */ - cacheSize?: number - - /** Branch containing the commit (default: 'main') */ - branch?: string -} - -/** - * Historical Storage Adapter - * - * Lazy-loading, read-only storage adapter for historical commit state. - * Implements billion-scale time-travel queries with bounded memory. - */ -export class HistoricalStorageAdapter extends BaseStorage { - private underlyingStorage: BaseStorage - private commitId: string - private branch: string - private cacheSize: number - - // LRU caches for lazy-loaded entities - private cache: LRUCache - - // Historical commit state (loaded lazily) - must match BaseStorage visibility - public commitLog?: CommitLog - public blobStorage?: BlobStorage - - constructor(options: HistoricalStorageAdapterOptions) { - super() - this.underlyingStorage = options.underlyingStorage - this.commitId = options.commitId - this.branch = options.branch || 'main' - this.cacheSize = options.cacheSize || 10000 - - this.cache = new LRUCache(this.cacheSize) - } - - /** - * Initialize historical storage adapter - * Loads commit metadata but NOT entity data (lazy loading) - */ - public async init(): Promise { - // Get COW components from underlying storage - // Fixed property names - use public properties without underscore prefix - this.commitLog = this.underlyingStorage.commitLog - this.blobStorage = this.underlyingStorage.blobStorage - - if (!this.commitLog || !this.blobStorage) { - throw new Error( - 'Historical storage requires underlying storage to have COW enabled. ' + - 'Call brain.init() first to initialize COW.' - ) - } - - // Verify commit exists - const commit = await this.commitLog.getCommit(this.commitId) - if (!commit) { - throw new Error(`Commit not found: ${this.commitId}`) - } - - // Initialize GraphAdjacencyIndex and type statistics - await super.init() - } - - // ============= Abstract Method Implementations ============= - - /** - * Read object from historical commit state - * Uses LRU cache to avoid repeated blob reads - */ - protected async readObjectFromPath(path: string): Promise { - // Check cache first - if (this.cache.has(path)) { - return this.cache.get(path) || null - } - - try { - // Import COW classes - const { CommitObject } = await import('../cow/CommitObject.js') - const { TreeObject } = await import('../cow/TreeObject.js') - const { isNullHash } = await import('../cow/constants.js') - - // Read commit - const commit = await CommitObject.read(this.blobStorage!, this.commitId) - if (isNullHash(commit.tree)) { - return null - } - - // Read tree - const tree = await TreeObject.read(this.blobStorage!, commit.tree) - - // Walk tree to find matching path - for await (const entry of TreeObject.walk(this.blobStorage!, tree)) { - if (entry.type === 'blob' && entry.name === path) { - // Read blob data - const blobData = await this.blobStorage!.read(entry.hash) - const data = JSON.parse(blobData.toString()) - - // Cache the result - this.cache.set(path, data) - - return data - } - } - - return null - } catch (error) { - // Path doesn't exist in historical state - return null - } - } - - /** - * List objects under path in historical commit state - */ - protected async listObjectsUnderPath(prefix: string): Promise { - try { - // Import COW classes - const { CommitObject } = await import('../cow/CommitObject.js') - const { TreeObject } = await import('../cow/TreeObject.js') - const { isNullHash } = await import('../cow/constants.js') - - // Read commit - const commit = await CommitObject.read(this.blobStorage!, this.commitId) - if (isNullHash(commit.tree)) { - return [] - } - - // Read tree - const tree = await TreeObject.read(this.blobStorage!, commit.tree) - - // Walk tree to find all paths matching prefix - const paths: string[] = [] - for await (const entry of TreeObject.walk(this.blobStorage!, tree)) { - if (entry.name.startsWith(prefix)) { - paths.push(entry.name) - } - } - - return paths - } catch (error) { - return [] - } - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - protected async writeObjectToPath(path: string, data: any): Promise { - throw new Error( - `Historical storage is read-only. Cannot write to path: ${path}` - ) - } - - /** - * DELETE BLOCKED: Historical storage is read-only - */ - protected async deleteObjectFromPath(path: string): Promise { - throw new Error( - `Historical storage is read-only. Cannot delete path: ${path}` - ) - } - - // =========================================================================== - // Raw binary-blob primitive (read-only) - // =========================================================================== - - /** - * WRITE BLOCKED: Historical storage is read-only. - * - * @param key - The blob key (unused; included for the error message). - * @throws Always — historical state is immutable. - */ - public async saveBinaryBlob(key: string, _data: Buffer): Promise { - throw new Error( - `Historical storage is read-only. Cannot save binary blob: ${key}` - ) - } - - /** - * Load the raw bytes of a binary blob from the historical commit state by - * walking the commit tree for the blob entry at `_blobs/.bin` and reading - * its content-addressed bytes verbatim (no JSON decode). Returns `null` if the - * blob was not present in this commit. - * - * @param key - The blob key (same convention as the live adapters). - * @returns The blob bytes as stored at this commit, or `null` if absent. - */ - public async loadBinaryBlob(key: string): Promise { - try { - const { CommitObject } = await import('../cow/CommitObject.js') - const { TreeObject } = await import('../cow/TreeObject.js') - const { isNullHash } = await import('../cow/constants.js') - - const commit = await CommitObject.read(this.blobStorage!, this.commitId) - if (isNullHash(commit.tree)) { - return null - } - - const tree = await TreeObject.read(this.blobStorage!, commit.tree) - const blobName = `_blobs/${key}.bin` - - for await (const entry of TreeObject.walk(this.blobStorage!, tree)) { - if (entry.type === 'blob' && entry.name === blobName) { - return await this.blobStorage!.read(entry.hash) - } - } - - return null - } catch (error) { - // Blob not present in historical state - return null - } - } - - /** - * DELETE BLOCKED: Historical storage is read-only. - * - * @param key - The blob key (unused; included for the error message). - * @throws Always — historical state is immutable. - */ - public async deleteBinaryBlob(key: string): Promise { - throw new Error( - `Historical storage is read-only. Cannot delete binary blob: ${key}` - ) - } - - /** - * Historical state lives in the content-addressed commit store, not on the - * local filesystem, so there is no mmap-able path. Always returns `null`. - * - * @param _key - The blob key (unused). - * @returns Always `null`. - */ - public getBinaryBlobPath(_key: string): string | null { - return null - } - - /** - * Get storage statistics from historical commit - */ - protected async getStatisticsData(): Promise { - return await this.readObjectFromPath('_system/statistics.json') - } - - /** - * WRITE BLOCKED: Cannot save statistics to historical storage - */ - protected async saveStatisticsData(data: StatisticsData): Promise { - throw new Error('Historical storage is read-only. Cannot save statistics.') - } - - /** - * Clear cache (does not affect historical data) - */ - public async clear(): Promise { - this.cache.clear() - } - - /** - * Get storage status - */ - public async getStorageStatus(): Promise<{ - type: string - used: number - quota: number | null - details?: Record - }> { - return { - type: 'historical', - used: this.cache.size, - quota: this.cacheSize, - details: { - commitId: this.commitId, - branch: this.branch, - cached: this.cache.size, - maxCache: this.cacheSize, - readOnly: true - } - } - } - - /** - * Check if COW has been explicitly disabled via clear() - * No-op for HistoricalStorageAdapter (read-only, doesn't manage COW) - * @returns Always false (read-only adapter doesn't manage COW state) - * @protected - */ - /** - * Removed checkClearMarker() and createClearMarker() methods - * COW is now always enabled - marker files are no longer used - */ - - // ============= Override Write Methods (Read-Only) ============= - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async saveNoun(noun: HNSWNoun): Promise { - throw new Error('Historical storage is read-only. Cannot save noun.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async saveNounMetadata(id: string, metadata: NounMetadata): Promise { - throw new Error('Historical storage is read-only. Cannot save noun metadata.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async deleteNoun(id: string): Promise { - throw new Error('Historical storage is read-only. Cannot delete noun.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async deleteNounMetadata(id: string): Promise { - throw new Error('Historical storage is read-only. Cannot delete noun metadata.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async saveVerb(verb: HNSWVerb): Promise { - throw new Error('Historical storage is read-only. Cannot save verb.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async saveVerbMetadata(id: string, metadata: VerbMetadata): Promise { - throw new Error('Historical storage is read-only. Cannot save verb metadata.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async deleteVerb(id: string): Promise { - throw new Error('Historical storage is read-only. Cannot delete verb.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async deleteVerbMetadata(id: string): Promise { - throw new Error('Historical storage is read-only. Cannot delete verb metadata.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async saveMetadata(id: string, metadata: any): Promise { - throw new Error('Historical storage is read-only. Cannot save metadata.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async saveVectorIndexData(nounId: string, hnswData: { - level: number - connections: Record - }): Promise { - throw new Error('Historical storage is read-only. Cannot save HNSW data.') - } - - /** - * WRITE BLOCKED: Historical storage is read-only - */ - public async saveHNSWSystem(systemData: { - entryPointId: string | null - maxLevel: number - }): Promise { - throw new Error('Historical storage is read-only. Cannot save HNSW system data.') - } - - // ============= Additional Abstract Methods ============= - - /** - * Get noun vector from historical state - */ - public async getNounVector(id: string): Promise { - const noun = await this.getNoun(id) - return noun?.vector || null - } - - /** - * Get HNSW data from historical state - */ - public async getVectorIndexData(nounId: string): Promise<{ - level: number - connections: Record - } | null> { - const path = `_system/hnsw/nodes/${nounId}.json` - return await this.readObjectFromPath(path) - } - - /** - * Get HNSW system data from historical state - */ - public async getHNSWSystem(): Promise<{ - entryPointId: string | null - maxLevel: number - } | null> { - return await this.readObjectFromPath('_system/hnsw/system.json') - } - - /** - * Initialize counts (no-op for historical storage) - * Counts are loaded from historical state metadata - */ - protected async initializeCounts(): Promise { - // No-op: Historical storage doesn't need to initialize counts - // They're read from commit state metadata - } - - /** - * WRITE BLOCKED: Cannot persist counts to historical storage - */ - protected async persistCounts(): Promise { - // No-op: Historical storage is read-only - } - - // =========================================================================== - // Generational record layer (8.0 MVCC) — not applicable to this adapter - // - // HistoricalStorageAdapter is the pre-8.0 commit-based time-travel view and - // is read-only by construction. The generation store never registers its - // write hooks on reader-mode instances, so these methods are unreachable in - // normal operation; they throw explicitly rather than fake behavior. - // =========================================================================== - - /** - * WRITE BLOCKED: the tx-log belongs to the live store, not a historical view. - */ - public async appendTxLogLine(_line: string): Promise { - throw new Error('Historical storage is read-only. Cannot append to the transaction log.') - } - - /** - * Not supported: historical commit views predate the generational tx-log. - */ - public async readTxLogLines(): Promise { - throw new Error( - 'Transaction-log reads are not supported on a historical commit view. ' + - 'Use the live store (or a Db from brain.now()/brain.asOf()).' - ) - } - - /** - * Not supported: snapshot a live store via db.persist(path) instead. - */ - public async snapshotToDirectory(_targetPath: string): Promise { - throw new Error( - 'snapshotToDirectory is not supported on a historical commit view. ' + - 'Call db.persist(path) on the live store instead.' - ) - } - - /** - * WRITE BLOCKED: cannot restore into a read-only historical view. - */ - public async restoreFromDirectory(_sourcePath: string): Promise { - throw new Error('Historical storage is read-only. Cannot restore from a snapshot.') - } -} diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index f7ea1125..0cf381cc 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -362,8 +362,13 @@ export class MemoryStorage extends BaseStorage { this.statisticsCache = null this.statisticsModified = false + // Drop the BlobStorage instance — its LRU cache holds deleted blobs. + // initializeBlobStorage() re-creates it (brain.clear() does this before + // the VFS is rebuilt). + this.blobStorage = undefined + // Clear write-through cache (inherited from BaseStorage) - // Without this, readWithInheritance() would return stale cached data + // Without this, readCanonicalObject() would return stale cached data // after clear(), causing "ghost" entities to appear this.clearWriteCache() } @@ -390,17 +395,6 @@ export class MemoryStorage extends BaseStorage { } } - /** - * Check if COW has been explicitly disabled via clear() - * No-op for MemoryStorage (doesn't persist) - * @returns Always false (marker doesn't persist in memory) - * @protected - */ - /** - * Removed checkClearMarker() and createClearMarker() methods - * COW is now always enabled - marker files are no longer used - */ - /** * Save statistics data to storage * @param statistics The statistics data to save diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index c7380caf..e32c8965 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -26,10 +26,8 @@ import { VERB_TYPE_COUNT } from '../types/graphTypes.js' import { getShardIdFromUuid } from './sharding.js' -import { RefManager } from './cow/RefManager.js' -import { BlobStorage, type COWStorageAdapter } from './cow/BlobStorage.js' -import { CommitLog } from './cow/CommitLog.js' -import { unwrapBinaryData } from './cow/binaryDataCodec.js' +import { BlobStorage, type BlobStoreAdapter } from './blobStorage.js' +import { unwrapBinaryData } from './binaryDataCodec.js' import { prodLog } from '../utils/logger.js' import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' @@ -187,8 +185,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Write-through cache for read-after-write consistency // Extended lifetime - persists until explicit flush() call - // Guarantees that immediately after writeObjectToBranch(), readWithInheritance() returns the data - // Cache key: resolved branchPath (includes branch scope for COW isolation) + // Guarantees that immediately after writeCanonicalObject(), readCanonicalObject() returns the data + // Cache key: storage-root-relative object path // Cache lifetime: write start → flush() call (provides safety net for batch operations) // Memory footprint: Bounded by batch size (typically <1000 items during imports) private writeCache = new Map() @@ -203,12 +201,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.writeCache.clear() } - // COW (Copy-on-Write) support - public refManager?: RefManager + /** + * Content-addressed blob store backing VFS file content (deduplicated, + * zstd-compressed, SHA-256 addressed). Lazily created by + * {@link initializeBlobStorage}; lives under the `_cas/` storage area. + */ public blobStorage?: BlobStorage - public commitLog?: CommitLog - public currentBranch: string = 'main' - // Removed cowEnabled flag - COW is ALWAYS enabled (mandatory, cannot be disabled) // Type-first indexing support // Built into all storage adapters for billion-scale efficiency @@ -423,13 +421,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Invalidate GraphAdjacencyIndex - * Call this when switching branches or clearing data to force re-creation + * Call this when clearing data to force re-creation * The next getGraphIndex() call will create a fresh instance and rebuild * @public */ public invalidateGraphIndex(): void { if (this.graphIndex) { - prodLog.info('[BaseStorage] Invalidating GraphAdjacencyIndex for branch switch/clear') + prodLog.info('[BaseStorage] Invalidating GraphAdjacencyIndex for clear()') // Stop any pending operations if (typeof (this.graphIndex as any).stopAutoFlush === 'function') { (this.graphIndex as any).stopAutoFlush() @@ -552,58 +550,34 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Lightweight COW enablement - just enables branch-scoped paths - * Called during init() to ensure all data is stored with branch prefixes from the start - * RefManager/BlobStorage/CommitLog are lazy-initialized on first fork() - * @param branch - Branch name to use (default: 'main') + * @description Initialize the content-addressed blob store (idempotent). + * Creates a {@link BlobStorage} over this adapter's object primitives, + * rooted at the `_cas/` storage area. The blob store backs VFS file + * content: deduplicated, SHA-256 addressed, zstd-compressed where it pays. * - * COW is always enabled - this method now just sets the branch name (idempotent) + * Called automatically during `brain.init()` (before the VFS is built) and + * again after `clear()` re-creates the storage area. + * + * @returns Promise that resolves when the blob store is ready. */ - public enableCOWLightweight(branch: string = 'main'): void { - this.currentBranch = branch - // RefManager/BlobStorage/CommitLog remain undefined until first fork() - } - - /** - * Initialize COW (Copy-on-Write) support - * Creates RefManager and BlobStorage for instant fork() capability - * - * Now called automatically by storageFactory (zero-config) - * - * @param options - COW initialization options - * @param options.branch - Initial branch name (default: 'main') - * @param options.enableCompression - Enable zstd compression for blobs (default: true) - * @returns Promise that resolves when COW is initialized - */ - public async initializeCOW(options?: { - branch?: string - enableCompression?: boolean - }): Promise { - // COW is ALWAYS enabled - idempotent initialization only - // Removed marker file check (cowEnabled flag removed, COW is mandatory) - - // Check if RefManager already initialized (idempotent) - if (this.refManager && this.blobStorage && this.commitLog) { + public async initializeBlobStorage(): Promise { + if (this.blobStorage) { return } - // Set current branch if provided - if (options?.branch) { - this.currentBranch = options.branch - } - - // Create COWStorageAdapter bridge - // This adapts BaseStorage's methods to the simple key-value interface - const cowAdapter: COWStorageAdapter = { + // Key-value bridge: adapts this adapter's object primitives to the + // BlobStoreAdapter interface. Key naming is an explicit type contract: + // - 'blob-meta:' → JSON (BlobStorage metadata) + // - 'blob:' → binary (possibly compressed blob bytes) + const casAdapter: BlobStoreAdapter = { get: async (key: string): Promise => { try { - const data = await this.readObjectFromPath(`_cow/${key}`) + const data = await this.readObjectFromPath(`_cas/${key}`) if (data === null) { return undefined } - // Use shared binaryDataCodec utility (single source of truth) // Unwraps binary data stored as {_binary: true, data: "base64..."} - // Fixes "Blob integrity check failed" - hash must be calculated on original content + // — hash verification must run on the original content bytes. return unwrapBinaryData(data) } catch (error) { return undefined @@ -611,32 +585,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { }, put: async (key: string, data: Buffer): Promise => { - // PERMANENT FIX: Use key naming convention (explicit type contract) - // NO GUESSING - key format explicitly declares data type: - // - // JSON keys (metadata and refs): - // - 'ref:*' → JSON (RefManager: refs, HEAD, branches) - // - 'blob-meta:hash' → JSON (BlobStorage: blob metadata) - // - 'commit-meta:hash'→ JSON (BlobStorage: commit metadata) - // - 'tree-meta:hash' → JSON (BlobStorage: tree metadata) - // - // Binary keys (blob data): - // - 'blob:hash' → Binary (BlobStorage: compressed/raw blob data) - // - 'commit:hash' → Binary (BlobStorage: commit object data) - // - 'tree:hash' → Binary (BlobStorage: tree object data) - // - // This eliminates the fragile JSON.parse() guessing that caused blob integrity - // failures when compressed data accidentally parsed as valid JSON. - const obj = key.includes('-meta:') || key.startsWith('ref:') - ? JSON.parse(data.toString()) // Metadata/refs: ALWAYS JSON.stringify'd - : { _binary: true, data: data.toString('base64') } // Blobs: ALWAYS binary (possibly compressed) + // Metadata keys are always JSON; blob keys are always binary. The key + // format decides — no content sniffing (JSON.parse guessing corrupted + // compressed blobs that happened to parse as JSON). + const obj = key.includes('-meta:') + ? JSON.parse(data.toString()) + : { _binary: true, data: data.toString('base64') } - await this.writeObjectToPath(`_cow/${key}`, obj) + await this.writeObjectToPath(`_cas/${key}`, obj) }, delete: async (key: string): Promise => { try { - await this.deleteObjectFromPath(`_cow/${key}`) + await this.deleteObjectFromPath(`_cas/${key}`) } catch (error) { // Ignore if doesn't exist } @@ -644,241 +605,85 @@ export abstract class BaseStorage extends BaseStorageAdapter { list: async (prefix: string): Promise => { try { - // Handle file prefixes, not just directory paths - // Refs are stored as files like: _cow/ref:refs/heads/main - // So list('ref:') should find all files starting with '_cow/ref:' - - // List the _cow directory and filter by prefix - const allPaths = await this.listObjectsUnderPath('_cow/') - const filteredPaths = allPaths.filter(p => { - // Remove _cow/ prefix to get the key - const key = p.replace(/^_cow\//, '') - return key.startsWith(prefix) - }) - - // Remove _cow/ prefix and return relative keys - return filteredPaths.map(p => p.replace(/^_cow\//, '')) + // Keys are stored as files like `_cas/blob:`, so listing is + // prefix filtering over the `_cas/` area with the area prefix + // stripped from the returned keys. + const allPaths = await this.listObjectsUnderPath('_cas/') + return allPaths + .map(p => p.replace(/^_cas\//, '')) + .filter(key => key.startsWith(prefix)) } catch (error: any) { - // If _cow directory doesn't exist yet, return empty array + // `_cas/` doesn't exist yet — empty store. return [] } } } - // Initialize RefManager - this.refManager = new RefManager(cowAdapter) - - // Initialize BlobStorage - this.blobStorage = new BlobStorage(cowAdapter, { - enableCompression: options?.enableCompression !== false - }) - - // Initialize CommitLog - this.commitLog = new CommitLog(this.blobStorage, this.refManager) - - // Check if main branch exists, create if not - const mainRef = await this.refManager.getRef('main') - if (!mainRef) { - // Create initial commit with empty tree - // Use NULL_HASH constant instead of hardcoded string - const { NULL_HASH } = await import('./cow/constants.js') - const emptyTreeHash = NULL_HASH - - // Import CommitBuilder - const { CommitBuilder } = await import('./cow/CommitObject.js') - - // Create initial commit object - const initialCommitHash = await CommitBuilder.create(this.blobStorage) - .tree(emptyTreeHash) - .parent(null) - .message('Initial commit') - .author('system') - .timestamp(Date.now()) - .build() - - // Create main branch pointing to initial commit - await this.refManager.createBranch('main', initialCommitHash, { - description: 'Initial branch', - author: 'system' - }) - } - - // Set HEAD to current branch - const currentRef = await this.refManager.getRef(this.currentBranch) - if (currentRef) { - await this.refManager.setHead(this.currentBranch) - } else { - // Branch doesn't exist, create it from main - const mainCommit = await this.refManager.resolveRef('main') - if (mainCommit) { - await this.refManager.createBranch(this.currentBranch, mainCommit, { - description: `Branch created from main`, - author: 'system' - }) - await this.refManager.setHead(this.currentBranch) - } - } - - // COW is always enabled - no flag to set + this.blobStorage = new BlobStorage(casAdapter) } /** - * Resolve branch-scoped path for COW isolation - * @protected - Available to subclasses for COW implementation + * @description Write a canonical object (write-cache coherent). The object + * lands in the write-through cache before the asynchronous write starts, so + * {@link readCanonicalObject} returns it immediately — read-after-write + * consistency within the process. The cache persists until `flush()`. + * + * @param path - Storage-root-relative object path. + * @param data - JSON-serializable object to write. + * @protected */ - protected resolveBranchPath(basePath: string, branch?: string): string { - // CRITICAL FIX: COW metadata (_cow/*) must NEVER be branch-scoped - // Refs, commits, and blobs are global metadata with their own internal branching. - // Branch-scoping COW paths causes fork() to write refs to wrong locations, - // leading to "Branch does not exist" errors on checkout. - if (basePath.startsWith('_cow/')) { - return basePath // COW metadata is global across all branches - } + protected async writeCanonicalObject(path: string, data: any): Promise { + this.writeCache.set(path, data) - // COW is always enabled - always use branch-scoped paths - const targetBranch = branch || this.currentBranch || 'main' - - // Branch-scoped path: branches// - return `branches/${targetBranch}/${basePath}` - } - - /** - * Write object to branch-specific path (COW layer) - * @protected - Available to subclasses for COW implementation - */ - protected async writeObjectToBranch(path: string, data: any, branch?: string): Promise { - const branchPath = this.resolveBranchPath(path, branch) - - // Add to write cache BEFORE async write (guarantees read-after-write consistency) - // Cache persists until flush() is called (extended lifetime for batch operations) - // This ensures readWithInheritance() returns data immediately, fixing "Source entity not found" bug - this.writeCache.set(branchPath, data) - - // Use write buffer if available (cloud adapters), otherwise write directly (filesystem) + // Use write buffer if available, otherwise write directly (filesystem) if (this.metadataWriteBuffer) { - await this.metadataWriteBuffer.write(branchPath, data) + await this.metadataWriteBuffer.write(path, data) } else { - await this.writeObjectToPath(branchPath, data) + await this.writeObjectToPath(path, data) } - // Cache is NOT cleared here anymore - persists until flush() + // Cache is NOT cleared here - persists until flush() // This provides a safety net for immediate queries after batch writes } /** - * Read object with inheritance from parent branches (COW layer) - * Tries current branch first, then walks commit history - * @protected - Available to subclasses for COW implementation + * @description Read a canonical object: write cache first (synchronous, + * guarantees read-after-write consistency), then the adapter. * - * COW is always enabled - always use branch-scoped paths with inheritance + * @param path - Storage-root-relative object path. + * @returns The object, or `null` when absent. + * @protected */ - protected async readWithInheritance(path: string, branch?: string): Promise { - const targetBranch = branch || this.currentBranch || 'main' - const branchPath = this.resolveBranchPath(path, targetBranch) - - // Check write cache FIRST (synchronous, instant) - // This guarantees read-after-write consistency within the same process - // Fixes bug: brain.add() → brain.relate() → "Source entity not found" - const cachedData = this.writeCache.get(branchPath) + protected async readCanonicalObject(path: string): Promise { + const cachedData = this.writeCache.get(path) if (cachedData !== undefined) { return cachedData } - // Try current branch first - let data = await this.readObjectFromPath(branchPath) - - if (data !== null) { - return data // Found in current branch - } - - // Not in branch, check if we're on main (no inheritance needed) - if (targetBranch === 'main') { - return null - } - - // Not in branch, walk commit history to find in parent - if (this.refManager && this.commitLog) { - try { - const commitHash = await this.refManager.resolveRef(targetBranch) - if (commitHash) { - // Walk parent commits until we find the data - for await (const commit of this.commitLog.walk(commitHash)) { - // Try reading from parent's branch path - const parentBranch = commit.metadata?.branch || 'main' - if (parentBranch === targetBranch) continue // Skip self - - const parentPath = this.resolveBranchPath(path, parentBranch) - data = await this.readObjectFromPath(parentPath) - if (data !== null) { - return data // Found in ancestor - } - } - } - } catch (error) { - // Commit walk failed, fall back to main - const mainPath = this.resolveBranchPath(path, 'main') - return this.readObjectFromPath(mainPath) - } - } - - // Last fallback: try main branch - const mainPath = this.resolveBranchPath(path, 'main') - return this.readObjectFromPath(mainPath) + return this.readObjectFromPath(path) } /** - * Delete object from branch-specific path (COW layer) - * @protected - Available to subclasses for COW implementation - */ - protected async deleteObjectFromBranch(path: string, branch?: string): Promise { - const branchPath = this.resolveBranchPath(path, branch) - - // Remove from write cache immediately (before async delete) - // Ensures subsequent reads don't return stale cached data - this.writeCache.delete(branchPath) - - return this.deleteObjectFromPath(branchPath) - } - - /** - * List objects under path in branch (COW layer) - * @protected - Available to subclasses for COW implementation - */ - protected async listObjectsInBranch(prefix: string, branch?: string): Promise { - const branchPrefix = this.resolveBranchPath(prefix, branch) - const paths = await this.listObjectsUnderPath(branchPrefix) - - // Remove branch prefix from results - const targetBranch = branch || this.currentBranch || 'main' - const prefixToRemove = `branches/${targetBranch}/` - - return paths.map(p => p.startsWith(prefixToRemove) ? p.substring(prefixToRemove.length) : p) - } - - /** - * List objects with inheritance - * Lists objects from current branch AND main branch, returns unique paths - * This enables fork to see parent's data in pagination operations + * @description Delete a canonical object, evicting the write-cache entry + * first so subsequent reads never return stale cached data. * - * Simplified approach: All branches inherit from main - * - * COW is always enabled - always use inheritance + * @param path - Storage-root-relative object path. + * @protected */ - protected async listObjectsWithInheritance(prefix: string, branch?: string): Promise { - const targetBranch = branch || this.currentBranch || 'main' + protected async deleteCanonicalObject(path: string): Promise { + this.writeCache.delete(path) + return this.deleteObjectFromPath(path) + } - // Collect paths from current branch - const pathsSet = new Set() - const currentBranchPaths = await this.listObjectsInBranch(prefix, targetBranch) - currentBranchPaths.forEach(p => pathsSet.add(p)) - - // If not on main, also list from main (all branches inherit from main) - if (targetBranch !== 'main') { - const mainPaths = await this.listObjectsInBranch(prefix, 'main') - mainPaths.forEach(p => pathsSet.add(p)) - } - - return Array.from(pathsSet) + /** + * @description List canonical objects under a storage-root-relative prefix. + * + * @param prefix - Storage-root-relative directory prefix. + * @returns Storage-root-relative paths of the objects found. + * @protected + */ + protected async listCanonicalObjects(prefix: string): Promise { + return this.listObjectsUnderPath(prefix) } // ============================================================================ @@ -888,11 +693,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { // an adapter — see the `GenerationStorage` contract in src/db/types.ts. // // Raw-object methods operate on storage-root-relative paths and deliberately - // bypass branch scoping and the write cache: the record layer owns the - // `_system/` + `_generations/` areas outright and its files must never be - // shadowed by branch resolution. Entity-raw methods, by contrast, go through - // the branch-scoped, write-cache-coherent helpers so before-images capture - // exactly the bytes the live read paths see. + // bypass the write cache: the record layer owns the `_system/` + + // `_generations/` areas outright. Entity-raw methods, by contrast, go + // through the write-cache-coherent canonical helpers so before-images + // capture exactly the bytes the live read paths see. // ============================================================================ /** @@ -923,8 +727,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Read a raw object at a storage-root-relative path. Bypasses branch - * scoping and the write cache (record-layer files are written through + * Read a raw object at a storage-root-relative path. Bypasses the write + * cache (record-layer files are written through * {@link BaseStorage.writeRawObject} only). * * @param path - Storage-root-relative object path (e.g. `_system/manifest.json`). @@ -1001,8 +805,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Read an entity's raw stored objects — the exact bytes at its canonical - * metadata + vector paths (branch-scoped, write-cache coherent). Used by - * the generation store to capture before-images. + * metadata + vector paths (write-cache coherent). Used by the generation + * store to capture before-images. * * @param id - The entity id. * @returns The raw stored metadata and vector objects (`null` per part when @@ -1011,8 +815,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }> { await this.ensureInitialized() const [metadata, vector] = await Promise.all([ - this.readWithInheritance(getNounMetadataPath(id)), - this.readWithInheritance(getNounVectorPath(id)) + this.readCanonicalObject(getNounMetadataPath(id)), + this.readCanonicalObject(getNounVectorPath(id)) ]) return { metadata: metadata ?? null, vector: vector ?? null } } @@ -1033,14 +837,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async writeNounRaw(id: string, record: { metadata: any | null; vector: any | null }): Promise { await this.ensureInitialized() if (record.metadata === null) { - await this.deleteObjectFromBranch(getNounMetadataPath(id)) + await this.deleteCanonicalObject(getNounMetadataPath(id)) } else { - await this.writeObjectToBranch(getNounMetadataPath(id), record.metadata) + await this.writeCanonicalObject(getNounMetadataPath(id), record.metadata) } if (record.vector === null) { - await this.deleteObjectFromBranch(getNounVectorPath(id)) + await this.deleteCanonicalObject(getNounVectorPath(id)) } else { - await this.writeObjectToBranch(getNounVectorPath(id), record.vector) + await this.writeCanonicalObject(getNounVectorPath(id), record.vector) } } @@ -1055,8 +859,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async readVerbRaw(id: string): Promise<{ metadata: any | null; vector: any | null }> { await this.ensureInitialized() const [metadata, vector] = await Promise.all([ - this.readWithInheritance(getVerbMetadataPath(id)), - this.readWithInheritance(getVerbVectorPath(id)) + this.readCanonicalObject(getVerbMetadataPath(id)), + this.readCanonicalObject(getVerbVectorPath(id)) ]) return { metadata: metadata ?? null, vector: vector ?? null } } @@ -1071,14 +875,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async writeVerbRaw(id: string, record: { metadata: any | null; vector: any | null }): Promise { await this.ensureInitialized() if (record.metadata === null) { - await this.deleteObjectFromBranch(getVerbMetadataPath(id)) + await this.deleteCanonicalObject(getVerbMetadataPath(id)) } else { - await this.writeObjectToBranch(getVerbMetadataPath(id), record.metadata) + await this.writeCanonicalObject(getVerbMetadataPath(id), record.metadata) } if (record.vector === null) { - await this.deleteObjectFromBranch(getVerbVectorPath(id)) + await this.deleteCanonicalObject(getVerbVectorPath(id)) } else { - await this.writeObjectToBranch(getVerbVectorPath(id), record.vector) + await this.writeCanonicalObject(getVerbVectorPath(id), record.vector) } } @@ -1140,6 +944,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.statisticsCache = null this.statisticsModified = false this.invalidateGraphIndex() + // Re-create the blob store: a restore replaced the `_cas/` area wholesale, + // and the old instance's LRU cache could serve blobs the restored store no + // longer contains. + if (this.blobStorage) { + this.blobStorage = undefined + await this.initializeBlobStorage() + } await this.loadTypeStatistics() await this.loadSubtypeStatistics() await this.loadVerbSubtypeStatistics() @@ -1359,8 +1170,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Batch read vectors and metadata in parallel const [vectorResults, metadataResults] = await Promise.all([ - this.readBatchWithInheritance(vectorPaths.map(p => p.path)), - this.readBatchWithInheritance(metadataPaths.map(p => p.path)) + this.readCanonicalObjectBatch(vectorPaths.map(p => p.path)), + this.readCanonicalObjectBatch(metadataPaths.map(p => p.path)) ]) // Combine vectors + metadata into HNSWVerbWithMetadata @@ -1725,14 +1536,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/nouns/${shardHex}` try { - const nounFiles = await this.listObjectsInBranch(shardDir) + const nounFiles = await this.listCanonicalObjects(shardDir) for (const nounPath of nounFiles) { if (collectedNouns.length >= targetCount) break if (!nounPath.includes('/vectors.json')) continue try { - const noun = await this.readWithInheritance(nounPath) + const noun = await this.readCanonicalObject(nounPath) if (noun) { const deserialized = this.deserializeNoun(noun) const metadata = await this.getNounMetadata(deserialized.id) @@ -1862,14 +1673,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/verbs/${shardHex}` try { - const verbFiles = await this.listObjectsInBranch(shardDir) + const verbFiles = await this.listCanonicalObjects(shardDir) for (const verbPath of verbFiles) { if (collectedVerbs.length >= targetCount) break if (!verbPath.includes('/vectors.json')) continue try { - const rawVerb = await this.readWithInheritance(verbPath) + const rawVerb = await this.readCanonicalObject(verbPath) if (!rawVerb) continue // Deserialize connections Map from JSON storage format @@ -2418,11 +2229,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ public abstract clear(): Promise - /** - * Removed checkClearMarker() and createClearMarker() abstract methods - * COW is now always enabled - marker files are no longer used - */ - /** * Get information about storage usage and capacity * This method should be implemented by each specific adapter @@ -2482,7 +2288,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async saveMetadata(id: string, metadata: NounMetadata): Promise { await this.ensureInitialized() const keyInfo = this.analyzeKey(id, 'system') - return this.writeObjectToBranch(keyInfo.fullPath, metadata) + return this.writeCanonicalObject(keyInfo.fullPath, metadata) } /** @@ -2494,13 +2300,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { const keyInfo = this.analyzeKey(id, 'system') // Try new distributed path first - const data = await this.readWithInheritance(keyInfo.fullPath) + const data = await this.readCanonicalObject(keyInfo.fullPath) if (data !== null) return data // Backward compat: if key was distributed, fall back to legacy flat path if (keyInfo.shardId !== null) { const legacyPath = `${SYSTEM_DIR}/${id}.json` - return this.readWithInheritance(legacyPath) + return this.readCanonicalObject(legacyPath) } return null @@ -2533,11 +2339,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getNounMetadataPath(id) // Determine if this is a new entity by checking if metadata already exists - const existingMetadata = await this.readWithInheritance(path) + const existingMetadata = await this.readCanonicalObject(path) const isNew = !existingMetadata - // Save the metadata (COW-aware - writes to branch-specific path) - await this.writeObjectToBranch(path, metadata) + // Save the metadata (write-cache coherent canonical write) + await this.writeCanonicalObject(path, metadata) // Record the id→type mapping so `saveNoun_internal()` (which receives only // an HNSWNoun and has no metadata access in its signature) can attribute @@ -2643,7 +2449,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Clean, simple, O(1) lookup - no type needed! const path = getNounMetadataPath(id) - return this.readWithInheritance(path) + return this.readCanonicalObject(path) } /** @@ -2691,7 +2497,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { })) // Batch read all paths (uses adapter's native batch API or parallel fallback) - const batchResults = await this.readBatchWithInheritance(pathsToFetch.map(p => p.path)) + const batchResults = await this.readCanonicalObjectBatch(pathsToFetch.map(p => p.path)) // Map results back to IDs for (const { path, id } of pathsToFetch) { @@ -2741,8 +2547,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Batch read vectors and metadata in parallel const [vectorResults, metadataResults] = await Promise.all([ - this.readBatchWithInheritance(vectorPaths.map(p => p.path)), - this.readBatchWithInheritance(metadataPaths.map(p => p.path)) + this.readCanonicalObjectBatch(vectorPaths.map(p => p.path)), + this.readCanonicalObjectBatch(metadataPaths.map(p => p.path)) ]) // Combine vectors + metadata into HNSWNounWithMetadata @@ -2784,48 +2590,35 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Batch read multiple storage paths with COW inheritance support + * Batch read multiple canonical storage paths * * Core batching primitive that all batch operations build upon. - * Handles write cache, branch inheritance, and adapter-specific batching. + * Handles the write cache and adapter-specific batching. * * **Performance**: - * - Uses adapter's native batch API when available (GCS, S3, Azure) + * - Uses adapter's native batch API when available * - Falls back to parallel reads for non-batch adapters * - Respects rate limits via StorageBatchConfig * - * @param paths Array of storage paths to read - * @param branch Optional branch (defaults to current branch) + * @param paths Array of storage-root-relative paths to read * @returns Map of path → data (only successful reads included) * * @protected - Available to subclasses and batch operations */ - protected async readBatchWithInheritance( - paths: string[], - branch?: string - ): Promise> { + protected async readCanonicalObjectBatch(paths: string[]): Promise> { if (paths.length === 0) return new Map() - const targetBranch = branch || this.currentBranch || 'main' const results = new Map() - // Resolve all paths to branch-specific paths - const branchPaths = paths.map(path => ({ - original: path, - resolved: this.resolveBranchPath(path, targetBranch) - })) - // Step 1: Check write cache first (synchronous, instant) const pathsToFetch: string[] = [] - const pathMapping = new Map() // resolved → original - for (const { original, resolved } of branchPaths) { - const cachedData = this.writeCache.get(resolved) + for (const path of paths) { + const cachedData = this.writeCache.get(path) if (cachedData !== undefined) { - results.set(original, cachedData) + results.set(path, cachedData) } else { - pathsToFetch.push(resolved) - pathMapping.set(resolved, original) + pathsToFetch.push(path) } } @@ -2834,42 +2627,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Step 2: Batch read from adapter - // Check if adapter supports native batch operations const batchData = await this.readBatchFromAdapter(pathsToFetch) - // Step 3: Process results and handle inheritance for missing items - const missingPaths: string[] = [] - - for (const [resolvedPath, data] of batchData.entries()) { - const originalPath = pathMapping.get(resolvedPath) - if (originalPath && data !== null) { - results.set(originalPath, data) - } - } - - // Identify paths that weren't found - for (const resolvedPath of pathsToFetch) { - if (!batchData.has(resolvedPath) || batchData.get(resolvedPath) === null) { - missingPaths.push(pathMapping.get(resolvedPath)!) - } - } - - // Step 4: Handle COW inheritance for missing items (if not on main branch). - // Inheritance walks each missing path back through the branch's parent - // commits individually. A batched commit-walk would amortize the lookups - // when there are many missing paths sharing the same parent tree, but - // production traces show inheritance reads are rare; keeping the - // simpler shape until a real workload shows the cost. - if (targetBranch !== 'main' && missingPaths.length > 0) { - for (const originalPath of missingPaths) { - try { - const data = await this.readWithInheritance(originalPath, targetBranch) - if (data !== null) { - results.set(originalPath, data) - } - } catch (error) { - // Skip failed reads (they won't be in results map) - } + for (const [path, data] of batchData.entries()) { + if (data !== null) { + results.set(path, data) } } @@ -2965,7 +2727,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Direct O(1) delete with ID-first path const path = getNounMetadataPath(id) - await this.deleteObjectFromBranch(path) + await this.deleteCanonicalObject(path) // Prune the id→type cache so a future re-add of the same id (e.g. churn // during tests) doesn't see a stale type. Lookup the prior type and @@ -3022,7 +2784,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (!verbType) { // Backward compatibility: fallback to old path if no verb type const keyInfo = this.analyzeKey(id, 'verb-metadata') - await this.writeObjectToBranch(keyInfo.fullPath, metadata) + await this.writeCanonicalObject(keyInfo.fullPath, metadata) // 8.0 MVCC: still an entity-visible write — advance the watermark. this.generationBumpHook?.() return @@ -3032,11 +2794,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getVerbMetadataPath(id) // Determine if this is a new verb by checking if metadata already exists - const existingMetadata = await this.readWithInheritance(path) + const existingMetadata = await this.readCanonicalObject(path) const isNew = !existingMetadata - // Save the metadata (COW-aware - writes to branch-specific path) - await this.writeObjectToBranch(path, metadata) + // Save the metadata (write-cache coherent canonical write) + await this.writeCanonicalObject(path, metadata) // Cache verb type for faster lookups @@ -3096,7 +2858,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getVerbMetadataPath(id) try { - const metadata = await this.readWithInheritance(path) + const metadata = await this.readCanonicalObject(path) return metadata || null } catch (error) { // Entity not found @@ -3112,7 +2874,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Direct O(1) delete with ID-first path const path = getVerbMetadataPath(id) - await this.deleteObjectFromBranch(path) + await this.deleteCanonicalObject(path) // Symmetric verb subtype decrement const priorEntry = this.verbSubtypeByIdCache.get(id) @@ -3322,11 +3084,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardHex = shard.toString(16).padStart(2, '0') const shardDir = `entities/nouns/${shardHex}` try { - const paths = await this.listObjectsInBranch(shardDir) + const paths = await this.listCanonicalObjects(shardDir) for (const path of paths) { if (!path.includes('/metadata.json')) continue try { - const metadata = await this.readWithInheritance(path) + const metadata = await this.readCanonicalObject(path) if (metadata && metadata.noun && typeof metadata.subtype === 'string' && metadata.subtype.length > 0) { this.incrementSubtypeCount(metadata.noun as NounType, metadata.subtype) // Path is `entities/nouns///metadata.json` — extract id segment. @@ -3439,11 +3201,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardHex = shard.toString(16).padStart(2, '0') const shardDir = `entities/verbs/${shardHex}` try { - const paths = await this.listObjectsInBranch(shardDir) + const paths = await this.listCanonicalObjects(shardDir) for (const path of paths) { if (!path.includes('/metadata.json')) continue try { - const metadata = await this.readWithInheritance(path) + const metadata = await this.readCanonicalObject(path) if (metadata && metadata.verb && typeof metadata.subtype === 'string' && metadata.subtype.length > 0) { const verb = metadata.verb as VerbType const subtype = metadata.subtype as string @@ -3547,13 +3309,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/nouns/${shardHex}` try { - const paths = await this.listObjectsInBranch(shardDir) + const paths = await this.listCanonicalObjects(shardDir) for (const path of paths) { if (!path.includes('/metadata.json')) continue try { - const metadata = await this.readWithInheritance(path) + const metadata = await this.readCanonicalObject(path) if (metadata && metadata.noun) { const typeIndex = TypeUtils.getNounIndex(metadata.noun) if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { @@ -3575,13 +3337,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/verbs/${shardHex}` try { - const paths = await this.listObjectsInBranch(shardDir) + const paths = await this.listCanonicalObjects(shardDir) for (const path of paths) { if (!path.includes('/metadata.json')) continue try { - const metadata = await this.readWithInheritance(path) + const metadata = await this.readCanonicalObject(path) if (metadata && metadata.verb) { const typeIndex = TypeUtils.getVerbIndex(metadata.verb) if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { @@ -3650,7 +3412,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (cached) return cached try { const metadataPath = getNounMetadataPath(id) - const metadata = await this.readWithInheritance(metadataPath) + const metadata = await this.readCanonicalObject(metadataPath) if (metadata && (metadata as NounMetadata).noun) { const type = (metadata as NounMetadata).noun as NounType // Warm the cache so subsequent sync calls hit fast. @@ -3766,8 +3528,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { const typeIndex = TypeUtils.getNounIndex(type) this.nounCountsByType[typeIndex]++ - // COW-aware write: Use COW helper for branch isolation - await this.writeObjectToBranch(path, noun) + // Write-cache coherent canonical write + await this.writeCanonicalObject(path, noun) // Periodically save statistics // Also save on first noun of each type to ensure low-count types are tracked @@ -3786,8 +3548,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getNounVectorPath(id) try { - // COW-aware read: Use COW helper for branch isolation - const noun = await this.readWithInheritance(path) + // Write-cache coherent canonical read + const noun = await this.readCanonicalObject(path) if (noun) { // Deserialize connections Map from JSON storage format return this.deserializeNoun(noun) @@ -3815,13 +3577,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/nouns/${shardHex}` try { - const nounFiles = await this.listObjectsInBranch(shardDir) + const nounFiles = await this.listCanonicalObjects(shardDir) for (const nounPath of nounFiles) { if (!nounPath.includes('/vectors.json')) continue try { - const noun = await this.readWithInheritance(nounPath) + const noun = await this.readCanonicalObject(nounPath) if (noun) { const deserialized = this.deserializeNoun(noun) @@ -3849,7 +3611,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected async deleteNoun_internal(id: string): Promise { // Direct O(1) delete with ID-first path const path = getNounVectorPath(id) - await this.deleteObjectFromBranch(path) + await this.deleteCanonicalObject(path) // Note: Type-specific counts will be decremented via metadata tracking // The real type is in metadata, accessible if needed via getNounMetadata(id) @@ -3869,8 +3631,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { const typeIndex = TypeUtils.getVerbIndex(type) this.verbCountsByType[typeIndex]++ - // COW-aware write: Use COW helper for branch isolation - await this.writeObjectToBranch(path, verb) + // Write-cache coherent canonical write + await this.writeCanonicalObject(path, verb) // GraphAdjacencyIndex updates are now handled EXCLUSIVELY by Brainy.relate() // via AddToGraphIndexOperation in the transaction system. This provides: @@ -3897,8 +3659,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { const path = getVerbVectorPath(id) try { - // COW-aware read: Use COW helper for branch isolation - const verb = await this.readWithInheritance(path) + // Write-cache coherent canonical read + const verb = await this.readCanonicalObject(path) if (verb) { // Deserialize connections Map from JSON storage format return this.deserializeVerb(verb) @@ -3945,8 +3707,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { const metadataPaths = verbIds.map(id => getVerbMetadataPath(id)) const [verbsMap, metadataMap] = await Promise.all([ - this.readBatchWithInheritance(verbPaths), - this.readBatchWithInheritance(metadataPaths) + this.readCanonicalObjectBatch(verbPaths), + this.readCanonicalObjectBatch(metadataPaths) ]) const results: HNSWVerbWithMetadata[] = [] @@ -3998,14 +3760,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/verbs/${shardHex}` try { - const verbFiles = await this.listObjectsInBranch(shardDir) + const verbFiles = await this.listCanonicalObjects(shardDir) shardsScanned++ for (const verbPath of verbFiles) { if (!verbPath.includes('/vectors.json')) continue try { - const rawVerb = await this.readWithInheritance(verbPath) + const rawVerb = await this.readCanonicalObject(verbPath) if (!rawVerb) continue verbsFound++ @@ -4015,7 +3777,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (verb.sourceId === sourceId) { const metadataPath = getVerbMetadataPath(verb.id) - const metadata = await this.readWithInheritance(metadataPath) + const metadata = await this.readCanonicalObject(metadataPath) results.push({ ...verb, @@ -4102,7 +3864,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { // List all verb files in this shard - const verbFiles = await this.listObjectsInBranch(shardDir) + const verbFiles = await this.listCanonicalObjects(shardDir) // Build paths for batch read const verbPaths: string[] = [] @@ -4123,8 +3885,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Batch read all verb files for this shard - const verbDataMap = await this.readBatchWithInheritance(verbPaths) - const metadataMap = await this.readBatchWithInheritance(metadataPaths) + const verbDataMap = await this.readCanonicalObjectBatch(verbPaths) + const metadataMap = await this.readCanonicalObjectBatch(metadataPaths) // Process results for (const [verbPath, rawVerbData] of verbDataMap.entries()) { @@ -4172,7 +3934,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get verbs by target (COW-aware implementation) + * Get verbs by target * Reverted to fix circular dependency deadlock * Fixed to directly list verb files instead of directories */ @@ -4233,13 +3995,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/verbs/${shardHex}` try { - const verbFiles = await this.listObjectsInBranch(shardDir) + const verbFiles = await this.listCanonicalObjects(shardDir) for (const verbPath of verbFiles) { if (!verbPath.includes('/vectors.json')) continue try { - const rawVerb = await this.readWithInheritance(verbPath) + const rawVerb = await this.readCanonicalObject(verbPath) if (!rawVerb) continue // CRITICAL - Deserialize connections Map from JSON storage format @@ -4247,7 +4009,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (verb.targetId === targetId) { const metadataPath = getVerbMetadataPath(verb.id) - const metadata = await this.readWithInheritance(metadataPath) + const metadata = await this.readCanonicalObject(metadataPath) results.push({ ...verb, @@ -4289,13 +4051,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/verbs/${shardHex}` try { - const verbFiles = await this.listObjectsInBranch(shardDir) + const verbFiles = await this.listCanonicalObjects(shardDir) for (const verbPath of verbFiles) { if (!verbPath.includes('/vectors.json')) continue try { - const rawVerb = await this.readWithInheritance(verbPath) + const rawVerb = await this.readCanonicalObject(verbPath) if (!rawVerb) continue // Deserialize connections Map from JSON storage format @@ -4348,7 +4110,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected async deleteVerb_internal(id: string): Promise { // Direct O(1) delete with ID-first path const path = getVerbVectorPath(id) - await this.deleteObjectFromBranch(path) + await this.deleteCanonicalObject(path) // Note: Type-specific counts will be decremented via metadata tracking // The real type is in metadata, accessible if needed via getVerbMetadata(id) diff --git a/src/storage/binaryDataCodec.ts b/src/storage/binaryDataCodec.ts new file mode 100644 index 00000000..65e8e4da --- /dev/null +++ b/src/storage/binaryDataCodec.ts @@ -0,0 +1,77 @@ +/** + * @module storage/binaryDataCodec + * @description Single source of truth for wrapping/unwrapping binary payloads + * stored in JSON-based object storage. Binary bytes (compressed blobs, raw + * buffers) are persisted as `{ _binary: true, data: "" }` so they can + * travel through adapters whose object primitive is JSON. `unwrapBinaryData` + * reverses that wrapping — and MUST be used everywhere bytes come back out, + * because content hashes are computed over the original bytes, not the + * wrapper. + * + * Used by `BaseStorage`'s blob-store bridge and by `BlobStorage`'s + * defense-in-depth verification path. + */ + +/** + * @description Wrapped binary data format used when storing binary data in + * JSON-based storage. + */ +export interface WrappedBinaryData { + _binary: true + data: string // base64-encoded +} + +/** + * @description Type guard for the wrapped binary format. + * @param data - Candidate value. + * @returns True when `data` is a `{ _binary: true, data: string }` wrapper. + */ +export function isWrappedBinary(data: any): data is WrappedBinaryData { + return ( + typeof data === 'object' && + data !== null && + data._binary === true && + typeof data.data === 'string' + ) +} + +/** + * @description Unwrap binary data from its JSON wrapper. + * + * Handles: + * - `Buffer` → `Buffer` (pass-through) + * - `{ _binary: true, data: "" }` → `Buffer` (unwrap) + * - Plain object → `Buffer` (JSON stringify — defensive) + * - String → `Buffer` + * + * @param data - Data to unwrap (Buffer, wrapped object, plain object, or string). + * @returns The unwrapped bytes. + * @throws Error when the value cannot be interpreted as binary data. + */ +export function unwrapBinaryData(data: any): Buffer { + // Case 1: Already a Buffer (no unwrapping needed) + if (Buffer.isBuffer(data)) { + return data + } + + // Case 2: Wrapped binary data {_binary: true, data: "base64..."} + if (isWrappedBinary(data)) { + return Buffer.from(data.data, 'base64') + } + + // Case 3: Plain object (shouldn't happen for binary blobs, but handle gracefully) + if (typeof data === 'object' && data !== null) { + return Buffer.from(JSON.stringify(data)) + } + + // Case 4: String (convert to Buffer) + if (typeof data === 'string') { + return Buffer.from(data) + } + + // Case 5: Invalid type + throw new Error( + `Invalid data type for unwrap: ${typeof data}. ` + + `Expected Buffer or {_binary: true, data: "base64..."}` + ) +} diff --git a/src/storage/blobStorage.ts b/src/storage/blobStorage.ts new file mode 100644 index 00000000..4d98c0bc --- /dev/null +++ b/src/storage/blobStorage.ts @@ -0,0 +1,498 @@ +/** + * @module storage/blobStorage + * @description Content-addressed blob store. Backs VFS file content with + * SHA-256 addressing (automatic deduplication), reference counting, zstd + * compression where it pays (MIME-aware: already-compressed media is stored + * raw), and an LRU read cache. + * + * The store persists through a narrow key-value bridge + * ({@link BlobStoreAdapter}) provided by `BaseStorage`, which roots all keys + * under the `_cas/` storage area. Key naming is an explicit type contract: + * `blob:` keys hold binary bytes, `blob-meta:` keys hold JSON + * metadata — the key format decides how bytes are encoded, never content + * sniffing. + */ + +import { createHash } from 'crypto' +import { unwrapBinaryData } from './binaryDataCodec.js' + +/** + * @description Key-value bridge the blob store persists through. Implemented + * by `BaseStorage.initializeBlobStorage()` over the adapter's raw object + * primitives. + */ +export interface BlobStoreAdapter { + /** Read the bytes stored under `key`, or `undefined` when absent. */ + get(key: string): Promise + /** Persist `data` under `key` (overwrites). */ + put(key: string, data: Buffer): Promise + /** Delete the value under `key`. Missing keys are ignored. */ + delete(key: string): Promise + /** List all keys starting with `prefix`. */ + list(prefix: string): Promise +} + +/** + * @description Metadata persisted alongside each blob (under + * `blob-meta:`). + */ +export interface BlobMetadata { + /** SHA-256 content hash (the blob's identity). */ + hash: string + /** Original (uncompressed) size in bytes. */ + size: number + /** Stored size in bytes (after compression, if any). */ + compressedSize: number + /** Compression applied to the stored bytes. */ + compression: 'none' | 'zstd' + /** Creation timestamp (epoch ms). */ + createdAt: number + /** Number of logical references to this blob (deduplicated writes). */ + refCount: number +} + +/** + * @description Options for {@link BlobStorage.write}. + */ +export interface BlobWriteOptions { + /** + * Compression strategy. `'auto'` (default) compresses payloads above 1 KB + * with zstd unless the MIME type says the bytes are already compressed. + * Explicit `'none'`/`'zstd'` is honoured as asserted by the caller. + */ + compression?: 'none' | 'zstd' | 'auto' + /** + * Content type of the payload (e.g. `image/jpeg`, `video/mp4`). + * + * When set on an `auto`-compression write, the store skips zstd for MIME + * types that are already heavily compressed (JPEG, PNG, WebP, MP4, WebM, + * MP3, ZIP, PDF, etc.). zstd over these formats wastes CPU and rarely + * shaves more than a single-digit percent — usually it actually grows the + * payload because the entropy is already maximised by the format itself. + * + * Has no effect when `compression` is `'none'` or `'zstd'` explicitly — + * the caller is asserting the choice and the store honours it. + */ + mimeType?: string +} + +/** + * MIME types whose payload is already heavily compressed. zstd over these is + * almost always a CPU-only loss — the bytes are already near entropy-maximal, + * so the output is the same size or slightly larger plus the cost of running + * the compressor. Used by `BlobStorage.selectCompression()` in `auto` mode. + * + * Conservative denylist (well-known formats only). Anything not in this set + * goes through the size heuristic. False negatives (compressing something we + * should have skipped) waste CPU; false positives (skipping something we + * could have compressed) waste a few percent of bytes. The denylist favours + * CPU-cycle safety because the formats listed here are the ones where + * gzip/zstd is reliably a net loss. + */ +const ALREADY_COMPRESSED_MIME_TYPES = new Set([ + // Images + 'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp', + 'image/avif', 'image/heic', 'image/heif', 'image/jp2', + // Video + 'video/mp4', 'video/webm', 'video/x-matroska', 'video/quicktime', + 'video/x-msvideo', 'video/mpeg', 'video/3gpp', 'video/x-ms-wmv', + // Audio + 'audio/mpeg', 'audio/mp4', 'audio/aac', 'audio/ogg', 'audio/webm', + 'audio/opus', 'audio/flac', 'audio/x-ms-wma', + // Archives + 'application/zip', 'application/gzip', 'application/x-gzip', + 'application/x-bzip2', 'application/x-7z-compressed', + 'application/x-rar-compressed', 'application/x-xz', 'application/x-zstd', + 'application/x-compress', 'application/vnd.rar', + // Documents with internal compression + 'application/pdf', 'application/epub+zip', + // Office formats (zip-based) + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'application/vnd.oasis.opendocument.text', + 'application/vnd.oasis.opendocument.spreadsheet', + 'application/vnd.oasis.opendocument.presentation' +]) + +/** + * @description True when the MIME type names a payload format known to be + * already heavily compressed. Strips any `;charset=…` / `;boundary=…` + * parameters and lowercases the bare type/subtype before lookup, so + * `'IMAGE/JPEG; charset=binary'` and `'image/jpeg'` resolve identically. + * @param mimeType - MIME type string, or `undefined`. + * @returns Whether `auto` compression should skip zstd for this payload. + */ +export function isAlreadyCompressedMimeType(mimeType: string | undefined): boolean { + if (!mimeType) return false + const bare = mimeType.split(';', 1)[0].trim().toLowerCase() + return ALREADY_COMPRESSED_MIME_TYPES.has(bare) +} + +/** + * LRU cache entry. + */ +interface CacheEntry { + data: Buffer + metadata: BlobMetadata + lastAccess: number + size: number +} + +/** + * @description Content-addressed, deduplicating, reference-counted blob + * store with MIME-aware zstd compression and an LRU read cache. See the + * module doc for the persistence contract. + * + * @example + * const hash = await blobStorage.write(buffer, { mimeType: 'image/png' }) + * const bytes = await blobStorage.read(hash) // verified against the hash + * await blobStorage.delete(hash) // decrements refCount first + */ +export class BlobStorage { + private adapter: BlobStoreAdapter + private cache: Map + private cacheMaxSize: number + private currentCacheSize: number + + // Compression (lazily loaded) + private zstdCompress?: (data: Buffer) => Promise + private zstdDecompress?: (data: Buffer) => Promise + private compressionReady = false + + // Configuration + private readonly CACHE_MAX_SIZE = 100 * 1024 * 1024 // 100MB default + private readonly COMPRESSION_THRESHOLD = 1024 // 1KB - don't compress smaller + + /** + * @param adapter - Key-value bridge to persist through. + * @param options - `cacheMaxSize` bounds the LRU read cache (bytes, + * default 100 MB). + */ + constructor(adapter: BlobStoreAdapter, options?: { cacheMaxSize?: number }) { + this.adapter = adapter + this.cache = new Map() + this.cacheMaxSize = options?.cacheMaxSize ?? this.CACHE_MAX_SIZE + this.currentCacheSize = 0 + } + + /** + * Lazy-load the zstd compression module. Falls back to uncompressed + * storage when the optional dependency is unavailable. + */ + private async ensureCompressionReady(): Promise { + if (this.compressionReady) return + try { + // Dynamic import to avoid loading if not needed + // @ts-ignore - Optional dependency, gracefully handled if missing + const zstd = await import('@mongodb-js/zstd') + this.zstdCompress = async (data: Buffer) => { + return Buffer.from(await zstd.compress(data, 3)) // Level 3 = fast + } + this.zstdDecompress = async (data: Buffer) => { + return Buffer.from(await zstd.decompress(data)) + } + } catch (error) { + console.warn('zstd compression not available, falling back to uncompressed') + this.zstdCompress = undefined + this.zstdDecompress = undefined + } + this.compressionReady = true + } + + /** + * @description Compute the SHA-256 content hash of `data`. + * @param data - Bytes to hash. + * @returns Hex-encoded SHA-256 hash. + */ + static hash(data: Buffer): string { + return createHash('sha256').update(data).digest('hex') + } + + /** + * @description Write a blob. Content-addressed: the SHA-256 hash of the + * bytes is the storage key, so identical payloads deduplicate (the + * existing blob's reference count is incremented instead of rewriting). + * + * @param data - Blob bytes. + * @param options - Compression strategy and MIME hint (see + * {@link BlobWriteOptions}). + * @returns The blob's SHA-256 hash. + */ + async write(data: Buffer, options: BlobWriteOptions = {}): Promise { + const hash = BlobStorage.hash(data) + + // Deduplication: identical content already stored — just add a reference. + if (await this.has(hash)) { + await this.incrementRefCount(hash) + return hash + } + + await this.ensureCompressionReady() + + // Determine compression strategy + const compression = this.selectCompression(data, options) + + // Compress if needed + let finalData = data + let compressedSize = data.length + + if (compression === 'zstd' && this.zstdCompress) { + finalData = await this.zstdCompress(data) + compressedSize = finalData.length + } + + // Record the ACTUAL compression state, not the intended one — prevents + // corruption if compression failed to initialize. + const actualCompression = finalData === data ? 'none' : compression + const metadata: BlobMetadata = { + hash, + size: data.length, + compressedSize, + compression: actualCompression, + createdAt: Date.now(), + refCount: 1 + } + + await this.adapter.put(`blob:${hash}`, finalData) + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + + // Write-through cache (caches the ORIGINAL bytes, not the compressed form) + this.addToCache(hash, data, metadata) + + return hash + } + + /** + * @description Read a blob: LRU cache first, then storage with + * decompression and integrity verification (the bytes are re-hashed and + * compared against the requested hash). + * + * @param hash - The blob's SHA-256 hash. + * @returns The original (decompressed) blob bytes. + * @throws Error when the blob is missing or fails integrity verification. + */ + async read(hash: string): Promise { + // Check cache first + const cached = this.getFromCache(hash) + if (cached) { + return cached.data + } + + const metadataBuffer = await this.adapter.get(`blob-meta:${hash}`) + if (!metadataBuffer) { + throw new Error(`Blob metadata not found: ${hash}`) + } + // Unwrap before parsing (defense-in-depth): metadata should come back as + // JSON bytes, but an adapter might return the wrapped binary format. + const metadata: BlobMetadata = JSON.parse(unwrapBinaryData(metadataBuffer).toString()) + + const data = await this.adapter.get(`blob:${hash}`) + if (!data) { + throw new Error(`Blob not found: ${hash}`) + } + + // Decompress if needed + let finalData = data + if (metadata.compression === 'zstd') { + if (!this.zstdDecompress) { + await this.ensureCompressionReady() + } + if (!this.zstdDecompress) { + throw new Error('zstd decompression not available') + } + finalData = await this.zstdDecompress(data) + } + + // Defense-in-depth unwrap: even though the bridge unwraps, verify it + // happened and re-unwrap if needed. Hash verification must run on the + // original content bytes. + const unwrappedData = unwrapBinaryData(finalData) + + // Integrity verification (always on — a content-addressed store that + // returns bytes not matching the address is corruption, not a result) + if (BlobStorage.hash(unwrappedData) !== hash) { + throw new Error(`Blob integrity check failed: ${hash}`) + } + + this.addToCache(hash, unwrappedData, metadata) + + return unwrappedData + } + + /** + * @description Whether a blob with this hash exists (cache or storage). + * @param hash - The blob's SHA-256 hash. + * @returns True when the blob exists. + */ + async has(hash: string): Promise { + if (this.cache.has(hash)) { + return true + } + const exists = await this.adapter.get(`blob:${hash}`) + return exists !== undefined + } + + /** + * @description Drop one reference to the blob. The stored bytes and + * metadata are physically deleted only when the reference count reaches + * zero — deduplicated content shared by other writers survives. + * + * @param hash - The blob's SHA-256 hash. + */ + async delete(hash: string): Promise { + const refCount = await this.decrementRefCount(hash) + + // Only delete if no references remain + if (refCount > 0) { + return + } + + await this.adapter.delete(`blob:${hash}`) + await this.adapter.delete(`blob-meta:${hash}`) + this.removeFromCache(hash) + } + + /** + * @description Read a blob's metadata without reading its bytes. + * @param hash - The blob's SHA-256 hash. + * @returns The metadata, or `undefined` when the blob does not exist. + */ + async getMetadata(hash: string): Promise { + const data = await this.adapter.get(`blob-meta:${hash}`) + if (data) { + return JSON.parse(unwrapBinaryData(data).toString()) + } + return undefined + } + + // ========== PRIVATE METHODS ========== + + /** + * Select the compression strategy for a write (see + * {@link BlobWriteOptions.compression}). + */ + private selectCompression( + data: Buffer, + options: BlobWriteOptions + ): 'none' | 'zstd' { + if (options.compression === 'none') { + return 'none' + } + + if (options.compression === 'zstd') { + return this.zstdCompress ? 'zstd' : 'none' + } + + // Auto mode + if (data.length < this.COMPRESSION_THRESHOLD) { + return 'none' // Too small to benefit + } + + // Content-type policy: skip already-compressed media. zstd over + // JPEG / MP4 / ZIP etc. is a CPU loss for no measurable byte savings, + // and on hot save paths (image / video uploads) it's the difference + // between fast and slow. Applies only to `auto`; explicit `'zstd'` is + // honoured because the caller is asserting the choice. + if (isAlreadyCompressedMimeType(options.mimeType)) { + return 'none' + } + + return this.zstdCompress ? 'zstd' : 'none' + } + + /** + * Increment the reference count for an existing blob. + */ + private async incrementRefCount(hash: string): Promise { + const metadata = await this.getMetadata(hash) + if (!metadata) { + throw new Error(`Cannot increment ref count, blob not found: ${hash}`) + } + + metadata.refCount++ + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + return metadata.refCount + } + + /** + * Decrement the reference count for a blob (floored at zero). + */ + private async decrementRefCount(hash: string): Promise { + const metadata = await this.getMetadata(hash) + if (!metadata) { + return 0 + } + + metadata.refCount = Math.max(0, metadata.refCount - 1) + await this.adapter.put(`blob-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) + return metadata.refCount + } + + /** + * Add a blob to the LRU cache (evicting least-recently-used entries to + * stay under the size bound). + */ + private addToCache(hash: string, data: Buffer, metadata: BlobMetadata): void { + if (data.length > this.cacheMaxSize) { + return // Blob too large for cache + } + + while ( + this.currentCacheSize + data.length > this.cacheMaxSize && + this.cache.size > 0 + ) { + this.evictLRU() + } + + this.cache.set(hash, { + data, + metadata, + lastAccess: Date.now(), + size: data.length + }) + + this.currentCacheSize += data.length + } + + /** + * Get a blob from the cache, refreshing its LRU position. + */ + private getFromCache(hash: string): CacheEntry | undefined { + const entry = this.cache.get(hash) + if (entry) { + entry.lastAccess = Date.now() // Update LRU + } + return entry + } + + /** + * Remove a blob from the cache. + */ + private removeFromCache(hash: string): void { + const entry = this.cache.get(hash) + if (entry) { + this.cache.delete(hash) + this.currentCacheSize -= entry.size + } + } + + /** + * Evict the least-recently-used cache entry. + */ + private evictLRU(): void { + let oldestHash: string | null = null + let oldestTime = Infinity + + for (const [hash, entry] of this.cache.entries()) { + if (entry.lastAccess < oldestTime) { + oldestTime = entry.lastAccess + oldestHash = hash + } + } + + if (oldestHash) { + this.removeFromCache(oldestHash) + } + } +} diff --git a/src/storage/cow/BlobStorage.ts b/src/storage/cow/BlobStorage.ts deleted file mode 100644 index e023377f..00000000 --- a/src/storage/cow/BlobStorage.ts +++ /dev/null @@ -1,763 +0,0 @@ -/** - * BlobStorage: Content-Addressable Blob Storage for COW (Copy-on-Write) - * - * State-of-the-art implementation featuring: - * - Content-addressable: SHA-256 hashing - * - Type-aware chunking: Separate vectors, metadata, relationships - * - Compression: zstd for JSON, optimized for vectors - * - LRU caching: Hot blob performance - * - Streaming: Multipart upload for large blobs - * - Batch operations: Parallel I/O - * - Integrity: Cryptographic verification - * - Observability: Metrics and tracing - * - * @module storage/cow/BlobStorage - */ - -import { createHash } from 'crypto' -import { NULL_HASH, isNullHash } from './constants.js' -import { unwrapBinaryData } from './binaryDataCodec.js' - -/** - * Simple key-value storage interface for COW primitives - * This will be implemented by BaseStorage when COW is integrated - */ -export interface COWStorageAdapter { - get(key: string): Promise - put(key: string, data: Buffer): Promise - delete(key: string): Promise - list(prefix: string): Promise -} - -/** - * Blob metadata stored alongside blob data - */ -export interface BlobMetadata { - hash: string // SHA-256 hash - size: number // Original size in bytes - compressedSize: number // Compressed size in bytes - compression: 'none' | 'zstd' - type: 'vector' | 'metadata' | 'tree' | 'commit' | 'blob' | 'raw' - createdAt: number // Timestamp - refCount: number // How many objects reference this blob -} - -/** - * Blob write options - */ -export interface BlobWriteOptions { - compression?: 'none' | 'zstd' | 'auto' // Auto chooses based on type - type?: 'vector' | 'metadata' | 'tree' | 'commit' | 'blob' | 'raw' - /** - * Content type of the payload (e.g. `image/jpeg`, `video/mp4`). - * - * When set on an `auto`-compression write, BlobStorage skips zstd for MIME - * types that are already heavily compressed (JPEG, PNG, WebP, MP4, WebM, - * MP3, ZIP, PDF, etc.). Gzip/zstd over these formats wastes CPU and rarely - * shaves more than a single-digit percent — usually it actually grows the - * payload because the entropy is already maximised by the format itself. - * - * Has no effect when `compression` is set to `'none'` or `'zstd'` explicitly - * — the caller is asserting the choice and BlobStorage honours it. - */ - mimeType?: string - skipVerification?: boolean // Skip hash verification (faster, less safe) -} - -/** - * MIME types whose payload is already heavily compressed. zstd over these is - * almost always a CPU-only loss — the bytes are already near entropy-maximal, - * so the output is the same size or slightly larger plus the cost of running - * the compressor. Used by `BlobStorage.selectCompression()` in `auto` mode. - * - * Conservative denylist (well-known formats only). Anything not in this set - * goes through the existing per-type heuristic. False negatives (compressing - * something we should have skipped) waste CPU; false positives (skipping - * something we could have compressed) waste a few percent of bytes. The - * denylist favours CPU-cycle safety because the formats listed here are the - * ones where gzip/zstd is reliably a net loss. - */ -const ALREADY_COMPRESSED_MIME_TYPES = new Set([ - // Images - 'image/jpeg', 'image/jpg', 'image/png', 'image/gif', 'image/webp', - 'image/avif', 'image/heic', 'image/heif', 'image/jp2', - // Video - 'video/mp4', 'video/webm', 'video/x-matroska', 'video/quicktime', - 'video/x-msvideo', 'video/mpeg', 'video/3gpp', 'video/x-ms-wmv', - // Audio - 'audio/mpeg', 'audio/mp4', 'audio/aac', 'audio/ogg', 'audio/webm', - 'audio/opus', 'audio/flac', 'audio/x-ms-wma', - // Archives - 'application/zip', 'application/gzip', 'application/x-gzip', - 'application/x-bzip2', 'application/x-7z-compressed', - 'application/x-rar-compressed', 'application/x-xz', 'application/x-zstd', - 'application/x-compress', 'application/vnd.rar', - // Documents with internal compression - 'application/pdf', 'application/epub+zip', - // Office formats (zip-based) - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - 'application/vnd.oasis.opendocument.text', - 'application/vnd.oasis.opendocument.spreadsheet', - 'application/vnd.oasis.opendocument.presentation' -]) - -/** - * @description True when the MIME type names a payload format known to be - * already heavily compressed. Strips any `;charset=…` / `;boundary=…` - * parameters and lowercases the bare type/subtype before lookup, so - * `'IMAGE/JPEG; charset=binary'` and `'image/jpeg'` resolve identically. - */ -export function isAlreadyCompressedMimeType(mimeType: string | undefined): boolean { - if (!mimeType) return false - const bare = mimeType.split(';', 1)[0].trim().toLowerCase() - return ALREADY_COMPRESSED_MIME_TYPES.has(bare) -} - -/** - * Blob read options - */ -export interface BlobReadOptions { - skipDecompression?: boolean // Return compressed data - skipCache?: boolean // Don't use cache - skipVerification?: boolean // Skip hash verification (faster, less safe) -} - -/** - * Blob statistics for observability - */ -export interface BlobStats { - totalBlobs: number - totalSize: number - compressedSize: number - cacheHits: number - cacheMisses: number - compressionRatio: number - avgBlobSize: number - dedupSavings: number // Bytes saved from deduplication -} - -/** - * LRU Cache entry - */ -interface CacheEntry { - data: Buffer - metadata: BlobMetadata - lastAccess: number - size: number -} - -/** - * State-of-the-art content-addressable blob storage - * - * Features: - * - Content addressing via SHA-256 - * - Type-aware compression (zstd, vector-optimized) - * - LRU caching with memory limits - * - Streaming for large blobs - * - Batch operations - * - Integrity verification - * - Observability metrics - */ -export class BlobStorage { - private adapter: COWStorageAdapter - private cache: Map - private cacheMaxSize: number - private currentCacheSize: number - private stats: BlobStats - - // Compression (lazily loaded) - private zstdCompress?: (data: Buffer) => Promise - private zstdDecompress?: (data: Buffer) => Promise - private compressionReady = false - - // Configuration - private readonly CACHE_MAX_SIZE = 100 * 1024 * 1024 // 100MB default - private readonly MULTIPART_THRESHOLD = 5 * 1024 * 1024 // 5MB - private readonly COMPRESSION_THRESHOLD = 1024 // 1KB - don't compress smaller - - constructor(adapter: COWStorageAdapter, options?: { - cacheMaxSize?: number - enableCompression?: boolean - }) { - this.adapter = adapter - this.cache = new Map() - this.cacheMaxSize = options?.cacheMaxSize ?? this.CACHE_MAX_SIZE - this.currentCacheSize = 0 - this.stats = { - totalBlobs: 0, - totalSize: 0, - compressedSize: 0, - cacheHits: 0, - cacheMisses: 0, - compressionRatio: 1.0, - avgBlobSize: 0, - dedupSavings: 0 - } - - // Lazy load compression (only if needed) - if (options?.enableCompression !== false) { - this.initCompression() - } - } - - /** - * Lazy load zstd compression module - * (Avoids loading if not needed) - */ - private async initCompression(): Promise { - try { - // Dynamic import to avoid loading if not needed - // @ts-ignore - Optional dependency, gracefully handled if missing - const zstd = await import('@mongodb-js/zstd') - this.zstdCompress = async (data: Buffer) => { - return Buffer.from(await zstd.compress(data, 3)) // Level 3 = fast - } - this.zstdDecompress = async (data: Buffer) => { - return Buffer.from(await zstd.decompress(data)) - } - } catch (error) { - console.warn('zstd compression not available, falling back to uncompressed') - this.zstdCompress = undefined - this.zstdDecompress = undefined - } - } - - /** - * Ensure compression is ready before write operations - * Fixes race condition where write happens before async compression init completes - */ - private async ensureCompressionReady(): Promise { - if (this.compressionReady) return - await this.initCompression() - this.compressionReady = true - } - - /** - * Compute SHA-256 hash of data - * - * @param data - Data to hash - * @returns SHA-256 hash as hex string - */ - static hash(data: Buffer): string { - return createHash('sha256').update(data).digest('hex') - } - - /** - * Write a blob to storage - * - * Features: - * - Content-addressable: hash determines storage key - * - Deduplication: existing blob not rewritten - * - Compression: auto-compress based on type - * - Multipart: for large blobs (>5MB) - * - Verification: hash verification - * - Caching: write-through cache - * - * @param data - Blob data to write - * @param options - Write options - * @returns Blob hash - */ - async write(data: Buffer, options: BlobWriteOptions = {}): Promise { - const hash = BlobStorage.hash(data) - - // Deduplication: Check if blob already exists - if (await this.has(hash)) { - // Update ref count - await this.incrementRefCount(hash) - this.stats.dedupSavings += data.length - return hash - } - - // Ensure compression is initialized before writing - // Fixes race condition where write happens before async init completes - await this.ensureCompressionReady() - - // Determine compression strategy - const compression = this.selectCompression(data, options) - - // Compress if needed - let finalData = data - let compressedSize = data.length - - if (compression === 'zstd' && this.zstdCompress) { - finalData = await this.zstdCompress(data) - compressedSize = finalData.length - } - - // Create metadata - // Store ACTUAL compression state, not intended - // Prevents corruption if compression failed to initialize - const actualCompression = finalData === data ? 'none' : compression - const metadata: BlobMetadata = { - hash, - size: data.length, - compressedSize, - compression: actualCompression, - type: options.type || 'blob', // CRITICAL FIX: Use 'blob' default to match storage prefix - createdAt: Date.now(), - refCount: 1 - } - - // Write blob data - if (finalData.length > this.MULTIPART_THRESHOLD) { - // Large blob: use streaming/multipart - await this.writeMultipart(hash, finalData, metadata) - } else { - // Small blob: single write - const prefix = options.type || 'blob' - await this.adapter.put(`${prefix}:${hash}`, finalData) - } - - // Write metadata - const prefix = options.type || 'blob' - await this.adapter.put(`${prefix}-meta:${hash}`, Buffer.from(JSON.stringify(metadata))) - - // Update cache (write-through) - this.addToCache(hash, data, metadata) - - // Update stats - this.stats.totalBlobs++ - this.stats.totalSize += data.length - this.stats.compressedSize += compressedSize - this.stats.compressionRatio = this.stats.totalSize / (this.stats.compressedSize || 1) - this.stats.avgBlobSize = this.stats.totalSize / this.stats.totalBlobs - - return hash - } - - /** - * Read a blob from storage - * - * Features: - * - Cache lookup first (LRU) - * - Decompression (if compressed) - * - Verification (optional hash check) - * - Streaming for large blobs - * - * @param hash - Blob hash - * @param options - Read options - * @returns Blob data - */ - async read(hash: string, options: BlobReadOptions = {}): Promise { - // Guard against NULL hash (sentinel value) - // NULL_HASH ('0000...0000') is used as a sentinel for "no parent" or "empty tree" - // It should NEVER be read as actual blob data - if (isNullHash(hash)) { - throw new Error( - `Cannot read NULL hash (${NULL_HASH}): ` + - `This is a sentinel value indicating "no parent commit" or "empty tree". ` + - `If you're seeing this error from CommitObject.walk(), there's a bug in commit traversal logic. ` + - `If you're seeing this from TreeObject operations, there's a bug in tree handling.` - ) - } - - // Check cache first - if (!options.skipCache) { - const cached = this.getFromCache(hash) - if (cached) { - this.stats.cacheHits++ - return cached.data - } - this.stats.cacheMisses++ - } - - // Try to read metadata to determine type (for backward compatibility) - // Try commit, tree, then blob prefixes - let prefix: string | null = null - let metadataBuffer: Buffer | undefined - let metadata: BlobMetadata | undefined - - for (const tryPrefix of ['commit', 'tree', 'blob']) { - metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`) - if (metadataBuffer) { - prefix = tryPrefix - // Unwrap metadata before parsing (defense-in-depth) - // Metadata should be JSON, but adapter might return wrapped format - const unwrappedMetadata = unwrapBinaryData(metadataBuffer) - metadata = JSON.parse(unwrappedMetadata.toString()) - break - } - } - - if (!prefix || !metadata) { - throw new Error(`Blob metadata not found: ${hash}`) - } - - // Read from storage using determined prefix - const data = await this.adapter.get(`${prefix}:${hash}`) - - if (!data) { - throw new Error(`Blob not found: ${hash}`) - } - - // Decompress if needed - let finalData = data - - if (metadata.compression === 'zstd' && !options.skipDecompression) { - if (!this.zstdDecompress) { - throw new Error('zstd decompression not available') - } - finalData = await this.zstdDecompress(data) - } - - // Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression) - // Even though COW adapter should unwrap, verify it happened and re-unwrap if needed - // This prevents "Blob integrity check failed" errors if adapter returns wrapped data - // Uses shared binaryDataCodec utility (single source of truth for unwrap logic) - const unwrappedData = unwrapBinaryData(finalData) - - // Verify hash (on unwrapped data) - if (!options.skipVerification && BlobStorage.hash(unwrappedData) !== hash) { - throw new Error(`Blob integrity check failed: ${hash}`) - } - - // Add to cache (only if not skipped) - if (!options.skipCache) { - this.addToCache(hash, unwrappedData, metadata) - } - - return unwrappedData - } - - /** - * Check if blob exists - * - * @param hash - Blob hash - * @returns True if blob exists - */ - async has(hash: string): Promise { - // Check cache first - if (this.cache.has(hash)) { - return true - } - - // Check storage - try all prefixes for backward compatibility - for (const prefix of ['commit', 'tree', 'blob']) { - const exists = await this.adapter.get(`${prefix}:${hash}`) - if (exists !== undefined) { - return true - } - } - - return false - } - - /** - * Delete a blob from storage - * - * Features: - * - Reference counting: only delete if refCount = 0 - * - Cascade: delete metadata too - * - Cache invalidation - * - * @param hash - Blob hash - */ - async delete(hash: string): Promise { - // Decrement ref count - const refCount = await this.decrementRefCount(hash) - - // Only delete if no references remain - if (refCount > 0) { - return - } - - // Determine prefix by checking which one exists - let prefix = 'blob' - for (const tryPrefix of ['commit', 'tree', 'blob', 'metadata', 'vector', 'raw']) { - const exists = await this.adapter.get(`${tryPrefix}:${hash}`) - if (exists !== undefined) { - prefix = tryPrefix - break - } - } - - // Delete blob data - await this.adapter.delete(`${prefix}:${hash}`) - - // Delete metadata - await this.adapter.delete(`${prefix}-meta:${hash}`) - - // Remove from cache - this.removeFromCache(hash) - - // Update stats - this.stats.totalBlobs-- - } - - /** - * Get blob metadata without reading full blob - * - * @param hash - Blob hash - * @returns Blob metadata - */ - async getMetadata(hash: string): Promise { - // Try to read metadata with type-aware prefix (backward compatible) - // Check all valid blob types: commit, tree, blob, metadata, vector, raw - for (const prefix of ['commit', 'tree', 'blob', 'metadata', 'vector', 'raw']) { - const data = await this.adapter.get(`${prefix}-meta:${hash}`) - if (data) { - return JSON.parse(data.toString()) - } - } - - return undefined - } - - /** - * Batch write multiple blobs in parallel - * - * @param blobs - Array of [data, options] tuples - * @returns Array of blob hashes - */ - async writeBatch(blobs: Array<[Buffer, BlobWriteOptions?]>): Promise { - return Promise.all( - blobs.map(([data, options]) => this.write(data, options)) - ) - } - - /** - * Batch read multiple blobs in parallel - * - * @param hashes - Array of blob hashes - * @param options - Read options - * @returns Array of blob data - */ - async readBatch(hashes: string[], options?: BlobReadOptions): Promise { - return Promise.all( - hashes.map(hash => this.read(hash, options)) - ) - } - - /** - * List all blobs (for garbage collection, debugging) - * - * @returns Array of blob hashes - */ - async listBlobs(): Promise { - // List all types of blobs - const hashes = new Set() - - for (const prefix of ['commit', 'tree', 'blob']) { - const keys = await this.adapter.list(`${prefix}:`) - keys.forEach((key: string) => { - const hash = key.replace(new RegExp(`^${prefix}:`), '') - hashes.add(hash) - }) - } - - return Array.from(hashes) - } - - /** - * Get storage statistics - * - * @returns Blob statistics - */ - getStats(): BlobStats { - return { ...this.stats } - } - - /** - * Clear cache (useful for testing, memory pressure) - */ - clearCache(): void { - this.cache.clear() - this.currentCacheSize = 0 - } - - /** - * Garbage collect unreferenced blobs - * - * @param referencedHashes - Set of hashes that should be kept - * @returns Number of blobs deleted - */ - async garbageCollect(referencedHashes: Set): Promise { - const allBlobs = await this.listBlobs() - let deleted = 0 - - for (const hash of allBlobs) { - if (!referencedHashes.has(hash)) { - // Check ref count - const metadata = await this.getMetadata(hash) - if (metadata && metadata.refCount === 0) { - await this.delete(hash) - deleted++ - } - } - } - - return deleted - } - - // ========== PRIVATE METHODS ========== - - /** - * Select compression strategy based on data and options - */ - private selectCompression( - data: Buffer, - options: BlobWriteOptions - ): 'none' | 'zstd' { - if (options.compression === 'none') { - return 'none' - } - - if (options.compression === 'zstd') { - return this.zstdCompress ? 'zstd' : 'none' - } - - // Auto mode - if (data.length < this.COMPRESSION_THRESHOLD) { - return 'none' // Too small to benefit - } - - // Content-type policy (2.5.0 #32): skip already-compressed media. zstd - // over JPEG / MP4 / ZIP etc. is a CPU loss for no measurable byte savings, - // and on hot save paths (image / video uploads) it's the difference - // between fast and slow. Applies only to `auto`; explicit `'zstd'` is - // honoured because the caller is asserting the choice. - if (isAlreadyCompressedMimeType(options.mimeType)) { - return 'none' - } - - // Compress metadata, trees, commits (text/JSON) - if (options.type === 'metadata' || options.type === 'tree' || options.type === 'commit') { - return this.zstdCompress ? 'zstd' : 'none' - } - - // Don't compress vectors (already dense) - if (options.type === 'vector') { - return 'none' - } - - // Default: compress - return this.zstdCompress ? 'zstd' : 'none' - } - - /** - * Write large blob using multipart upload - * (Future enhancement: stream to adapter if supported) - */ - private async writeMultipart( - hash: string, - data: Buffer, - metadata: BlobMetadata - ): Promise { - // Single-blob write. Brainy 8.0 ships filesystem + memory only; the - // multipart-upload concern that motivated this method (S3/R2/GCS) is gone - // with the cloud adapters per BR-BRAINY-80-STORAGE-SIMPLIFY. - const prefix = metadata.type || 'blob' - await this.adapter.put(`${prefix}:${hash}`, data) - } - - /** - * Increment reference count for a blob - */ - private async incrementRefCount(hash: string): Promise { - const metadata = await this.getMetadata(hash) - if (!metadata) { - throw new Error(`Cannot increment ref count, blob not found: ${hash}`) - } - - metadata.refCount++ - - const prefix = metadata.type || 'blob' - await this.adapter.put( - `${prefix}-meta:${hash}`, - Buffer.from(JSON.stringify(metadata)) - ) - - return metadata.refCount - } - - /** - * Decrement reference count for a blob - */ - private async decrementRefCount(hash: string): Promise { - const metadata = await this.getMetadata(hash) - if (!metadata) { - return 0 - } - - metadata.refCount = Math.max(0, metadata.refCount - 1) - - const prefix = metadata.type || 'blob' - await this.adapter.put( - `${prefix}-meta:${hash}`, - Buffer.from(JSON.stringify(metadata)) - ) - - return metadata.refCount - } - - /** - * Add blob to LRU cache - */ - private addToCache(hash: string, data: Buffer, metadata: BlobMetadata): void { - // Check if adding would exceed cache size - if (data.length > this.cacheMaxSize) { - return // Blob too large for cache - } - - // Evict old entries if needed - while ( - this.currentCacheSize + data.length > this.cacheMaxSize && - this.cache.size > 0 - ) { - this.evictLRU() - } - - // Add to cache - this.cache.set(hash, { - data, - metadata, - lastAccess: Date.now(), - size: data.length - }) - - this.currentCacheSize += data.length - } - - /** - * Get blob from cache - */ - private getFromCache(hash: string): CacheEntry | undefined { - const entry = this.cache.get(hash) - if (entry) { - entry.lastAccess = Date.now() // Update LRU - } - return entry - } - - /** - * Remove blob from cache - */ - private removeFromCache(hash: string): void { - const entry = this.cache.get(hash) - if (entry) { - this.cache.delete(hash) - this.currentCacheSize -= entry.size - } - } - - /** - * Evict least recently used entry from cache - */ - private evictLRU(): void { - let oldestHash: string | null = null - let oldestTime = Infinity - - for (const [hash, entry] of this.cache.entries()) { - if (entry.lastAccess < oldestTime) { - oldestTime = entry.lastAccess - oldestHash = hash - } - } - - if (oldestHash) { - this.removeFromCache(oldestHash) - } - } -} diff --git a/src/storage/cow/CommitLog.ts b/src/storage/cow/CommitLog.ts deleted file mode 100644 index ca9a1650..00000000 --- a/src/storage/cow/CommitLog.ts +++ /dev/null @@ -1,531 +0,0 @@ -/** - * CommitLog: Commit history traversal and querying for COW (Copy-on-Write) - * - * Provides efficient commit history operations: - * - Walk commit graph (DAG traversal) - * - Find commits by time, author, operation - * - Time-travel queries (asOf) - * - Commit statistics and analytics - * - * Optimizations: - * - Commit index for fast timestamp lookups - * - Parent cache for efficient traversal - * - Lazy loading (only read commits when needed) - * - * @module storage/cow/CommitLog - */ - -import { BlobStorage } from './BlobStorage.js' -import { CommitObject } from './CommitObject.js' -import { RefManager } from './RefManager.js' - -/** - * Commit index entry (for fast lookups) - */ -interface CommitIndexEntry { - hash: string - timestamp: number - parentHash: string | null -} - -/** - * Commit log statistics - */ -export interface CommitLogStats { - totalCommits: number - oldestCommit: number // Timestamp - newestCommit: number // Timestamp - authors: Set - operations: Set - avgCommitInterval: number // Average time between commits (ms) -} - -/** - * CommitLog: Efficient commit history traversal and querying - * - * Pure implementation - modern, clean, fast - */ -export class CommitLog { - private blobStorage: BlobStorage - private refManager: RefManager - private index: Map - private indexValid: boolean - - constructor(blobStorage: BlobStorage, refManager: RefManager) { - this.blobStorage = blobStorage - this.refManager = refManager - this.index = new Map() - this.indexValid = false - } - - /** - * Walk commit history from a starting point - * - * Yields commits in reverse chronological order (newest first) - * - * @param startRef - Starting ref/commit (e.g., 'main', commit hash) - * @param options - Walk options - */ - async *walk( - startRef: string = 'main', - options?: { - maxDepth?: number - until?: number - stopAt?: string - filter?: (commit: CommitObject) => boolean - } - ): AsyncIterableIterator { - // Resolve ref to commit hash - let startHash: string - - if (/^[a-f0-9]{64}$/.test(startRef)) { - // Already a commit hash - startHash = startRef - } else { - // Resolve ref - const commitHash = await this.refManager.resolveRef(startRef) - if (!commitHash) { - throw new Error(`Ref not found: ${startRef}`) - } - startHash = commitHash - } - - // Walk using CommitObject (delegates to it) - yield* CommitObject.walk(this.blobStorage, startHash, options) - } - - /** - * Find commit at or before a specific timestamp - * - * Uses index for fast O(log n) lookup - * - * @param ref - Starting ref (e.g., 'main') - * @param timestamp - Target timestamp - * @returns Commit at or before timestamp, or null - */ - async findAtTime(ref: string, timestamp: number): Promise { - // Build index if needed - await this.buildIndex(ref) - - // Binary search in index - const entries = Array.from(this.index.values()).sort( - (a, b) => b.timestamp - a.timestamp // Newest first - ) - - let left = 0 - let right = entries.length - 1 - let result: CommitIndexEntry | null = null - - while (left <= right) { - const mid = Math.floor((left + right) / 2) - const entry = entries[mid] - - if (entry.timestamp <= timestamp) { - result = entry - right = mid - 1 // Look for newer commit - } else { - left = mid + 1 // Look for older commit - } - } - - if (!result) { - return null - } - - // Read full commit - return CommitObject.read(this.blobStorage, result.hash) - } - - /** - * Get commit by hash - * - * @param hash - Commit hash - * @returns Commit object - */ - async getCommit(hash: string): Promise { - return CommitObject.read(this.blobStorage, hash) - } - - /** - * Get commits in time range - * - * @param ref - Starting ref - * @param startTime - Start of time range - * @param endTime - End of time range - * @returns Array of commits in range (newest first) - */ - async getInTimeRange( - ref: string, - startTime: number, - endTime: number - ): Promise { - const commits: CommitObject[] = [] - - for await (const commit of this.walk(ref, { until: startTime })) { - if (commit.timestamp >= startTime && commit.timestamp <= endTime) { - commits.push(commit) - } - } - - return commits - } - - /** - * Get commits by author - * - * @param ref - Starting ref - * @param author - Author name - * @param options - Additional options - * @returns Array of commits by author - */ - async getByAuthor( - ref: string, - author: string, - options?: { maxCount?: number; since?: number } - ): Promise { - const commits: CommitObject[] = [] - let count = 0 - - for await (const commit of this.walk(ref, { until: options?.since })) { - if (commit.author === author) { - commits.push(commit) - count++ - - if (options?.maxCount && count >= options.maxCount) { - break - } - } - } - - return commits - } - - /** - * Get commits by operation type - * - * @param ref - Starting ref - * @param operation - Operation type (e.g., 'add', 'update', 'delete') - * @param options - Additional options - * @returns Array of commits by operation - */ - async getByOperation( - ref: string, - operation: string, - options?: { maxCount?: number; since?: number } - ): Promise { - const commits: CommitObject[] = [] - let count = 0 - - for await (const commit of this.walk(ref, { until: options?.since })) { - if (commit.metadata?.operation === operation) { - commits.push(commit) - count++ - - if (options?.maxCount && count >= options.maxCount) { - break - } - } - } - - return commits - } - - /** - * Get commit history as array - * - * @param ref - Starting ref - * @param options - Walk options - * @returns Array of commits (newest first) - */ - async getHistory( - ref: string, - options?: { - maxCount?: number - since?: number - until?: number - } - ): Promise { - const commits: CommitObject[] = [] - let count = 0 - - for await (const commit of this.walk(ref, { - maxDepth: options?.maxCount, - until: options?.until - })) { - if (options?.since && commit.timestamp < options.since) { - continue - } - - commits.push(commit) - count++ - - if (options?.maxCount && count >= options.maxCount) { - break - } - } - - return commits - } - - /** - * Stream commit history (memory-efficient for large histories) - * - * Yields commits one at a time without accumulating in memory. - * Use this for large commit histories (1000s of commits) where - * memory efficiency is important. - * - * @param ref - Starting ref - * @param options - Walk options - * @yields Commits in reverse chronological order (newest first) - * - * @example - * ```typescript - * // Stream all commits without memory accumulation - * for await (const commit of commitLog.streamHistory('main', { maxCount: 10000 })) { - * console.log(commit.message) - * } - * ``` - */ - async *streamHistory( - ref: string, - options?: { - maxCount?: number - since?: number - until?: number - } - ): AsyncIterableIterator { - let count = 0 - - for await (const commit of this.walk(ref, { - maxDepth: options?.maxCount, - until: options?.until - })) { - // Filter by since timestamp if provided - if (options?.since && commit.timestamp < options.since) { - continue - } - - yield commit - count++ - - // Stop after maxCount commits - if (options?.maxCount && count >= options.maxCount) { - break - } - } - } - - /** - * Count commits between two commits - * - * @param fromRef - Starting ref/commit - * @param toRef - Ending ref/commit (optional, defaults to fromRef's parent) - * @returns Number of commits between - */ - async countBetween(fromRef: string, toRef?: string): Promise { - const fromHash = await this.resolveToHash(fromRef) - const toHash = toRef ? await this.resolveToHash(toRef) : null - - return CommitObject.countBetween(this.blobStorage, fromHash, toHash!) - } - - /** - * Find common ancestor of two commits (merge base) - * - * @param ref1 - First ref/commit - * @param ref2 - Second ref/commit - * @returns Common ancestor commit or null - */ - async findCommonAncestor( - ref1: string, - ref2: string - ): Promise { - const hash1 = await this.resolveToHash(ref1) - const hash2 = await this.resolveToHash(ref2) - - return CommitObject.findCommonAncestor(this.blobStorage, hash1, hash2) - } - - /** - * Get commit log statistics - * - * @param ref - Starting ref - * @param options - Options - * @returns Commit log statistics - */ - async getStats( - ref: string = 'main', - options?: { maxDepth?: number } - ): Promise { - const authors = new Set() - const operations = new Set() - const timestamps: number[] = [] - let totalCommits = 0 - - for await (const commit of this.walk(ref, options)) { - totalCommits++ - authors.add(commit.author) - timestamps.push(commit.timestamp) - - if (commit.metadata?.operation) { - operations.add(commit.metadata.operation) - } - } - - // Calculate average interval - let avgInterval = 0 - if (timestamps.length > 1) { - const intervals: number[] = [] - for (let i = 0; i < timestamps.length - 1; i++) { - intervals.push(timestamps[i] - timestamps[i + 1]) - } - avgInterval = intervals.reduce((a, b) => a + b, 0) / intervals.length - } - - return { - totalCommits, - oldestCommit: timestamps[timestamps.length - 1] ?? 0, - newestCommit: timestamps[0] ?? 0, - authors, - operations, - avgCommitInterval: avgInterval - } - } - - /** - * Check if commit is ancestor of another commit - * - * @param ancestorRef - Potential ancestor ref/commit - * @param descendantRef - Descendant ref/commit - * @returns True if ancestor is in descendant's history - */ - async isAncestor(ancestorRef: string, descendantRef: string): Promise { - const ancestorHash = await this.resolveToHash(ancestorRef) - const descendantHash = await this.resolveToHash(descendantRef) - - // Walk from descendant, check if we encounter ancestor - for await (const commit of this.walk(descendantHash)) { - const commitHash = CommitObject.hash(commit) - if (commitHash === ancestorHash) { - return true - } - } - - return false - } - - /** - * Get recent commits (last N) - * - * @param ref - Starting ref - * @param count - Number of commits to retrieve - * @returns Array of recent commits - */ - async getRecent(ref: string, count: number = 10): Promise { - return this.getHistory(ref, { maxCount: count }) - } - - /** - * Find commits with tag - * - * @param ref - Starting ref - * @param tag - Tag to search for - * @returns Array of commits with tag - */ - async findWithTag(ref: string, tag: string): Promise { - const commits: CommitObject[] = [] - - for await (const commit of this.walk(ref)) { - if (CommitObject.hasTag(commit, tag)) { - commits.push(commit) - } - } - - return commits - } - - /** - * Get first (oldest) commit - * - * @param ref - Starting ref - * @returns Oldest commit - */ - async getFirstCommit(ref: string): Promise { - let oldest: CommitObject | null = null - - for await (const commit of this.walk(ref)) { - oldest = commit - } - - return oldest - } - - /** - * Get latest commit - * - * @param ref - Starting ref - * @returns Latest commit - */ - async getLatestCommit(ref: string): Promise { - for await (const commit of this.walk(ref, { maxDepth: 1 })) { - return commit - } - - return null - } - - /** - * Clear index (useful for testing, after new commits) - */ - clearIndex(): void { - this.index.clear() - this.indexValid = false - } - - // ========== PRIVATE METHODS ========== - - /** - * Build commit index for fast lookups - * - * @param ref - Starting ref - */ - private async buildIndex(ref: string): Promise { - if (this.indexValid) { - return // Already built - } - - this.index.clear() - - for await (const commit of this.walk(ref)) { - const hash = CommitObject.hash(commit) - - this.index.set(hash, { - hash, - timestamp: commit.timestamp, - parentHash: commit.parent - }) - } - - this.indexValid = true - } - - /** - * Resolve ref or hash to commit hash - * - * @param refOrHash - Ref name or commit hash - * @returns Commit hash - */ - private async resolveToHash(refOrHash: string): Promise { - if (/^[a-f0-9]{64}$/.test(refOrHash)) { - return refOrHash // Already a hash - } - - const hash = await this.refManager.resolveRef(refOrHash) - - if (!hash) { - throw new Error(`Ref not found: ${refOrHash}`) - } - - return hash - } -} diff --git a/src/storage/cow/CommitObject.ts b/src/storage/cow/CommitObject.ts deleted file mode 100644 index 57645695..00000000 --- a/src/storage/cow/CommitObject.ts +++ /dev/null @@ -1,563 +0,0 @@ -/** - * CommitObject: Snapshot metadata for COW (Copy-on-Write) - * - * Similar to Git commits, represents a point-in-time snapshot of a Brainy instance. - * Commits reference tree objects and parent commits, forming a directed acyclic graph (DAG). - * - * Structure: - * - tree: hash of root tree object (contains all data) - * - parent: hash of parent commit (null for initial commit) - * - message: human-readable commit message - * - author: who/what created this commit - * - timestamp: when this commit was created - * - metadata: additional commit metadata (tags, etc.) - * - * @module storage/cow/CommitObject - */ - -import { BlobStorage } from './BlobStorage.js' -import { isNullHash } from './constants.js' - -/** - * Commit object structure - */ -export interface CommitObject { - tree: string // Root tree hash - parent: string | null // Parent commit hash (null for initial commit) - message: string // Commit message - author: string // Author (user, system, augmentation) - timestamp: number // Unix timestamp (milliseconds) - metadata?: { - tags?: string[] // Tags (e.g., ['v1.0.0', 'production']) - branch?: string // Branch name (e.g., 'main', 'experiment') - operation?: string // Operation type (e.g., 'add', 'update', 'delete', 'merge') - entityCount?: number // Number of entities at this commit - relationshipCount?: number // Number of relationships at this commit - [key: string]: any // Custom metadata - } -} - -/** - * CommitBuilder: Fluent API for building commit objects - * - * Example: - * ```typescript - * const commit = await CommitBuilder.create(blobStorage) - * .tree(treeHash) - * .parent(parentHash) - * .message('Add user entities') - * .author('system') - * .tag('v1.0.0') - * .build() - * ``` - */ -export class CommitBuilder { - private _tree?: string - private _parent?: string | null - private _message: string = 'Auto-commit' - private _author: string = 'system' - private _timestamp: number = Date.now() - private _metadata: NonNullable = {} - private blobStorage: BlobStorage - - constructor(blobStorage: BlobStorage) { - this.blobStorage = blobStorage - } - - static create(blobStorage: BlobStorage): CommitBuilder { - return new CommitBuilder(blobStorage) - } - - /** - * Set tree hash - */ - tree(hash: string): this { - this._tree = hash - return this - } - - /** - * Set parent commit hash (null for initial commit) - */ - parent(hash: string | null): this { - this._parent = hash - return this - } - - /** - * Set commit message - */ - message(message: string): this { - this._message = message - return this - } - - /** - * Set commit author - */ - author(author: string): this { - this._author = author - return this - } - - /** - * Set commit timestamp (defaults to now) - */ - timestamp(timestamp: number | Date): this { - this._timestamp = typeof timestamp === 'number' ? timestamp : timestamp.getTime() - return this - } - - /** - * Add tag to commit - */ - tag(tag: string): this { - if (!this._metadata.tags) { - this._metadata.tags = [] - } - this._metadata.tags.push(tag) - return this - } - - /** - * Set branch name - */ - branch(branch: string): this { - this._metadata.branch = branch - return this - } - - /** - * Set operation type - */ - operation(operation: string): this { - this._metadata.operation = operation - return this - } - - /** - * Set entity count - */ - entityCount(count: number): this { - this._metadata.entityCount = count - return this - } - - /** - * Set relationship count - */ - relationshipCount(count: number): this { - this._metadata.relationshipCount = count - return this - } - - /** - * Set custom metadata - */ - meta(key: string, value: any): this { - this._metadata[key] = value - return this - } - - /** - * Build and persist the commit object - * - * @returns Commit hash - */ - async build(): Promise { - if (!this._tree) { - throw new Error('CommitBuilder: tree hash is required') - } - - const commit: CommitObject = { - tree: this._tree, - parent: this._parent ?? null, - message: this._message, - author: this._author, - timestamp: this._timestamp, - metadata: Object.keys(this._metadata).length > 0 ? this._metadata : undefined - } - - return CommitObject.write(this.blobStorage, commit) - } -} - -/** - * CommitObject: Represents a point-in-time snapshot in COW storage - */ -export class CommitObject { - /** - * Serialize commit object to Buffer - * - * Format: JSON (simple, debuggable) - * Future: Could use protobuf for efficiency - * - * @param commit - Commit object - * @returns Serialized commit - */ - static serialize(commit: CommitObject): Buffer { - return Buffer.from(JSON.stringify(commit, null, 0)) // Compact JSON - } - - /** - * Deserialize commit object from Buffer - * - * @param data - Serialized commit - * @returns Commit object - */ - static deserialize(data: Buffer): CommitObject { - const commit = JSON.parse(data.toString()) - - // Validate structure - if (typeof commit.tree !== 'string' || commit.tree.length !== 64) { - throw new Error('Invalid commit object: tree must be 64-char SHA-256') - } - - if (commit.parent !== null && (typeof commit.parent !== 'string' || commit.parent.length !== 64)) { - throw new Error('Invalid commit object: parent must be 64-char SHA-256 or null') - } - - if (typeof commit.message !== 'string') { - throw new Error('Invalid commit object: message must be string') - } - - if (typeof commit.author !== 'string') { - throw new Error('Invalid commit object: author must be string') - } - - if (typeof commit.timestamp !== 'number') { - throw new Error('Invalid commit object: timestamp must be number') - } - - if (commit.metadata !== undefined && typeof commit.metadata !== 'object') { - throw new Error('Invalid commit object: metadata must be object') - } - - return commit - } - - /** - * Compute hash of commit object - * - * @param commit - Commit object - * @returns SHA-256 hash - */ - static hash(commit: CommitObject): string { - const data = CommitObject.serialize(commit) - return BlobStorage.hash(data) - } - - /** - * Write commit object to blob storage - * - * @param blobStorage - Blob storage instance - * @param commit - Commit object - * @returns Commit hash - */ - static async write(blobStorage: BlobStorage, commit: CommitObject): Promise { - const data = CommitObject.serialize(commit) - return blobStorage.write(data, { type: 'commit', compression: 'auto' }) - } - - /** - * Read commit object from blob storage - * - * @param blobStorage - Blob storage instance - * @param hash - Commit hash - * @returns Commit object - */ - static async read(blobStorage: BlobStorage, hash: string): Promise { - const data = await blobStorage.read(hash) - return CommitObject.deserialize(data) - } - - /** - * Check if commit is initial commit (has no parent) - * - * @param commit - Commit object - * @returns True if initial commit - */ - static isInitial(commit: CommitObject): boolean { - return commit.parent === null - } - - /** - * Check if commit is merge commit (has multiple parents) - * (Future enhancement: support merge commits with multiple parents) - * - * @param commit - Commit object - * @returns True if merge commit - */ - static isMerge(commit: CommitObject): boolean { - // For now, we only support single-parent commits - // Future: extend to support multiple parents - return commit.metadata?.merge !== undefined - } - - /** - * Check if commit has a specific tag - * - * @param commit - Commit object - * @param tag - Tag to check - * @returns True if commit has tag - */ - static hasTag(commit: CommitObject, tag: string): boolean { - return commit.metadata?.tags?.includes(tag) ?? false - } - - /** - * Get all tags from commit - * - * @param commit - Commit object - * @returns Array of tags - */ - static getTags(commit: CommitObject): string[] { - return commit.metadata?.tags ?? [] - } - - /** - * Walk commit history (traverse DAG) - * - * Yields commits in reverse chronological order (newest first) - * - * @param blobStorage - Blob storage instance - * @param startHash - Starting commit hash - * @param options - Walk options - */ - static async *walk( - blobStorage: BlobStorage, - startHash: string, - options?: { - maxDepth?: number // Maximum depth to walk - until?: number // Stop at this timestamp - stopAt?: string // Stop at this commit hash - filter?: (commit: CommitObject) => boolean - } - ): AsyncIterableIterator { - let currentHash: string | null = startHash - let depth = 0 - - // Guard against NULL hash (sentinel for "no parent") - // The initial commit has parent = null or NULL_HASH ('0000...0000') - // We must stop walking when we reach it, not try to read it - while (currentHash && !isNullHash(currentHash)) { - // Check max depth - if (options?.maxDepth && depth >= options.maxDepth) { - break - } - - // Read commit - const commit = await CommitObject.read(blobStorage, currentHash) - - // Check filter - if (options?.filter && !options.filter(commit)) { - currentHash = commit.parent - depth++ - continue - } - - // Yield commit - yield commit - - // Check stop conditions - if (options?.until && commit.timestamp < options.until) { - break - } - - if (options?.stopAt && currentHash === options.stopAt) { - break - } - - // Move to parent (can be null or NULL_HASH for initial commit) - currentHash = commit.parent - depth++ - } - } - - /** - * Find commit at or before a specific timestamp - * - * @param blobStorage - Blob storage instance - * @param startHash - Starting commit hash (e.g., 'main' ref) - * @param timestamp - Target timestamp - * @returns Commit at or before timestamp, or null if not found - */ - static async findAtTime( - blobStorage: BlobStorage, - startHash: string, - timestamp: number - ): Promise { - for await (const commit of CommitObject.walk(blobStorage, startHash)) { - if (commit.timestamp <= timestamp) { - return commit - } - } - - return null - } - - /** - * Get commit history as array (newest first) - * - * @param blobStorage - Blob storage instance - * @param startHash - Starting commit hash - * @param options - Walk options - * @returns Array of commits - */ - static async getHistory( - blobStorage: BlobStorage, - startHash: string, - options?: { - maxDepth?: number - until?: number - stopAt?: string - } - ): Promise { - const commits: CommitObject[] = [] - - for await (const commit of CommitObject.walk(blobStorage, startHash, options)) { - commits.push(commit) - } - - return commits - } - - /** - * Find common ancestor of two commits (merge base) - * - * Useful for diff/merge operations - * - * @param blobStorage - Blob storage instance - * @param hash1 - First commit hash - * @param hash2 - Second commit hash - * @returns Common ancestor commit, or null if not found - */ - static async findCommonAncestor( - blobStorage: BlobStorage, - hash1: string, - hash2: string - ): Promise { - // Build set of all ancestors of commit1 - const ancestors1 = new Set() - - for await (const commit of CommitObject.walk(blobStorage, hash1)) { - ancestors1.add(CommitObject.hash(commit)) - } - - // Walk commit2 history, find first commit in ancestors1 - for await (const commit of CommitObject.walk(blobStorage, hash2)) { - const commitHash = CommitObject.hash(commit) - if (ancestors1.has(commitHash)) { - return commit - } - } - - return null - } - - /** - * Count commits between two commits - * - * @param blobStorage - Blob storage instance - * @param fromHash - Starting commit hash - * @param toHash - Ending commit hash - * @returns Number of commits between (inclusive) - */ - static async countBetween( - blobStorage: BlobStorage, - fromHash: string, - toHash: string - ): Promise { - let count = 0 - - for await (const commit of CommitObject.walk(blobStorage, fromHash, { - stopAt: toHash - })) { - count++ - } - - return count - } - - /** - * Get commits in time range - * - * @param blobStorage - Blob storage instance - * @param startHash - Starting commit hash - * @param startTime - Start of time range - * @param endTime - End of time range - * @returns Array of commits in range - */ - static async getInTimeRange( - blobStorage: BlobStorage, - startHash: string, - startTime: number, - endTime: number - ): Promise { - const commits: CommitObject[] = [] - - for await (const commit of CommitObject.walk(blobStorage, startHash, { - until: startTime - })) { - if (commit.timestamp >= startTime && commit.timestamp <= endTime) { - commits.push(commit) - } - } - - return commits - } - - /** - * Get commits by author - * - * @param blobStorage - Blob storage instance - * @param startHash - Starting commit hash - * @param author - Author name - * @param options - Walk options - * @returns Array of commits by author - */ - static async getByAuthor( - blobStorage: BlobStorage, - startHash: string, - author: string, - options?: { maxDepth?: number } - ): Promise { - const commits: CommitObject[] = [] - - for await (const commit of CommitObject.walk(blobStorage, startHash, { - ...options, - filter: c => c.author === author - })) { - commits.push(commit) - } - - return commits - } - - /** - * Get commits by operation type - * - * @param blobStorage - Blob storage instance - * @param startHash - Starting commit hash - * @param operation - Operation type (e.g., 'add', 'update', 'delete') - * @param options - Walk options - * @returns Array of commits by operation - */ - static async getByOperation( - blobStorage: BlobStorage, - startHash: string, - operation: string, - options?: { maxDepth?: number } - ): Promise { - const commits: CommitObject[] = [] - - for await (const commit of CommitObject.walk(blobStorage, startHash, { - ...options, - filter: c => c.metadata?.operation === operation - })) { - commits.push(commit) - } - - return commits - } -} diff --git a/src/storage/cow/RefManager.ts b/src/storage/cow/RefManager.ts deleted file mode 100644 index dfd013d8..00000000 --- a/src/storage/cow/RefManager.ts +++ /dev/null @@ -1,547 +0,0 @@ -/** - * RefManager: Branch and reference management for COW (Copy-on-Write) - * - * Similar to Git refs, manages symbolic names (branches, tags) that point to commits. - * - * Structure: - * - refs/heads/main → commit hash (main branch) - * - refs/heads/experiment → commit hash (experiment branch) - * - refs/tags/v1.0.0 → commit hash (version tag) - * - HEAD → ref name (current branch) - * - * Features: - * - Branch management (create, delete, list) - * - Tag management - * - HEAD pointer (current branch) - * - Fast-forward and force updates - * - Atomic operations - * - * @module storage/cow/RefManager - */ - -import type { COWStorageAdapter } from './BlobStorage.js' -import { compareCodePoints } from '../../utils/collation.js' - -/** - * Reference type - */ -export type RefType = 'branch' | 'tag' | 'remote' - -/** - * Reference object - */ -export interface Ref { - name: string // Full ref name (e.g., 'refs/heads/main') - commitHash: string // Commit hash this ref points to - type: RefType // Reference type - createdAt: number // When ref was created - updatedAt: number // When ref was last updated - metadata?: { - description?: string - author?: string - [key: string]: any - } -} - -/** - * Ref update options - */ -export interface RefUpdateOptions { - force?: boolean // Force update (allow non-fast-forward) - createOnly?: boolean // Only create, fail if exists - updateOnly?: boolean // Only update, fail if doesn't exist - expectedOldValue?: string // CAS: only update if current value matches -} - -/** - * RefManager: Manages branches, tags, and HEAD pointer - * - * Pure implementation - no backward compatibility - */ -export class RefManager { - private adapter: COWStorageAdapter - private cache: Map - private cacheValid: boolean - - constructor(adapter: COWStorageAdapter) { - this.adapter = adapter - this.cache = new Map() - this.cacheValid = false - } - - /** - * Get reference by name - * - * @param name - Reference name (e.g., 'main', 'refs/heads/main') - * @returns Reference object or undefined - */ - async getRef(name: string): Promise { - const fullName = this.normalizeRefName(name) - - // Check cache - if (this.cacheValid && this.cache.has(fullName)) { - return this.cache.get(fullName) - } - - // Read from storage - const data = await this.adapter.get(`ref:${fullName}`) - - if (!data) { - return undefined - } - - const ref = JSON.parse(data.toString()) as Ref - - // Update cache - this.cache.set(fullName, ref) - - return ref - } - - /** - * Set reference to point to commit - * - * @param name - Reference name - * @param commitHash - Commit hash to point to - * @param options - Update options - */ - async setRef( - name: string, - commitHash: string, - options: RefUpdateOptions = {} - ): Promise { - const fullName = this.normalizeRefName(name) - - // Validate commit hash format - if (!/^[a-f0-9]{64}$/.test(commitHash)) { - throw new Error(`Invalid commit hash: ${commitHash}`) - } - - // Check if ref exists - const existing = await this.getRef(fullName) - - - // Handle createOnly - if (options.createOnly && existing) { - throw new Error(`Ref already exists: ${fullName}`) - } - - // Handle updateOnly - if (options.updateOnly && !existing) { - throw new Error(`Ref does not exist: ${fullName}`) - } - - // Handle CAS (Compare-And-Swap) - if (options.expectedOldValue !== undefined) { - if (!existing || existing.commitHash !== options.expectedOldValue) { - throw new Error( - `Ref update failed: expected ${options.expectedOldValue}, ` + - `got ${existing?.commitHash ?? 'none'}` - ) - } - } - - // Fast-forward verification is intentionally permissive: COW is a - // single-writer subsystem locked by the storage adapter's writer lock, - // so out-of-order updates can't reach this point. If multi-writer COW - // is ever introduced, replace with a commit-ancestry walk. - - // Create/update ref - const ref: Ref = { - name: fullName, - commitHash, - type: this.getRefType(fullName), - createdAt: existing?.createdAt ?? Date.now(), - updatedAt: Date.now(), - metadata: existing?.metadata - } - - // Write to storage - await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref))) - - // Update cache - this.cache.set(fullName, ref) - this.cacheValid = false // Invalidate for listRefs - } - - /** - * Delete reference - * - * @param name - Reference name - */ - async deleteRef(name: string): Promise { - const fullName = this.normalizeRefName(name) - - // Don't allow deleting HEAD - if (fullName === 'HEAD') { - throw new Error('Cannot delete HEAD') - } - - // Don't allow deleting main if it's the only branch - if (fullName === 'refs/heads/main') { - const branches = await this.listRefs('branch') - if (branches.length === 1) { - throw new Error('Cannot delete last branch') - } - } - - // Delete from storage - await this.adapter.delete(`ref:${fullName}`) - - // Update cache - this.cache.delete(fullName) - this.cacheValid = false - } - - /** - * List all references - * - * @param type - Filter by type (optional) - * @returns Array of references - */ - async listRefs(type?: RefType): Promise { - // Get all ref keys - const keys = await this.adapter.list('ref:') - - const refs: Ref[] = [] - - for (const key of keys) { - const refName = key.replace(/^ref:/, '') - - // Skip HEAD in listings (it's special) - if (refName === 'HEAD') { - continue - } - - const ref = await this.getRef(refName) - - if (ref) { - // Filter by type if requested - if (!type || ref.type === type) { - refs.push(ref) - } - } - } - - // Mark cache as valid - this.cacheValid = true - - return refs.sort((a, b) => compareCodePoints(a.name, b.name)) - } - - /** - * Copy reference (create branch from existing ref) - * - * @param sourceName - Source reference name - * @param targetName - Target reference name - * @param options - Update options - */ - async copyRef( - sourceName: string, - targetName: string, - options: RefUpdateOptions = {} - ): Promise { - const sourceRef = await this.getRef(sourceName) - - if (!sourceRef) { - throw new Error(`Source ref not found: ${sourceName}`) - } - - // Set target ref to same commit as source - await this.setRef(targetName, sourceRef.commitHash, options) - } - - /** - * Get current HEAD (current branch) - * - * @returns HEAD reference or undefined - */ - async getHead(): Promise { - const data = await this.adapter.get('ref:HEAD') - - if (!data) { - return undefined - } - - const head = JSON.parse(data.toString()) as { ref: string } - - // Resolve symbolic ref - return this.getRef(head.ref) - } - - /** - * Set HEAD to point to branch - * - * @param branchName - Branch name (e.g., 'main', 'refs/heads/experiment') - */ - async setHead(branchName: string): Promise { - const fullName = this.normalizeRefName(branchName) - - // Verify branch exists - const branch = await this.getRef(fullName) - - if (!branch) { - throw new Error(`Branch not found: ${fullName}`) - } - - if (branch.type !== 'branch') { - throw new Error(`Cannot set HEAD to non-branch ref: ${fullName}`) - } - - // Set HEAD (symbolic ref) - const head = { ref: fullName } - await this.adapter.put('ref:HEAD', Buffer.from(JSON.stringify(head))) - } - - /** - * Get current commit hash (resolves HEAD) - * - * @returns Current commit hash or undefined - */ - async getCurrentCommit(): Promise { - const head = await this.getHead() - return head?.commitHash - } - - /** - * Create branch - * - * @param name - Branch name (e.g., 'experiment') - * @param commitHash - Commit hash to point to - * @param options - Create options - */ - async createBranch( - name: string, - commitHash: string, - options?: { description?: string; author?: string } - ): Promise { - const fullName = this.normalizeRefName(name, 'branch') - - await this.setRef(fullName, commitHash, { createOnly: true }) - - // Update metadata if provided - if (options?.description || options?.author) { - const ref = await this.getRef(fullName) - if (ref) { - ref.metadata = { - ...ref.metadata, - description: options.description, - author: options.author - } - await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref))) - } - } - } - - /** - * Delete branch - * - * @param name - Branch name - */ - async deleteBranch(name: string): Promise { - const fullName = this.normalizeRefName(name, 'branch') - await this.deleteRef(fullName) - } - - /** - * List all branches - * - * @returns Array of branch references - */ - async listBranches(): Promise { - return this.listRefs('branch') - } - - /** - * Create tag - * - * @param name - Tag name (e.g., 'v1.0.0') - * @param commitHash - Commit hash to point to - * @param options - Create options - */ - async createTag( - name: string, - commitHash: string, - options?: { description?: string; author?: string } - ): Promise { - const fullName = this.normalizeRefName(name, 'tag') - - await this.setRef(fullName, commitHash, { createOnly: true }) - - // Update metadata if provided - if (options?.description || options?.author) { - const ref = await this.getRef(fullName) - if (ref) { - ref.metadata = { - ...ref.metadata, - description: options.description, - author: options.author - } - await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref))) - } - } - } - - /** - * Delete tag - * - * @param name - Tag name - */ - async deleteTag(name: string): Promise { - const fullName = this.normalizeRefName(name, 'tag') - await this.deleteRef(fullName) - } - - /** - * List all tags - * - * @returns Array of tag references - */ - async listTags(): Promise { - return this.listRefs('tag') - } - - /** - * Check if reference exists - * - * @param name - Reference name - * @returns True if reference exists - */ - async hasRef(name: string): Promise { - const ref = await this.getRef(name) - return ref !== undefined - } - - /** - * Update reference to new commit (with validation) - * - * @param name - Reference name - * @param newCommitHash - New commit hash - * @param oldCommitHash - Expected old commit hash (for safety) - */ - async updateRef( - name: string, - newCommitHash: string, - oldCommitHash?: string - ): Promise { - const options: RefUpdateOptions = {} - - if (oldCommitHash) { - options.expectedOldValue = oldCommitHash - } - - await this.setRef(name, newCommitHash, options) - } - - /** - * Update metadata on an existing ref (merge semantics). - * - * @param name - Reference name - * @param metadata - Metadata fields to merge into the ref - */ - async updateRefMetadata(name: string, metadata: Record): Promise { - const fullName = this.normalizeRefName(name) - const ref = await this.getRef(fullName) - if (!ref) throw new Error(`Ref not found: ${fullName}`) - ref.metadata = { ...ref.metadata, ...metadata } - ref.updatedAt = Date.now() - await this.adapter.put(`ref:${fullName}`, Buffer.from(JSON.stringify(ref))) - this.cache.set(fullName, ref) - } - - /** - * Get commit hash for reference - * - * @param name - Reference name - * @returns Commit hash or undefined - */ - async resolveRef(name: string): Promise { - const ref = await this.getRef(name) - return ref?.commitHash - } - - /** - * Find references pointing to commit - * - * @param commitHash - Commit hash - * @returns Array of references pointing to this commit - */ - async findRefsPointingTo(commitHash: string): Promise { - const allRefs = await this.listRefs() - return allRefs.filter(ref => ref.commitHash === commitHash) - } - - /** - * Clear cache (useful for testing) - */ - clearCache(): void { - this.cache.clear() - this.cacheValid = false - } - - // ========== PRIVATE METHODS ========== - - /** - * Normalize reference name to full format - * - * Examples: - * - 'main' → 'refs/heads/main' - * - 'v1.0.0' (with type='tag') → 'refs/tags/v1.0.0' - * - 'refs/heads/experiment' → 'refs/heads/experiment' - * - * @param name - Reference name - * @param type - Reference type hint - * @returns Full reference name - */ - private normalizeRefName(name: string, type?: RefType): string { - // Already full format - if (name.startsWith('refs/')) { - return name - } - - // HEAD is special - if (name === 'HEAD') { - return 'HEAD' - } - - // Infer type from name if not provided - if (!type) { - // Tags usually start with 'v' or contain dots - if (name.startsWith('v') && /\d/.test(name)) { - type = 'tag' - } else { - type = 'branch' // Default to branch - } - } - - // Add prefix - switch (type) { - case 'branch': - return `refs/heads/${name}` - case 'tag': - return `refs/tags/${name}` - case 'remote': - return `refs/remotes/${name}` - default: - return name - } - } - - /** - * Get reference type from full name - * - * @param fullName - Full reference name - * @returns Reference type - */ - private getRefType(fullName: string): RefType { - if (fullName.startsWith('refs/heads/')) { - return 'branch' - } else if (fullName.startsWith('refs/tags/')) { - return 'tag' - } else if (fullName.startsWith('refs/remotes/')) { - return 'remote' - } else { - return 'branch' // Default - } - } -} diff --git a/src/storage/cow/TreeObject.ts b/src/storage/cow/TreeObject.ts deleted file mode 100644 index 9f73b057..00000000 --- a/src/storage/cow/TreeObject.ts +++ /dev/null @@ -1,359 +0,0 @@ -/** - * TreeObject: Directory structure for COW (Copy-on-Write) - * - * Similar to Git trees, represents the structure of a Brainy instance at a point in time. - * Trees contain entries mapping names to blob hashes. - * - * Structure: - * - entities/ → tree hash (entity blobs) - * - indexes/nouns → blob hash (HNSW noun index) - * - indexes/metadata → blob hash (metadata index) - * - indexes/graph → blob hash (graph adjacency index) - * - indexes/deleted → blob hash (deleted items index) - * - * @module storage/cow/TreeObject - */ - -import { BlobStorage } from './BlobStorage.js' -import { compareCodePoints } from '../../utils/collation.js' - -/** - * Tree entry: name → blob hash mapping - */ -export interface TreeEntry { - name: string - hash: string - type: 'blob' | 'tree' // blob = leaf, tree = subtree - size: number // Original size (uncompressed) -} - -/** - * Tree object structure - */ -export interface TreeObject { - entries: TreeEntry[] - createdAt: number -} - -/** - * TreeBuilder: Fluent API for building tree objects - * - * Example: - * ```typescript - * const tree = await TreeBuilder.create(blobStorage) - * .addBlob('entities/abc123', entityHash, size) - * .addBlob('indexes/nouns', nounsHash, size) - * .build() - * ``` - */ -export class TreeBuilder { - private entries: TreeEntry[] = [] - private blobStorage: BlobStorage - - constructor(blobStorage: BlobStorage) { - this.blobStorage = blobStorage - } - - static create(blobStorage: BlobStorage): TreeBuilder { - return new TreeBuilder(blobStorage) - } - - /** - * Add a blob entry to the tree - * - * @param name - Entry name (e.g., 'entities/abc123') - * @param hash - Blob hash - * @param size - Original blob size - */ - addBlob(name: string, hash: string, size: number): this { - this.entries.push({ - name, - hash, - type: 'blob', - size - }) - return this - } - - /** - * Add a subtree entry to the tree - * - * @param name - Subtree name (e.g., 'entities/') - * @param treeHash - Tree hash - * @param size - Total size of subtree - */ - addTree(name: string, treeHash: string, size: number): this { - this.entries.push({ - name, - hash: treeHash, - type: 'tree', - size - }) - return this - } - - /** - * Build and persist the tree object - * - * @returns Tree hash - */ - async build(): Promise { - const tree: TreeObject = { - // Code-point (UTF-8 byte) order so the serialized tree — and therefore its - // content hash — is reproducible across environments (localeCompare's default - // locale varies by OS/Node/ICU). Sorting happens only here at write time; - // deserialize() preserves stored order, so existing trees keep their hashes - // and stay valid — no migration needed. - entries: this.entries.sort((a, b) => compareCodePoints(a.name, b.name)), - createdAt: Date.now() - } - - return TreeObject.write(this.blobStorage, tree) - } -} - -/** - * TreeObject: Represents directory structure in COW storage - */ -export class TreeObject { - /** - * Serialize tree object to Buffer - * - * Format: JSON (simple, debuggable) - * Future: Could use protobuf for efficiency - * - * @param tree - Tree object - * @returns Serialized tree - */ - static serialize(tree: TreeObject): Buffer { - return Buffer.from(JSON.stringify(tree, null, 0)) // Compact JSON - } - - /** - * Deserialize tree object from Buffer - * - * @param data - Serialized tree - * @returns Tree object - */ - static deserialize(data: Buffer): TreeObject { - const tree = JSON.parse(data.toString()) - - // Validate structure - if (!tree.entries || !Array.isArray(tree.entries)) { - throw new Error('Invalid tree object: missing entries array') - } - - if (typeof tree.createdAt !== 'number') { - throw new Error('Invalid tree object: missing or invalid createdAt') - } - - // Validate entries - for (const entry of tree.entries) { - if (typeof entry.name !== 'string') { - throw new Error(`Invalid tree entry: name must be string`) - } - if (typeof entry.hash !== 'string' || entry.hash.length !== 64) { - throw new Error(`Invalid tree entry: hash must be 64-char SHA-256`) - } - if (entry.type !== 'blob' && entry.type !== 'tree') { - throw new Error(`Invalid tree entry: type must be 'blob' or 'tree'`) - } - if (typeof entry.size !== 'number' || entry.size < 0) { - throw new Error(`Invalid tree entry: size must be non-negative number`) - } - } - - return tree - } - - /** - * Compute hash of tree object - * - * @param tree - Tree object - * @returns SHA-256 hash - */ - static hash(tree: TreeObject): string { - const data = TreeObject.serialize(tree) - return BlobStorage.hash(data) - } - - /** - * Write tree object to blob storage - * - * @param blobStorage - Blob storage instance - * @param tree - Tree object - * @returns Tree hash - */ - static async write(blobStorage: BlobStorage, tree: TreeObject): Promise { - const data = TreeObject.serialize(tree) - return blobStorage.write(data, { type: 'tree', compression: 'auto' }) - } - - /** - * Read tree object from blob storage - * - * @param blobStorage - Blob storage instance - * @param hash - Tree hash - * @returns Tree object - */ - static async read(blobStorage: BlobStorage, hash: string): Promise { - const data = await blobStorage.read(hash) - return TreeObject.deserialize(data) - } - - /** - * Get specific entry from tree - * - * @param tree - Tree object - * @param name - Entry name - * @returns Tree entry or undefined - */ - static getEntry(tree: TreeObject, name: string): TreeEntry | undefined { - return tree.entries.find(e => e.name === name) - } - - /** - * Get all blob entries from tree (non-recursive) - * - * @param tree - Tree object - * @returns Array of blob entries - */ - static getBlobs(tree: TreeObject): TreeEntry[] { - return tree.entries.filter(e => e.type === 'blob') - } - - /** - * Get all subtree entries from tree (non-recursive) - * - * @param tree - Tree object - * @returns Array of tree entries - */ - static getSubtrees(tree: TreeObject): TreeEntry[] { - return tree.entries.filter(e => e.type === 'tree') - } - - /** - * Walk tree recursively, yielding all blob entries - * - * @param blobStorage - Blob storage instance - * @param tree - Tree object - */ - static async *walk( - blobStorage: BlobStorage, - tree: TreeObject - ): AsyncIterableIterator { - for (const entry of tree.entries) { - if (entry.type === 'blob') { - yield entry - } else { - // Recurse into subtree - const subtree = await TreeObject.read(blobStorage, entry.hash) - yield* TreeObject.walk(blobStorage, subtree) - } - } - } - - /** - * Compute total size of tree (recursive) - * - * @param blobStorage - Blob storage instance - * @param tree - Tree object - * @returns Total size in bytes - */ - static async getTotalSize(blobStorage: BlobStorage, tree: TreeObject): Promise { - let totalSize = 0 - - for await (const entry of TreeObject.walk(blobStorage, tree)) { - totalSize += entry.size - } - - return totalSize - } - - /** - * Create a new tree by updating a single entry - * (Copy-on-write: creates new tree, doesn't modify original) - * - * @param blobStorage - Blob storage instance - * @param tree - Original tree - * @param name - Entry name to update - * @param hash - New blob/tree hash - * @param size - New size - * @returns New tree hash - */ - static async updateEntry( - blobStorage: BlobStorage, - tree: TreeObject, - name: string, - hash: string, - size: number - ): Promise { - const builder = TreeBuilder.create(blobStorage) - - // Copy all entries except the one being updated - for (const entry of tree.entries) { - if (entry.name === name) { - // Replace with new entry - if (entry.type === 'blob') { - builder.addBlob(name, hash, size) - } else { - builder.addTree(name, hash, size) - } - } else { - // Keep existing entry - if (entry.type === 'blob') { - builder.addBlob(entry.name, entry.hash, entry.size) - } else { - builder.addTree(entry.name, entry.hash, entry.size) - } - } - } - - // If entry didn't exist, add it - if (!TreeObject.getEntry(tree, name)) { - builder.addBlob(name, hash, size) - } - - return builder.build() - } - - /** - * Diff two trees, return changed/added/deleted entries - * - * @param tree1 - First tree (base) - * @param tree2 - Second tree (comparison) - * @returns Diff result - */ - static diff(tree1: TreeObject, tree2: TreeObject): { - added: TreeEntry[] - modified: TreeEntry[] - deleted: TreeEntry[] - } { - const entries1 = new Map(tree1.entries.map(e => [e.name, e])) - const entries2 = new Map(tree2.entries.map(e => [e.name, e])) - - const added: TreeEntry[] = [] - const modified: TreeEntry[] = [] - const deleted: TreeEntry[] = [] - - // Find added and modified - for (const [name, entry2] of entries2) { - const entry1 = entries1.get(name) - - if (!entry1) { - added.push(entry2) - } else if (entry1.hash !== entry2.hash) { - modified.push(entry2) - } - } - - // Find deleted - for (const [name, entry1] of entries1) { - if (!entries2.has(name)) { - deleted.push(entry1) - } - } - - return { added, modified, deleted } - } -} diff --git a/src/storage/cow/binaryDataCodec.ts b/src/storage/cow/binaryDataCodec.ts deleted file mode 100644 index 296daf2b..00000000 --- a/src/storage/cow/binaryDataCodec.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Binary Data Codec: Single Source of Truth for Wrap/Unwrap Operations - * - * This module provides the ONLY implementation of binary data encoding/decoding - * used across all storage adapters and blob storage. - * - * Design Principles: - * - DRY: One implementation, used everywhere - * - Single Responsibility: Only handles binary ↔ JSON conversion - * - Type-Safe: Proper TypeScript types - * - Defensive: Handles all edge cases - * - * Used by: - * - BaseStorage COW adapter (write/read operations) - * - BlobStorage (defense-in-depth verification) - * - All storage adapters (via BaseStorage) - * - * @module storage/cow/binaryDataCodec - */ - -/** - * Wrapped binary data format - * Used when storing binary data in JSON-based storage - */ -export interface WrappedBinaryData { - _binary: true - data: string // base64-encoded -} - -/** - * Check if data is wrapped binary format - */ -export function isWrappedBinary(data: any): data is WrappedBinaryData { - return ( - typeof data === 'object' && - data !== null && - data._binary === true && - typeof data.data === 'string' - ) -} - -/** - * Unwrap binary data from JSON wrapper - * - * This is the SINGLE SOURCE OF TRUTH for unwrapping binary data. - * All storage operations MUST use this function. - * - * Handles: - * - Buffer → Buffer (pass-through) - * - {_binary: true, data: "base64..."} → Buffer (unwrap) - * - Plain object → Buffer (JSON stringify) - * - Other types → Error - * - * @param data - Data to unwrap (may be Buffer, wrapped object, or plain object) - * @returns Unwrapped Buffer - * @throws Error if data type is invalid - */ -export function unwrapBinaryData(data: any): Buffer { - // Case 1: Already a Buffer (no unwrapping needed) - if (Buffer.isBuffer(data)) { - return data - } - - // Case 2: Wrapped binary data {_binary: true, data: "base64..."} - if (isWrappedBinary(data)) { - return Buffer.from(data.data, 'base64') - } - - // Case 3: Plain object (shouldn't happen for binary blobs, but handle gracefully) - if (typeof data === 'object' && data !== null) { - return Buffer.from(JSON.stringify(data)) - } - - // Case 4: String (convert to Buffer) - if (typeof data === 'string') { - return Buffer.from(data) - } - - // Case 5: Invalid type - throw new Error( - `Invalid data type for unwrap: ${typeof data}. ` + - `Expected Buffer or {_binary: true, data: "base64..."}` - ) -} - -/** - * Ensure data is a Buffer - * - * Convenience function that combines type checking and unwrapping. - * Use this when you need to ensure you have a Buffer. - * - * @param data - Data that should be or can be converted to Buffer - * @returns Buffer - */ -export function ensureBuffer(data: any): Buffer { - if (Buffer.isBuffer(data)) { - return data - } - return unwrapBinaryData(data) -} diff --git a/src/storage/cow/constants.ts b/src/storage/cow/constants.ts deleted file mode 100644 index 0dd05a42..00000000 --- a/src/storage/cow/constants.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * COW Storage Constants - * - * Sentinel values and utilities for Copy-On-Write storage system. - * - * @module storage/cow/constants - */ - -/** - * NULL_HASH - Sentinel value for "no parent commit" or "empty tree" - * - * In Git-like COW systems, we need a way to represent: - * - Initial commit (has no parent) - * - Empty tree (contains no files) - * - * We use a 64-character zero hash as a sentinel value. - * This should NEVER be used as an actual content hash. - * - * @constant - * @example - * ```typescript - * const builder = CommitBuilder.create(storage) - * .tree(NULL_HASH) // Empty tree - * .parent(null) // No parent (use null, not NULL_HASH) - * .build() - * ``` - */ -export const NULL_HASH = '0000000000000000000000000000000000000000000000000000000000000000' - -/** - * Check if a hash is the NULL sentinel value - * - * @param hash - Hash to check (can be string or null) - * @returns true if hash is null or NULL_HASH - * - * @example - * ```typescript - * if (isNullHash(commit.parent)) { - * console.log('This is the initial commit') - * } - * ``` - */ -export function isNullHash(hash: string | null | undefined): boolean { - return hash === null || hash === undefined || hash === NULL_HASH -} - -/** - * Check if a hash is valid (non-null, non-empty, proper format) - * - * @param hash - Hash to check - * @returns true if hash is a valid SHA-256 hash - */ -export function isValidHash(hash: string | null | undefined): boolean { - if (isNullHash(hash)) { - return false - } - // SHA-256 hash must be exactly 64 hexadecimal characters - return typeof hash === 'string' && /^[a-f0-9]{64}$/.test(hash) -} diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts index 2d6db868..1fbdf255 100644 --- a/src/storage/storageFactory.ts +++ b/src/storage/storageFactory.ts @@ -52,30 +52,10 @@ export interface StorageOptions { [key: string]: any } - /** Branch name for COW storage (filesystem only). */ - branch?: string - - /** Whether to enable COW blob compression. Default `true`. */ - enableCompression?: boolean - /** Operation tuning (timeouts, retry budgets) passed through to the adapter. */ operationConfig?: OperationConfig } -/** - * Attach the COW (copy-on-write) options to the adapter so the storage's - * branch + compression settings are honored when the adapter's - * `initializeCOW()` hook fires later in `brain.init()`. - */ -function configureCOW(storage: any, options?: StorageOptions): void { - if (typeof storage?.initializeCOW === 'function') { - storage._cowOptions = { - branch: options?.branch || 'main', - enableCompression: options?.enableCompression !== false - } - } -} - /** * Resolve `StorageOptions` to a concrete storage adapter. * @@ -85,9 +65,7 @@ function configureCOW(storage: any, options?: StorageOptions): void { * - `forceFileSystemStorage: true` returns `FileSystemStorage`. */ export async function createStorage(options: StorageOptions = {}): Promise { - const storage = await pickAdapter(options) - configureCOW(storage, options) - return storage + return pickAdapter(options) } async function pickAdapter(options: StorageOptions): Promise { diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 00aed0d2..52528a8a 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -20,8 +20,7 @@ import type { Operation, RollbackAction } from '../types.js' * * Rollback strategy: * - Remove item from index - * - * Note: Works with both JsHnswVectorIndex and TypeAwareHNSWIndex + */ export class AddToHNSWOperation implements Operation { readonly name = 'AddToHNSW' diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index 98902834..a75a2dd0 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -5,10 +5,9 @@ * Each operation can be executed and rolled back atomically. * * Supports: - * - All 8 storage adapters (FileSystem, S3, Memory, OPFS, GCS, Azure, R2, TypeAware) + * - Both storage adapters (FileSystem, Memory) * - Nouns (entities) and Verbs (relationships) * - Metadata and vector data - * - COW (Copy-on-Write) storage */ import type { StorageAdapter, HNSWNoun, HNSWVerb, NounMetadata, VerbMetadata } from '../../coreTypes.js' diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 17b5a33f..8cb46dd0 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1099,7 +1099,6 @@ export interface BrainyConfig { * including plugin-provided factories (e.g. Cortex mmap). */ rootDirectory?: string options?: any - branch?: string // COW branch name (default: 'main') } // Index configuration diff --git a/src/types/brainyInterface.ts b/src/types/brainyInterface.ts index 05e1cca6..c09d8fac 100644 --- a/src/types/brainyInterface.ts +++ b/src/types/brainyInterface.ts @@ -164,7 +164,8 @@ export interface BrainyInterface { /** * Run pending data migrations, or preview what would change. * - * @param options - Pass { dryRun: true } to preview without writing + * @param options - Pass `{ dryRun: true }` to preview without writing; + * pass `{ backupTo: path }` to persist a pre-migration snapshot. * @returns Migration result or preview depending on options * * @example @@ -173,9 +174,9 @@ export interface BrainyInterface { * const preview = await brain.migrate({ dryRun: true }) * console.log(preview.affectedEntities) * - * // Apply - * const result = await brain.migrate() - * console.log(result.backupBranch) // 'pre-migration-7.17.0' + * // Apply with a restore point + * const result = await brain.migrate({ backupTo: '/backups/pre-migration' }) + * console.log(result.backupPath) // '/backups/pre-migration' * ``` */ migrate(options?: MigrateOptions): Promise diff --git a/src/versioning/VersionDiff.ts b/src/versioning/VersionDiff.ts deleted file mode 100644 index 69f7dc15..00000000 --- a/src/versioning/VersionDiff.ts +++ /dev/null @@ -1,459 +0,0 @@ -/** - * VersionDiff - Deep Object Comparison for Entity Versions - * - * Provides deep diff between entity versions: - * - Field-level change detection - * - Nested object comparison - * - Array diffing - * - Type change detection - * - Human-readable diff output - * - * NO MOCKS - Production implementation - */ - -import type { NounMetadata } from '../coreTypes.js' - -/** - * Types of changes in a diff - */ -export type ChangeType = 'added' | 'removed' | 'modified' | 'type-changed' - -/** - * A single field change in a diff - */ -export interface FieldChange { - /** Path to the field (e.g., 'metadata.user.name') */ - path: string - - /** Type of change */ - type: ChangeType - - /** Old value (undefined for 'added') */ - oldValue?: any - - /** New value (undefined for 'removed') */ - newValue?: any - - /** Old type (for 'type-changed') */ - oldType?: string - - /** New type (for 'type-changed') */ - newType?: string -} - -/** - * Complete diff between two versions - */ -export interface VersionDiff { - /** Entity ID being compared */ - entityId: string - - /** From version number */ - fromVersion: number - - /** To version number */ - toVersion: number - - /** Fields that were added */ - added: FieldChange[] - - /** Fields that were removed */ - removed: FieldChange[] - - /** Fields that were modified */ - modified: FieldChange[] - - /** Fields whose type changed */ - typeChanged: FieldChange[] - - /** Total number of changes */ - totalChanges: number - - /** Whether versions are identical */ - identical: boolean -} - -/** - * Options for diff comparison - */ -export interface DiffOptions { - /** Entity ID (for context in output) */ - entityId: string - - /** From version number */ - fromVersion: number - - /** To version number */ - toVersion: number - - /** Ignore these fields in comparison */ - ignoreFields?: string[] - - /** Maximum depth for nested object comparison (default: 10) */ - maxDepth?: number - - /** Include unchanged fields in output (default: false) */ - includeUnchanged?: boolean -} - -/** - * Compare two entity versions and generate diff - * - * @param from Old version entity - * @param to New version entity - * @param options Diff options - * @returns Diff between versions - */ -export function compareEntityVersions( - from: NounMetadata, - to: NounMetadata, - options: DiffOptions -): VersionDiff { - const added: FieldChange[] = [] - const removed: FieldChange[] = [] - const modified: FieldChange[] = [] - const typeChanged: FieldChange[] = [] - - const ignoreFields = new Set(options.ignoreFields || []) - const maxDepth = options.maxDepth ?? 10 - - // Compare objects recursively - compareObjects(from, to, '', added, removed, modified, typeChanged, ignoreFields, 0, maxDepth) - - const totalChanges = added.length + removed.length + modified.length + typeChanged.length - const identical = totalChanges === 0 - - return { - entityId: options.entityId, - fromVersion: options.fromVersion, - toVersion: options.toVersion, - added, - removed, - modified, - typeChanged, - totalChanges, - identical - } -} - -/** - * Recursively compare two objects - */ -function compareObjects( - from: any, - to: any, - path: string, - added: FieldChange[], - removed: FieldChange[], - modified: FieldChange[], - typeChanged: FieldChange[], - ignoreFields: Set, - depth: number, - maxDepth: number -): void { - if (depth >= maxDepth) { - // Hit max depth - treat as single value - if (!deepEqual(from, to)) { - modified.push({ - path, - type: 'modified', - oldValue: from, - newValue: to - }) - } - return - } - - // Get all keys from both objects - const fromKeys = new Set(Object.keys(from || {})) - const toKeys = new Set(Object.keys(to || {})) - const allKeys = new Set([...fromKeys, ...toKeys]) - - for (const key of allKeys) { - const fieldPath = path ? `${path}.${key}` : key - - // Skip ignored fields - if (ignoreFields.has(fieldPath) || ignoreFields.has(key)) { - continue - } - - const fromHas = fromKeys.has(key) - const toHas = toKeys.has(key) - - if (!fromHas && toHas) { - // Field added - added.push({ - path: fieldPath, - type: 'added', - newValue: to[key] - }) - } else if (fromHas && !toHas) { - // Field removed - removed.push({ - path: fieldPath, - type: 'removed', - oldValue: from[key] - }) - } else { - // Field exists in both - check for changes - const fromValue = from[key] - const toValue = to[key] - - const fromType = getValueType(fromValue) - const toType = getValueType(toValue) - - if (fromType !== toType) { - // Type changed - typeChanged.push({ - path: fieldPath, - type: 'type-changed', - oldValue: fromValue, - newValue: toValue, - oldType: fromType, - newType: toType - }) - } else if (fromType === 'object' && toType === 'object') { - // Recursively compare nested objects - compareObjects( - fromValue, - toValue, - fieldPath, - added, - removed, - modified, - typeChanged, - ignoreFields, - depth + 1, - maxDepth - ) - } else if (fromType === 'array' && toType === 'array') { - // Compare arrays - if (!arraysEqual(fromValue, toValue)) { - modified.push({ - path: fieldPath, - type: 'modified', - oldValue: fromValue, - newValue: toValue - }) - } - } else { - // Primitive value comparison - if (!deepEqual(fromValue, toValue)) { - modified.push({ - path: fieldPath, - type: 'modified', - oldValue: fromValue, - newValue: toValue - }) - } - } - } - } -} - -/** - * Get human-readable type of a value - */ -function getValueType(value: any): string { - if (value === null) return 'null' - if (value === undefined) return 'undefined' - if (Array.isArray(value)) return 'array' - return typeof value -} - -/** - * Deep equality check - */ -function deepEqual(a: any, b: any): boolean { - if (a === b) return true - if (a === null || b === null) return false - if (a === undefined || b === undefined) return false - - const typeA = getValueType(a) - const typeB = getValueType(b) - - if (typeA !== typeB) return false - - if (typeA === 'array') { - return arraysEqual(a, b) - } - - if (typeA === 'object') { - return objectsEqual(a, b) - } - - // Primitive comparison - return a === b -} - -/** - * Compare arrays for equality - */ -function arraysEqual(a: any[], b: any[]): boolean { - if (a.length !== b.length) return false - - for (let i = 0; i < a.length; i++) { - if (!deepEqual(a[i], b[i])) { - return false - } - } - - return true -} - -/** - * Compare objects for equality - */ -function objectsEqual(a: any, b: any): boolean { - const keysA = Object.keys(a) - const keysB = Object.keys(b) - - if (keysA.length !== keysB.length) return false - - for (const key of keysA) { - if (!keysB.includes(key)) return false - if (!deepEqual(a[key], b[key])) return false - } - - return true -} - -/** - * Format diff as human-readable string - * - * @param diff Diff to format - * @returns Formatted string - */ -export function formatDiff(diff: VersionDiff): string { - const lines: string[] = [] - - lines.push(`Diff: ${diff.entityId} v${diff.fromVersion} → v${diff.toVersion}`) - lines.push('') - - if (diff.identical) { - lines.push('No changes') - return lines.join('\n') - } - - lines.push(`Total changes: ${diff.totalChanges}`) - lines.push('') - - if (diff.added.length > 0) { - lines.push(`Added (${diff.added.length}):`) - for (const change of diff.added) { - lines.push(` + ${change.path}: ${formatValue(change.newValue)}`) - } - lines.push('') - } - - if (diff.removed.length > 0) { - lines.push(`Removed (${diff.removed.length}):`) - for (const change of diff.removed) { - lines.push(` - ${change.path}: ${formatValue(change.oldValue)}`) - } - lines.push('') - } - - if (diff.modified.length > 0) { - lines.push(`Modified (${diff.modified.length}):`) - for (const change of diff.modified) { - lines.push(` ~ ${change.path}:`) - lines.push(` ${formatValue(change.oldValue)}`) - lines.push(` → ${formatValue(change.newValue)}`) - } - lines.push('') - } - - if (diff.typeChanged.length > 0) { - lines.push(`Type Changed (${diff.typeChanged.length}):`) - for (const change of diff.typeChanged) { - lines.push(` ! ${change.path}: ${change.oldType} → ${change.newType}`) - lines.push(` ${formatValue(change.oldValue)}`) - lines.push(` → ${formatValue(change.newValue)}`) - } - } - - return lines.join('\n') -} - -/** - * Format value for display - */ -function formatValue(value: any): string { - if (value === null) return 'null' - if (value === undefined) return 'undefined' - if (typeof value === 'string') return `"${value}"` - if (typeof value === 'object') { - try { - return JSON.stringify(value) - } catch { - return '[Object]' - } - } - return String(value) -} - -/** - * Get summary statistics about a diff - */ -export function getDiffStats(diff: VersionDiff): { - changedFields: number - addedFields: number - removedFields: number - modifiedFields: number - typeChangedFields: number -} { - return { - changedFields: diff.totalChanges, - addedFields: diff.added.length, - removedFields: diff.removed.length, - modifiedFields: diff.modified.length, - typeChangedFields: diff.typeChanged.length - } -} - -/** - * Check if diff has any changes - */ -export function hasChanges(diff: VersionDiff): boolean { - return !diff.identical -} - -/** - * Get all changed field paths - */ -export function getChangedPaths(diff: VersionDiff): string[] { - const paths = new Set() - - for (const change of diff.added) paths.add(change.path) - for (const change of diff.removed) paths.add(change.path) - for (const change of diff.modified) paths.add(change.path) - for (const change of diff.typeChanged) paths.add(change.path) - - return Array.from(paths).sort() -} - -/** - * Filter diff to only include specific paths - */ -export function filterDiff(diff: VersionDiff, paths: string[]): VersionDiff { - const pathSet = new Set(paths) - - const filterChanges = (changes: FieldChange[]) => - changes.filter((c) => pathSet.has(c.path) || paths.some((p) => c.path.startsWith(p + '.'))) - - const added = filterChanges(diff.added) - const removed = filterChanges(diff.removed) - const modified = filterChanges(diff.modified) - const typeChanged = filterChanges(diff.typeChanged) - - return { - ...diff, - added, - removed, - modified, - typeChanged, - totalChanges: added.length + removed.length + modified.length + typeChanged.length, - identical: added.length + removed.length + modified.length + typeChanged.length === 0 - } -} diff --git a/src/versioning/VersionIndex.ts b/src/versioning/VersionIndex.ts deleted file mode 100644 index b6bb10ce..00000000 --- a/src/versioning/VersionIndex.ts +++ /dev/null @@ -1,310 +0,0 @@ -/** - * VersionIndex - Pure Key-Value Version Storage - * - * Stores version metadata using simple key-value storage: - * - Version list per entity: __version_meta_{entityId}_{branch} - * - Content stored separately by VersionStorage - * - * Key Design Decisions: - * - Versions are NOT entities (no brain.add()) - * - Versions do NOT pollute find() results - * - Simple O(1) lookups per entity - * - Versions per entity is typically small (10-1000) - * - * NO MOCKS - Production implementation - */ - -import type { EntityVersion, VersionQuery } from './VersionManager.js' - -/** - * Internal storage structure for version metadata - */ -interface VersionMetadataStore { - entityId: string - branch: string - versions: VersionEntry[] -} - -/** - * Individual version entry in the store - */ -interface VersionEntry { - version: number - timestamp: number - contentHash: string - commitHash?: string - tag?: string - description?: string - author?: string -} - -/** - * VersionIndex - Pure key-value version metadata storage - * - * Uses simple JSON storage instead of creating entities. - * This ensures versions never appear in find() results. - */ -export class VersionIndex { - private brain: any // Brainy instance - private initialized: boolean = false - - constructor(brain: any) { - this.brain = brain - } - - /** - * Initialize version index - */ - async initialize(): Promise { - if (this.initialized) return - this.initialized = true - } - - /** - * Add version to index - * - * Stores version entry in key-value storage. - * Handles deduplication by content hash. - * - * @param version Version metadata to store - */ - async addVersion(version: EntityVersion): Promise { - await this.initialize() - - const key = this.getMetaKey(version.entityId, version.branch) - const store = await this.loadStore(key) || { - entityId: version.entityId, - branch: version.branch, - versions: [] - } - - // Check for duplicate content hash (deduplication) - const existing = store.versions.find(v => v.contentHash === version.contentHash) - if (existing) { - // Update tag/description if provided on duplicate save - let updated = false - if (version.tag && version.tag !== existing.tag) { - existing.tag = version.tag - updated = true - } - if (version.description && version.description !== existing.description) { - existing.description = version.description - updated = true - } - if (updated) { - await this.saveStore(key, store) - } - return - } - - // Add new version entry - store.versions.push({ - version: version.version, - timestamp: version.timestamp, - contentHash: version.contentHash, - commitHash: version.commitHash, - tag: version.tag, - description: version.description, - author: version.author - }) - - await this.saveStore(key, store) - } - - /** - * Get versions for an entity - * - * @param query Version query with filters - * @returns List of versions (newest first) - */ - async getVersions(query: VersionQuery): Promise { - await this.initialize() - - const branch = query.branch || this.brain.currentBranch - const key = this.getMetaKey(query.entityId, branch) - const store = await this.loadStore(key) - if (!store) return [] - - // Convert entries to EntityVersion format - let versions: EntityVersion[] = store.versions.map(entry => ({ - version: entry.version, - entityId: store.entityId, - branch: store.branch, - timestamp: entry.timestamp, - contentHash: entry.contentHash, - commitHash: entry.commitHash || '', - tag: entry.tag, - description: entry.description, - author: entry.author - })) - - // Apply filters - if (query.tag) { - versions = versions.filter(v => v.tag === query.tag) - } - if (query.startDate) { - versions = versions.filter(v => v.timestamp >= query.startDate!) - } - if (query.endDate) { - versions = versions.filter(v => v.timestamp <= query.endDate!) - } - - // Sort newest first (highest version number first) - versions.sort((a, b) => b.version - a.version) - - // Apply pagination - const start = query.offset || 0 - const end = query.limit ? start + query.limit : undefined - return versions.slice(start, end) - } - - /** - * Get specific version - * - * @param entityId Entity ID - * @param version Version number - * @param branch Branch name - * @returns Version metadata or null - */ - async getVersion( - entityId: string, - version: number, - branch: string - ): Promise { - const versions = await this.getVersions({ entityId, branch }) - return versions.find(v => v.version === version) || null - } - - /** - * Get version by tag - * - * @param entityId Entity ID - * @param tag Version tag - * @param branch Branch name - * @returns Version metadata or null - */ - async getVersionByTag( - entityId: string, - tag: string, - branch: string - ): Promise { - const versions = await this.getVersions({ entityId, branch, tag }) - return versions[0] || null - } - - /** - * Get version count for entity - * - * @param entityId Entity ID - * @param branch Branch name - * @returns Number of versions - */ - async getVersionCount(entityId: string, branch: string): Promise { - const key = this.getMetaKey(entityId, branch) - const store = await this.loadStore(key) - return store?.versions.length || 0 - } - - /** - * Remove version from index - * - * @param entityId Entity ID - * @param version Version number - * @param branch Branch name - */ - async removeVersion( - entityId: string, - version: number, - branch: string - ): Promise { - await this.initialize() - - const key = this.getMetaKey(entityId, branch) - const store = await this.loadStore(key) - if (!store) return - - const initialLength = store.versions.length - store.versions = store.versions.filter(v => v.version !== version) - - // Only save if something was removed - if (store.versions.length < initialLength) { - await this.saveStore(key, store) - } - } - - /** - * Clear all versions for an entity - * - * @param entityId Entity ID - * @param branch Branch name - * @returns Number of versions deleted - */ - async clearVersions(entityId: string, branch: string): Promise { - const key = this.getMetaKey(entityId, branch) - const store = await this.loadStore(key) - if (!store) return 0 - - const count = store.versions.length - - // Delete the store by saving null/empty - await this.saveStore(key, { entityId, branch, versions: [] }) - - return count - } - - /** - * Get all versioned entities (for cleanup/debugging) - * - * Note: This is an expensive operation that requires scanning. - * In the simple key-value approach, we don't maintain a global index. - * This method returns an empty array - use storage-level scanning if needed. - * - * @returns Empty array (not supported in simple approach) - */ - async getVersionedEntities(): Promise { - // In the simple key-value approach, we don't maintain a global index - // of all versioned entities. This would require scanning storage. - // For most use cases, you know which entities you've versioned. - return [] - } - - // ============= Private Helpers ============= - - /** - * Generate storage key for version metadata - * - * @param entityId Entity ID - * @param branch Branch name - * @returns Storage key - */ - private getMetaKey(entityId: string, branch: string): string { - return `__version_meta_${entityId}_${branch}` - } - - /** - * Load version store from storage - * - * @param key Storage key - * @returns Version store or null - */ - private async loadStore(key: string): Promise { - try { - const store = await this.brain.storageAdapter.getMetadata(key) - // Handle empty store - if (!store || !store.versions) return null - return store as VersionMetadataStore - } catch { - return null - } - } - - /** - * Save version store to storage - * - * @param key Storage key - * @param store Version store - */ - private async saveStore(key: string, store: VersionMetadataStore): Promise { - await this.brain.storageAdapter.saveMetadata(key, store) - } -} diff --git a/src/versioning/VersionManager.ts b/src/versioning/VersionManager.ts deleted file mode 100644 index 78041a42..00000000 --- a/src/versioning/VersionManager.ts +++ /dev/null @@ -1,667 +0,0 @@ -/** - * VersionManager - Entity-Level Versioning Engine - * - * Provides entity-level version control with: - * - save() - Create entity version - * - restore() - Restore entity to specific version (now updates all indexes) - * - list() - List all versions of an entity - * - compare() - Deep diff between versions - * - prune() - Remove old versions (retention policies) - * - * Architecture: - * - Versions stored as key-value pairs, NOT as entities (no index pollution) - * - Content-addressable: SHA-256 hashing for deduplication - * - Space-efficient: Only stores unique content - * - Branch-aware: Versions tied to current branch - * - restore() uses brain.update() to refresh ALL indexes (HNSW, metadata, graph) - * - * Storage keys: - * - Version metadata: __version_meta_{entityId}_{branch} - * - Version content: __system_version_{entityId}_{contentHash} - * - * ZERO-CONFIG - Works automatically with existing storage infrastructure. - * NO MOCKS - Production implementation. - */ - -import { BaseStorage } from '../storage/baseStorage.js' -import { VersionStorage } from './VersionStorage.js' -import { VersionIndex } from './VersionIndex.js' -import { VersionDiff, compareEntityVersions } from './VersionDiff.js' -import type { NounMetadata } from '../coreTypes.js' - -export interface EntityVersion { - /** Version number (1-indexed, sequential per entity) */ - version: number - - /** Entity ID */ - entityId: string - - /** Branch this version was created on */ - branch: string - - /** Commit hash containing this version */ - commitHash: string - - /** Timestamp of version creation */ - timestamp: number - - /** Optional user-provided tag (e.g., 'v1.0', 'before-refactor') */ - tag?: string - - /** Optional description */ - description?: string - - /** Content hash (SHA-256 of entity data) */ - contentHash: string - - /** Author of this version */ - author?: string - - /** Metadata about the version */ - metadata?: Record -} - -export interface VersionQuery { - /** Entity ID to query versions for */ - entityId: string - - /** Optional: Filter by branch (default: current branch) */ - branch?: string - - /** Optional: Limit number of versions returned */ - limit?: number - - /** Optional: Skip first N versions */ - offset?: number - - /** Optional: Filter by tag */ - tag?: string - - /** Optional: Filter by date range */ - startDate?: number - endDate?: number -} - -export interface SaveVersionOptions { - /** Optional tag for this version (e.g., 'v1.0', 'milestone-1') */ - tag?: string - - /** Optional description */ - description?: string - - /** Optional author name */ - author?: string - - /** Optional custom metadata */ - metadata?: Record - - /** Optional: Create commit automatically (default: false) */ - createCommit?: boolean - - /** Optional: Commit message if createCommit is true */ - commitMessage?: string -} - -export interface RestoreOptions { - /** Optional: Create version before restoring (for undo) */ - createSnapshot?: boolean - - /** Optional: Tag for snapshot before restore */ - snapshotTag?: string -} - -export interface PruneOptions { - /** Keep N most recent versions (required if keepVersions not set) */ - keepRecent?: number - - /** Keep versions newer than timestamp (optional) */ - keepAfter?: number - - /** Keep tagged versions (default: true) */ - keepTagged?: boolean - - /** Dry run - don't actually delete (default: false) */ - dryRun?: boolean -} - -/** - * VersionManager - Core versioning engine - */ -export class VersionManager { - private brain: any // Brainy instance - private versionStorage: VersionStorage - private versionIndex: VersionIndex - private initialized: boolean = false - - constructor(brain: any) { - this.brain = brain - this.versionStorage = new VersionStorage(brain) - this.versionIndex = new VersionIndex(brain) - } - - /** - * Check if an entity is a VFS file - * VFS files store content in BlobStorage, not in entity.data - * - * @param entity Entity metadata object - * @returns True if entity is a VFS file - */ - private isVFSFile(entity: any): boolean { - return ( - entity?.isVFS === true && - entity?.vfsType === 'file' && - typeof entity?.path === 'string' - ) - } - - /** - * Check if content is text-based for encoding decisions - * @param mimeType MIME type of the content - * @returns True if content should be stored as UTF-8 string - */ - private isTextContent(mimeType?: string): boolean { - if (!mimeType) return false - return ( - mimeType.startsWith('text/') || - mimeType === 'application/json' || - mimeType === 'application/javascript' || - mimeType === 'application/typescript' || - mimeType === 'application/xml' || - mimeType.includes('+xml') || - mimeType.includes('+json') - ) - } - - /** - * Initialize versioning system (lazy) - */ - async initialize(): Promise { - if (this.initialized) return - - await this.versionStorage.initialize() - await this.versionIndex.initialize() - - this.initialized = true - } - - /** - * Save a version of an entity - * - * Creates a version snapshot of the current entity state. - * If createCommit is true, also creates a commit containing this version. - * - * @param entityId Entity ID to version - * @param options Save options - * @returns Created version metadata - */ - async save( - entityId: string, - options: SaveVersionOptions = {} - ): Promise { - await this.initialize() - - // Get current entity state - const entity = await this.brain.getNounMetadata(entityId) - if (!entity) { - throw new Error(`Entity ${entityId} not found`) - } - - // FIX: For VFS file entities, fetch current content from blob storage - // The entity.data field contains stale embedding text, not actual file content - // VFS files store their real content in BlobStorage (content-addressable) - if (this.isVFSFile(entity)) { - if (!this.brain.vfs) { - throw new Error( - `Cannot version VFS file ${entityId}: VFS not initialized. ` + - `Ensure brain.vfs is available before versioning VFS files.` - ) - } - - // Read fresh content from blob storage via VFS - const freshContent: Buffer = await this.brain.vfs.readFile(entity.path) - - // Store content with appropriate encoding - // Text files as UTF-8 string (readable, smaller) - // Binary files as base64 (safe for JSON serialization) - if (this.isTextContent(entity.mimeType)) { - entity.data = freshContent.toString('utf-8') - } else { - entity.data = freshContent.toString('base64') - entity._vfsEncoding = 'base64' // Flag for restore to decode - } - } - - // Get current branch - const currentBranch = this.brain.currentBranch - - // Get next version number - const existingVersions = await this.versionIndex.getVersions({ - entityId, - branch: currentBranch - }) - const nextVersion = existingVersions.length + 1 - - // Calculate content hash - const contentHash = this.versionStorage.hashEntity(entity) - - // Check for duplicate (same content as last version) - if (existingVersions.length > 0) { - const lastVersion = existingVersions[existingVersions.length - 1] - if (lastVersion.contentHash === contentHash) { - // Content unchanged - return last version instead of creating duplicate - return lastVersion - } - } - - // Create commit if requested - let commitHash: string - if (options.createCommit) { - const commitMessage = - options.commitMessage || `Version ${nextVersion} of entity ${entityId}` - - // Use brain's commit method (note: single options object) - await this.brain.commit({ - message: commitMessage, - author: options.author, - metadata: { - versionedEntity: entityId, - version: nextVersion, - ...options.metadata - } - }) - - // Get the commit hash that was just created - const refManager = this.brain.refManager - const ref = await refManager.getRef(currentBranch) - commitHash = ref.commitHash - } else { - // Use current HEAD commit - const refManager = this.brain.refManager - const ref = await refManager.getRef(currentBranch) - if (!ref) { - throw new Error( - `No commit exists on branch ${currentBranch}. Create a commit first or use createCommit: true` - ) - } - commitHash = ref.commitHash - } - - // Create version metadata - const version: EntityVersion = { - version: nextVersion, - entityId, - branch: currentBranch, - commitHash, - timestamp: Date.now(), - contentHash, - tag: options.tag, - description: options.description, - author: options.author, - metadata: options.metadata - } - - // Store version - await this.versionStorage.saveVersion(version, entity) - await this.versionIndex.addVersion(version) - - return version - } - - /** - * Get all versions of an entity - * - * @param entityId Entity ID - * @param query Optional query filters - * @returns List of versions (newest first) - */ - async list( - entityId: string, - query: Partial = {} - ): Promise { - await this.initialize() - - const currentBranch = this.brain.currentBranch - - return this.versionIndex.getVersions({ - entityId, - branch: query.branch || currentBranch, - limit: query.limit, - offset: query.offset, - tag: query.tag, - startDate: query.startDate, - endDate: query.endDate - }) - } - - /** - * Get a specific version of an entity - * - * @param entityId Entity ID - * @param version Version number (1-indexed) - * @returns Version metadata - */ - async getVersion( - entityId: string, - version: number - ): Promise { - await this.initialize() - - const currentBranch = this.brain.currentBranch - return this.versionIndex.getVersion(entityId, version, currentBranch) - } - - /** - * Get version by tag - * - * @param entityId Entity ID - * @param tag Version tag - * @returns Version metadata - */ - async getVersionByTag( - entityId: string, - tag: string - ): Promise { - await this.initialize() - - const currentBranch = this.brain.currentBranch - return this.versionIndex.getVersionByTag(entityId, tag, currentBranch) - } - - /** - * Restore entity to a specific version - * - * Overwrites current entity state with the specified version. - * Optionally creates a snapshot before restoring for undo capability. - * - * @param entityId Entity ID - * @param version Version number or tag - * @param options Restore options - * @returns Restored version metadata - */ - async restore( - entityId: string, - version: number | string, - options: RestoreOptions = {} - ): Promise { - await this.initialize() - - // Create snapshot before restoring (for undo) - if (options.createSnapshot) { - await this.save(entityId, { - tag: options.snapshotTag || 'before-restore', - description: `Snapshot before restoring to version ${version}`, - metadata: { restoringTo: version } - }) - } - - // Get target version - let targetVersion: EntityVersion | null - if (typeof version === 'number') { - targetVersion = await this.getVersion(entityId, version) - } else { - targetVersion = await this.getVersionByTag(entityId, version) - } - - if (!targetVersion) { - throw new Error( - `Version ${version} not found for entity ${entityId}` - ) - } - - // Load versioned entity data - const versionedEntity = await this.versionStorage.loadVersion(targetVersion) - if (!versionedEntity) { - throw new Error( - `Version data not found for entity ${entityId} version ${version}` - ) - } - - // FIX: For VFS file entities, write content back to blob storage - // The versioned data contains the actual file content (not stale embedding text) - // Using vfs.writeFile() ensures proper blob creation and metadata update - if (this.isVFSFile(versionedEntity)) { - if (!this.brain.vfs) { - throw new Error( - `Cannot restore VFS file ${entityId}: VFS not initialized. ` + - `Ensure brain.vfs is available before restoring VFS files.` - ) - } - - // Decode content based on how it was stored - let content: Buffer - if (versionedEntity._vfsEncoding === 'base64') { - // Binary file stored as base64 - content = Buffer.from(versionedEntity.data as string, 'base64') - } else { - // Text file stored as UTF-8 string - content = Buffer.from(versionedEntity.data as string, 'utf-8') - } - - // Write content back to VFS - this handles: - // - BlobStorage write (new hash) - // - Entity metadata update - // - Path resolver cache update - await this.brain.vfs.writeFile(versionedEntity.path, content) - - return targetVersion - } - - // For non-VFS entities, use existing brain.update() logic - // Extract standard fields vs custom metadata - // NounMetadata has: noun, data, createdAt, updatedAt, createdBy, service, confidence, weight - const { - noun, - data, - createdAt, - updatedAt, - createdBy, - service, - confidence, - weight, - ...customMetadata - } = versionedEntity - - // Use brain.update() to restore - this updates ALL indexes (HNSW, metadata, graph) - // This is critical: saveNounMetadata() only saves to storage without updating indexes - await this.brain.update({ - id: entityId, - data: data, - type: noun, - metadata: customMetadata, - confidence: confidence, - weight: weight, - merge: false // Replace entirely, don't merge with existing metadata - }) - - return targetVersion - } - - /** - * Compare two versions of an entity - * - * @param entityId Entity ID - * @param fromVersion Version number or tag (older) - * @param toVersion Version number or tag (newer) - * @returns Diff between versions - */ - async compare( - entityId: string, - fromVersion: number | string, - toVersion: number | string - ): Promise { - await this.initialize() - - // Get versions - const fromVer = - typeof fromVersion === 'number' - ? await this.getVersion(entityId, fromVersion) - : await this.getVersionByTag(entityId, fromVersion) - - const toVer = - typeof toVersion === 'number' - ? await this.getVersion(entityId, toVersion) - : await this.getVersionByTag(entityId, toVersion) - - if (!fromVer) { - throw new Error( - `Version ${fromVersion} not found for entity ${entityId}` - ) - } - if (!toVer) { - throw new Error( - `Version ${toVersion} not found for entity ${entityId}` - ) - } - - // Load entity data - const fromEntity = await this.versionStorage.loadVersion(fromVer) - const toEntity = await this.versionStorage.loadVersion(toVer) - - if (!fromEntity || !toEntity) { - throw new Error('Failed to load version data for comparison') - } - - // Compare versions - return compareEntityVersions(fromEntity, toEntity, { - fromVersion: fromVer.version, - toVersion: toVer.version, - entityId - }) - } - - /** - * Prune old versions based on retention policy - * - * @param entityId Entity ID (or '*' for all entities) - * @param options Prune options - * @returns Number of versions deleted - */ - async prune( - entityId: string, - options: PruneOptions - ): Promise<{ deleted: number; kept: number }> { - await this.initialize() - - if (!options.keepRecent && !options.keepAfter) { - throw new Error( - 'Must specify either keepRecent or keepAfter in prune options' - ) - } - - const currentBranch = this.brain.currentBranch - - // Get all versions - const versions = await this.versionIndex.getVersions({ - entityId, - branch: currentBranch - }) - - // Determine which versions to keep - const toKeep = new Set() - const toDelete: EntityVersion[] = [] - - // Keep recent versions - if (options.keepRecent) { - const recentVersions = versions.slice(0, options.keepRecent) - recentVersions.forEach((v) => toKeep.add(v.version)) - } - - // Keep versions after timestamp - if (options.keepAfter) { - versions - .filter((v) => v.timestamp >= options.keepAfter!) - .forEach((v) => toKeep.add(v.version)) - } - - // Keep tagged versions - if (options.keepTagged !== false) { - versions - .filter((v) => v.tag !== undefined) - .forEach((v) => toKeep.add(v.version)) - } - - // Build delete list - for (const version of versions) { - if (!toKeep.has(version.version)) { - toDelete.push(version) - } - } - - // Dry run - just return counts - if (options.dryRun) { - return { - deleted: toDelete.length, - kept: toKeep.size - } - } - - // Delete versions - for (const version of toDelete) { - await this.versionStorage.deleteVersion(version) - await this.versionIndex.removeVersion( - entityId, - version.version, - currentBranch - ) - } - - return { - deleted: toDelete.length, - kept: toKeep.size - } - } - - /** - * Get version count for an entity - * - * @param entityId Entity ID - * @returns Number of versions - */ - async getVersionCount(entityId: string): Promise { - await this.initialize() - - const currentBranch = this.brain.currentBranch - return this.versionIndex.getVersionCount(entityId, currentBranch) - } - - /** - * Check if entity has versions - * - * @param entityId Entity ID - * @returns True if entity has versions - */ - async hasVersions(entityId: string): Promise { - const count = await this.getVersionCount(entityId) - return count > 0 - } - - /** - * Get latest version of an entity - * - * @param entityId Entity ID - * @returns Latest version metadata or null - */ - async getLatest(entityId: string): Promise { - await this.initialize() - - const versions = await this.list(entityId, { limit: 1 }) - return versions[0] || null - } - - /** - * Clear all versions for an entity - * - * @param entityId Entity ID - * @returns Number of versions deleted - */ - async clear(entityId: string): Promise { - await this.initialize() - - const result = await this.prune(entityId, { - keepRecent: 0, - keepTagged: false - }) - - return result.deleted - } -} diff --git a/src/versioning/VersionStorage.ts b/src/versioning/VersionStorage.ts deleted file mode 100644 index abc7d4cc..00000000 --- a/src/versioning/VersionStorage.ts +++ /dev/null @@ -1,250 +0,0 @@ -/** - * VersionStorage - Hybrid Storage for Entity Versions - * - * Implements content-addressable storage for entity versions: - * - SHA-256 content hashing for deduplication - * - Uses BaseStorage.saveMetadata/getMetadata for storage - * - Integrates with COW commit system - * - Space-efficient: Only stores unique content - * - * Storage structure: - * Version content is stored using system metadata keys: - * __system_version_{entityId}_{contentHash} - * - * This integrates with BaseStorage's routing which places system keys - * in the _system/ directory, keeping version data separate from entities. - * - * NO MOCKS - Production implementation - */ - -import { createHash } from 'crypto' -import { BaseStorage } from '../storage/baseStorage.js' -import type { NounMetadata } from '../coreTypes.js' -import type { EntityVersion } from './VersionManager.js' - -/** - * VersionStorage - Content-addressable version storage - */ -export class VersionStorage { - private brain: any // Brainy instance - private initialized: boolean = false - - constructor(brain: any) { - this.brain = brain - } - - /** - * Initialize version storage directories - */ - async initialize(): Promise { - if (this.initialized) return - - // Version storage uses the same storage adapter as the main database - // Directories are created automatically by the storage adapter - - this.initialized = true - } - - /** - * Calculate SHA-256 hash of entity content - * - * Used for content-addressable storage and deduplication - * - * @param entity Entity to hash - * @returns SHA-256 hash (hex string) - */ - hashEntity(entity: NounMetadata): string { - // Create stable JSON representation (sorted keys) - const stableJson = this.toStableJson(entity) - return createHash('sha256').update(stableJson).digest('hex') - } - - /** - * Convert entity to stable JSON (sorted keys for consistent hashing) - */ - private toStableJson(obj: any): string { - if (obj === null) return 'null' - if (obj === undefined) return 'undefined' - if (typeof obj !== 'object') return JSON.stringify(obj) - - if (Array.isArray(obj)) { - const items = obj.map((item) => this.toStableJson(item)) - return `[${items.join(',') }]` - } - - // Sort object keys for stable hashing - const sortedKeys = Object.keys(obj).sort() - const pairs = sortedKeys.map((key) => { - const value = this.toStableJson(obj[key]) - return `"${key}":${value}` - }) - - return `{${pairs.join(',')}}` - } - - /** - * Save entity version to content-addressable storage - * - * @param version Version metadata - * @param entity Entity data - */ - async saveVersion( - version: EntityVersion, - entity: NounMetadata - ): Promise { - await this.initialize() - - // Content-addressable path: .brainy/versions/entities/{entityId}/{contentHash}.json - const versionPath = this.getVersionPath( - version.entityId, - version.contentHash - ) - - // Check if content already exists (deduplication) - const exists = await this.contentExists(versionPath) - if (exists) { - // Content already stored - no need to write again - return - } - - // Store entity data - await this.writeVersionData(versionPath, entity) - } - - /** - * Load entity version from storage - * - * @param version Version metadata - * @returns Entity data or null if not found - */ - async loadVersion(version: EntityVersion): Promise { - await this.initialize() - - const versionPath = this.getVersionPath( - version.entityId, - version.contentHash - ) - - try { - return await this.readVersionData(versionPath) - } catch (error) { - console.error(`Failed to load version ${version.version}:`, error) - return null - } - } - - /** - * Delete entity version from storage - * - * @param version Version to delete - */ - async deleteVersion(version: EntityVersion): Promise { - await this.initialize() - - const versionPath = this.getVersionPath( - version.entityId, - version.contentHash - ) - - await this.deleteVersionData(versionPath) - } - - /** - * Get version storage key - * - * Uses __system_ prefix so BaseStorage routes to system storage (_system/ directory) - * This keeps version data separate from entity data. - * - * @param entityId Entity ID - * @param contentHash Content hash - * @returns Storage key for version content - */ - private getVersionPath(entityId: string, contentHash: string): string { - // Use system-prefixed key for BaseStorage.saveMetadata/getMetadata - // BaseStorage recognizes __system_ prefix and routes to _system/ directory - return `__system_version_${entityId}_${contentHash}` - } - - /** - * Check if content exists in storage - * - * @param key Storage key - * @returns True if exists - */ - private async contentExists(key: string): Promise { - try { - // Use getMetadata to check existence - const adapter = this.brain.storageAdapter - if (!adapter) return false - - const data = await adapter.getMetadata(key) - return data !== null - } catch { - return false - } - } - - /** - * Write version data to storage - * - * @param key Storage key - * @param entity Entity data - */ - private async writeVersionData( - key: string, - entity: NounMetadata - ): Promise { - const adapter = this.brain.storageAdapter - - if (!adapter) { - throw new Error('Storage adapter not available') - } - - // Use saveMetadata for storing version content - // The key is system-prefixed so it routes to _system/ directory - await adapter.saveMetadata(key, entity) - } - - /** - * Read version data from storage - * - * @param key Storage key - * @returns Entity data - */ - private async readVersionData(key: string): Promise { - const adapter = this.brain.storageAdapter - - if (!adapter) { - throw new Error('Storage adapter not available') - } - - // Use getMetadata for reading version content - const entity = await adapter.getMetadata(key) - if (!entity) { - throw new Error(`Version data not found: ${key}`) - } - - return entity - } - - /** - * Delete version data from storage - * - * Note: Version content is content-addressed and immutable. - * Deleting the version index entry (via VersionIndex.removeVersion) is sufficient. - * The content may be shared with other versions (same contentHash). - * - * We don't actually delete version content to avoid breaking - * other versions that may reference the same content hash. - * A separate garbage collection process could clean up unreferenced content. - * - * @param key Storage key (unused - kept for API compatibility) - */ - private async deleteVersionData(_key: string): Promise { - // Version content is content-addressed and may be shared. - // We don't delete it here to prevent breaking other versions. - // The version INDEX is deleted via VersionIndex.removeVersion(). - // A GC process could clean up unreferenced content in the future. - } - -} diff --git a/src/versioning/VersioningAPI.ts b/src/versioning/VersioningAPI.ts deleted file mode 100644 index 5a946bc9..00000000 --- a/src/versioning/VersioningAPI.ts +++ /dev/null @@ -1,520 +0,0 @@ -/** - * VersioningAPI - Public API for Entity Versioning - * - * User-friendly wrapper around VersionManager with: - * - Clean, simple API - * - Smart defaults - * - Error handling - * - Type safety - * - * Usage: - * const version = await brain.versions.save('entity-123', { tag: 'v1.0' }) - * const versions = await brain.versions.list('entity-123') - * await brain.versions.restore('entity-123', 5) - * const diff = await brain.versions.compare('entity-123', 2, 5) - * - * NO MOCKS - Production implementation - */ - -import { VersionManager } from './VersionManager.js' -import type { - EntityVersion, - SaveVersionOptions, - RestoreOptions, - PruneOptions, - VersionQuery -} from './VersionManager.js' -import type { VersionDiff, DiffOptions } from './VersionDiff.js' -import type { BaseStorage } from '../storage/baseStorage.js' - -/** - * VersioningAPI - User-friendly versioning interface - */ -export class VersioningAPI { - private manager: VersionManager - private brain: any // Brainy instance - - constructor(brain: any) { - this.brain = brain - this.manager = new VersionManager(brain as BaseStorage) - } - - /** - * Save current state of entity as a new version - * - * Creates a version snapshot of the current entity state. - * Automatically handles deduplication - if content hasn't changed, - * returns the last version instead of creating a duplicate. - * - * @param entityId Entity ID to version - * @param options Save options - * @returns Created (or existing) version metadata - * - * @example - * ```typescript - * // Simple save - * const version = await brain.versions.save('user-123') - * - * // Save with tag and description - * const version = await brain.versions.save('user-123', { - * tag: 'v1.0', - * description: 'Initial release', - * author: 'alice' - * }) - * - * // Save and create commit - * const version = await brain.versions.save('user-123', { - * tag: 'milestone-1', - * createCommit: true, - * commitMessage: 'Milestone 1 complete' - * }) - * ``` - */ - async save( - entityId: string, - options: SaveVersionOptions = {} - ): Promise { - return this.manager.save(entityId, options) - } - - /** - * List all versions of an entity - * - * Returns versions sorted by version number (newest first). - * Supports filtering by tag, date range, and pagination. - * - * @param entityId Entity ID - * @param options Query options - * @returns List of versions (newest first) - * - * @example - * ```typescript - * // Get all versions - * const versions = await brain.versions.list('user-123') - * - * // Get last 10 versions - * const recent = await brain.versions.list('user-123', { limit: 10 }) - * - * // Get tagged versions - * const tagged = await brain.versions.list('user-123', { tag: 'v*' }) - * - * // Get versions from last 30 days - * const recent = await brain.versions.list('user-123', { - * startDate: Date.now() - 30 * 24 * 60 * 60 * 1000 - * }) - * ``` - */ - async list( - entityId: string, - options: Partial = {} - ): Promise { - return this.manager.list(entityId, options) - } - - /** - * Get specific version of an entity - * - * @param entityId Entity ID - * @param version Version number (1-indexed) - * @returns Version metadata or null if not found - * - * @example - * ```typescript - * const version = await brain.versions.getVersion('user-123', 5) - * if (version) { - * console.log(`Version ${version.version} created at ${new Date(version.timestamp)}`) - * } - * ``` - */ - async getVersion( - entityId: string, - version: number - ): Promise { - return this.manager.getVersion(entityId, version) - } - - /** - * Get version by tag - * - * @param entityId Entity ID - * @param tag Version tag - * @returns Version metadata or null if not found - * - * @example - * ```typescript - * const version = await brain.versions.getVersionByTag('user-123', 'v1.0') - * ``` - */ - async getVersionByTag( - entityId: string, - tag: string - ): Promise { - return this.manager.getVersionByTag(entityId, tag) - } - - /** - * Get latest version of an entity - * - * @param entityId Entity ID - * @returns Latest version or null if no versions exist - * - * @example - * ```typescript - * const latest = await brain.versions.getLatest('user-123') - * ``` - */ - async getLatest(entityId: string): Promise { - return this.manager.getLatest(entityId) - } - - /** - * Get version content without restoring - * - * Allows you to preview version data without modifying the current entity. - * - * @param entityId Entity ID - * @param version Version number or tag - * @returns Version content - * - * @example - * ```typescript - * // Preview version 5 without restoring - * const oldData = await brain.versions.getContent('user-123', 5) - * console.log('Version 5 had name:', oldData.name) - * - * // Compare with current - * const current = await brain.getNounMetadata('user-123') - * console.log('Current name:', current.name) - * ``` - */ - async getContent( - entityId: string, - version: number | string - ): Promise { - // Get version metadata - let versionMeta: EntityVersion | null - if (typeof version === 'number') { - versionMeta = await this.manager.getVersion(entityId, version) - } else { - versionMeta = await this.manager.getVersionByTag(entityId, version) - } - - if (!versionMeta) { - throw new Error( - `Version ${version} not found for entity ${entityId}` - ) - } - - // Load version content - const versionStorage = (this.manager as any).versionStorage - const content = await versionStorage.loadVersion(versionMeta) - - if (!content) { - throw new Error( - `Version content not found for entity ${entityId} version ${version}` - ) - } - - return content - } - - /** - * Restore entity to a specific version - * - * Overwrites current entity state with the specified version. - * Optionally creates a snapshot before restoring for undo capability. - * - * @param entityId Entity ID - * @param version Version number or tag to restore to - * @param options Restore options - * @returns Restored version metadata - * - * @example - * ```typescript - * // Simple restore - * await brain.versions.restore('user-123', 5) - * - * // Restore with safety snapshot - * await brain.versions.restore('user-123', 5, { - * createSnapshot: true, - * snapshotTag: 'before-restore' - * }) - * - * // Restore by tag - * await brain.versions.restore('user-123', 'v1.0') - * ``` - */ - async restore( - entityId: string, - version: number | string, - options: RestoreOptions = {} - ): Promise { - return this.manager.restore(entityId, version, options) - } - - /** - * Compare two versions of an entity - * - * Generates a deep diff showing added, removed, modified, and type-changed fields. - * - * @param entityId Entity ID - * @param fromVersion Version number or tag (older) - * @param toVersion Version number or tag (newer) - * @returns Diff between versions - * - * @example - * ```typescript - * // Compare version 2 to version 5 - * const diff = await brain.versions.compare('user-123', 2, 5) - * - * console.log(`Added fields: ${diff.added.length}`) - * console.log(`Removed fields: ${diff.removed.length}`) - * console.log(`Modified fields: ${diff.modified.length}`) - * - * // Print human-readable diff - * import { formatDiff } from './VersionDiff.js' - * console.log(formatDiff(diff)) - * ``` - */ - async compare( - entityId: string, - fromVersion: number | string, - toVersion: number | string - ): Promise { - return this.manager.compare(entityId, fromVersion, toVersion) - } - - /** - * Prune old versions based on retention policy - * - * Removes old versions while preserving recent and tagged versions. - * Use dryRun to preview what would be deleted without actually deleting. - * - * @param entityId Entity ID (or '*' for all entities - NOT IMPLEMENTED YET) - * @param options Prune options - * @returns Count of deleted and kept versions - * - * @example - * ```typescript - * // Keep last 10 versions, delete rest - * const result = await brain.versions.prune('user-123', { - * keepRecent: 10 - * }) - * console.log(`Deleted ${result.deleted} versions`) - * - * // Keep last 30 days, preserve tagged versions - * const result = await brain.versions.prune('user-123', { - * keepAfter: Date.now() - 30 * 24 * 60 * 60 * 1000, - * keepTagged: true - * }) - * - * // Dry run - see what would be deleted - * const result = await brain.versions.prune('user-123', { - * keepRecent: 5, - * dryRun: true - * }) - * console.log(`Would delete ${result.deleted} versions`) - * ``` - */ - async prune( - entityId: string, - options: PruneOptions - ): Promise<{ deleted: number; kept: number }> { - return this.manager.prune(entityId, options) - } - - /** - * Get version count for an entity - * - * @param entityId Entity ID - * @returns Number of versions - * - * @example - * ```typescript - * const count = await brain.versions.count('user-123') - * console.log(`Entity has ${count} versions`) - * ``` - */ - async count(entityId: string): Promise { - return this.manager.getVersionCount(entityId) - } - - /** - * Check if entity has any versions - * - * @param entityId Entity ID - * @returns True if entity has versions - * - * @example - * ```typescript - * if (await brain.versions.hasVersions('user-123')) { - * console.log('Entity is versioned') - * } - * ``` - */ - async hasVersions(entityId: string): Promise { - return this.manager.hasVersions(entityId) - } - - /** - * Clear all versions for an entity - * - * WARNING: This permanently deletes all version history! - * - * @param entityId Entity ID - * @returns Number of versions deleted - * - * @example - * ```typescript - * const deleted = await brain.versions.clear('user-123') - * console.log(`Deleted ${deleted} versions`) - * ``` - */ - async clear(entityId: string): Promise { - return this.manager.clear(entityId) - } - - // ===== CONVENIENCE METHODS ===== - - /** - * Quick save with auto-generated tag - * - * Generates tag in format: 'auto-{timestamp}' - * - * @param entityId Entity ID - * @param description Optional description - * @returns Created version - */ - async quickSave( - entityId: string, - description?: string - ): Promise { - return this.save(entityId, { - tag: `auto-${Date.now()}`, - description - }) - } - - /** - * Undo last change (restore to previous version) - * - * Safely restores to previous version with automatic snapshot. - * - * @param entityId Entity ID - * @returns Restored version metadata - */ - async undo(entityId: string): Promise { - const versions = await this.list(entityId, { limit: 2 }) - - if (versions.length < 2) { - // No previous version to restore to - return null - } - - const previousVersion = versions[1] // Second most recent - - return this.restore(entityId, previousVersion.version, { - createSnapshot: true, - snapshotTag: 'before-undo' - }) - } - - /** - * Get version history summary - * - * @param entityId Entity ID - * @returns Summary statistics - */ - async history(entityId: string): Promise<{ - total: number - oldest?: EntityVersion - newest?: EntityVersion - tagged: number - branches: string[] - }> { - const versions = await this.list(entityId) - - const tagged = versions.filter((v) => v.tag !== undefined).length - const branches = [...new Set(versions.map((v) => v.branch))] - - return { - total: versions.length, - oldest: versions[versions.length - 1], - newest: versions[0], - tagged, - branches - } - } - - /** - * Revert to previous version (alias for undo with better semantics) - * - * Restores entity to the previous version with automatic safety snapshot. - * - * @param entityId Entity ID - * @returns Restored version or null if not possible - * - * @example - * ```typescript - * // Made a mistake? Revert! - * await brain.update('user-123', { name: 'Wrong Name' }) - * await brain.versions.revert('user-123') // Back to previous state - * ``` - */ - async revert(entityId: string): Promise { - return this.undo(entityId) - } - - /** - * Tag an existing version - * - * Updates the tag of an existing version. - * Note: This creates a new version with the tag, not modifying the existing one. - * - * @param entityId Entity ID - * @param version Version number - * @param tag New tag - * @returns Updated version - */ - async tag( - entityId: string, - version: number, - tag: string - ): Promise { - // Get the version - const existingVersion = await this.getVersion(entityId, version) - if (!existingVersion) { - throw new Error(`Version ${version} not found for entity ${entityId}`) - } - - // Create new version with tag - // Note: This is a limitation - we can't modify existing versions - // because they're immutable. Instead, we create a new version. - return this.save(entityId, { - tag, - description: `Tagged version ${version} as ${tag}`, - metadata: { originalVersion: version } - }) - } - - /** - * Get diff between entity's current state and a version - * - * @param entityId Entity ID - * @param version Version to compare against - * @returns Diff showing changes - */ - async diffWithCurrent( - entityId: string, - version: number | string - ): Promise { - // Save current state (will be deduplicated if unchanged) - const currentVersion = await this.save(entityId, { - tag: 'temp-compare', - description: 'Temporary version for comparison' - }) - - // Compare - return this.compare(entityId, version, currentVersion.version) - } -} diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 297f51e4..2454d8a5 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -104,8 +104,11 @@ export class VirtualFileSystem implements IVirtualFileSystem { private get blobStorage() { // TypeScript doesn't know about blobStorage on storage, use type assertion const storage = this.brain['storage'] as any - if (!storage || !('blobStorage' in storage)) { - throw new Error('BlobStorage not available. Requires COW-enabled storage adapter.') + if (!storage?.blobStorage) { + throw new Error( + 'BlobStorage not available. The storage adapter must run ' + + 'initializeBlobStorage() before the VFS is used (brain.init() does this).' + ) } return storage.blobStorage } @@ -437,9 +440,13 @@ export class VirtualFileSystem implements IVirtualFileSystem { existingId = null } + // Detect MIME type BEFORE the blob write so the store's auto-compression + // policy can skip zstd for already-compressed media (JPEG, MP4, ZIP, …). + const mimeType = mimeDetector.detectMimeType(name, buffer) + // Unified blob storage for ALL files (no size-based branching) - // Store in BlobStorage (content-addressable, auto-deduplication, streaming) - const blobHash = await this.blobStorage.write(buffer) + // Store in BlobStorage (content-addressable, auto-deduplication) + const blobHash = await this.blobStorage.write(buffer, { mimeType }) // Get blob metadata (size, compression info) const blobMetadata = await this.blobStorage.getMetadata(blobHash) @@ -448,12 +455,9 @@ export class VirtualFileSystem implements IVirtualFileSystem { type: 'blob', hash: blobHash, size: buffer.length, - compressed: blobMetadata?.compressed + compressed: blobMetadata ? blobMetadata.compression !== 'none' : undefined } - // Detect MIME type (using comprehensive MimeTypeDetector) - const mimeType = mimeDetector.detectMimeType(name, buffer) - // Create metadata const metadata: VFSMetadata = { path, @@ -2716,11 +2720,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { return results } - /** - * Get all versions of a file (semantic versioning) - */ - - createReadStream(path: string, options?: ReadStreamOptions): NodeJS.ReadableStream { // Lazy import to avoid circular dependencies const { VFSReadStream } = require('./streams/VFSReadStream.js') diff --git a/tests/helpers/TestWrappingAdapter.ts b/tests/helpers/TestWrappingAdapter.ts index 22da3c89..f088307a 100644 --- a/tests/helpers/TestWrappingAdapter.ts +++ b/tests/helpers/TestWrappingAdapter.ts @@ -1,10 +1,10 @@ /** - * TestWrappingAdapter: Real COW storage adapter for testing + * TestWrappingAdapter: Real blob store adapter for testing * * This adapter ACTUALLY wraps binary data in JSON (like production storage) * to properly test the unwrap logic in BlobStorage. * - * Unlike InMemoryCOWAdapter which stores Buffers directly, + * Unlike InMemoryBlobAdapter which stores Buffers directly, * this adapter simulates real storage behavior: * 1. Compresses with gzip * 2. Wraps binary as {_binary: true, data: "base64..."} @@ -13,14 +13,14 @@ * This ensures tests exercise the ACTUAL code path that caused v5.10.0 regression. */ -import { COWStorageAdapter } from '../../src/storage/cow/BlobStorage.js' +import { BlobStoreAdapter } from '../../src/storage/blobStorage.js' import { gzip, gunzip } from 'zlib' import { promisify } from 'util' const gzipAsync = promisify(gzip) const gunzipAsync = promisify(gunzip) -export class TestWrappingAdapter implements COWStorageAdapter { +export class TestWrappingAdapter implements BlobStoreAdapter { private storage = new Map() async get(key: string): Promise { @@ -39,7 +39,7 @@ export class TestWrappingAdapter implements COWStorageAdapter { } async put(key: string, data: Buffer): Promise { - // v6.2.0: Use key-based dispatch (matches baseStorage COW adapter) + // Use key-based dispatch (matches the baseStorage blob adapter) // NO GUESSING - key format explicitly declares data type const obj = key.includes('-meta:') || key.startsWith('ref:') ? JSON.parse(data.toString()) // Metadata/refs: ALWAYS JSON diff --git a/tests/integration/commit-ref-bug.test.ts b/tests/integration/commit-ref-bug.test.ts deleted file mode 100644 index a0c1b9c7..00000000 --- a/tests/integration/commit-ref-bug.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Integration test to reproduce commit() ref update bug - * Bug: commit() creates blobs but doesn't update branch refs - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import * as fs from 'fs' -import * as path from 'path' -import * as zlib from 'zlib' - -const TEST_DATA_PATH = './test-commit-ref-bug-data' - -describe('Commit Ref Update Bug (v5.2.0)', () => { - let brain: Brainy - - beforeEach(async () => { - // Clean up test data - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: TEST_DATA_PATH, - branch: 'main', - enableCompression: true - }, - silent: false // Enable logging - }) - - await brain.init() - }) - - afterEach(() => { - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - }) - - it('should update branch ref after commit', async () => { - // Add an entity - await brain.add({ - data: 'Test entity', - type: 'concept', - metadata: { test: true } - }) - - // Create first commit - const commit1Hash = await brain.commit({ - message: 'First commit', - author: 'test@example.com' - }) - - console.log(`\n=== Commit 1 created: ${commit1Hash} ===`) - - // Check the ref file directly - const refPath = path.join(TEST_DATA_PATH, '_cow', 'ref:refs', 'heads', 'main.gz') - - expect(fs.existsSync(refPath), 'Ref file should exist').toBe(true) - - const refContent1 = JSON.parse( - zlib.gunzipSync(fs.readFileSync(refPath)).toString() - ) - - console.log('Ref after commit 1:', { - commitHash: refContent1.commitHash, - updatedAt: new Date(refContent1.updatedAt).toISOString() - }) - - // THIS IS THE BUG: commitHash should equal commit1Hash - expect(refContent1.commitHash).toBe(commit1Hash) - expect(refContent1.commitHash).not.toBe('0'.repeat(64)) - - // Add another entity - await brain.add({ - data: 'Second entity', - type: 'concept' - }) - - // Create second commit - const commit2Hash = await brain.commit({ - message: 'Second commit', - author: 'test@example.com' - }) - - console.log(`\n=== Commit 2 created: ${commit2Hash} ===`) - - // Check ref again - const refContent2 = JSON.parse( - zlib.gunzipSync(fs.readFileSync(refPath)).toString() - ) - - console.log('Ref after commit 2:', { - commitHash: refContent2.commitHash, - updatedAt: new Date(refContent2.updatedAt).toISOString() - }) - - // THIS IS THE BUG: commitHash should equal commit2Hash - expect(refContent2.commitHash).toBe(commit2Hash) - expect(refContent2.updatedAt).toBeGreaterThan(refContent1.updatedAt) - }) -}) diff --git a/tests/integration/commit-state-capture.test.ts b/tests/integration/commit-state-capture.test.ts deleted file mode 100644 index d6cf2ea4..00000000 --- a/tests/integration/commit-state-capture.test.ts +++ /dev/null @@ -1,573 +0,0 @@ -/** - * Commit State Capture Integration Tests (v5.4.0 Phase 1) - * - * Tests the new captureState parameter on commit() that enables - * historical time-travel by capturing entity snapshots in trees. - * - * Tests: - * 1. Lightweight commit (captureState: false uses NULL_HASH) - * 2. Default behavior (captureState omitted → captures state) - * 3. Empty workspace handling - * 4. Performance benchmarks (100, 1K, 10K entities) - * 5. BlobStorage deduplication - * 6. Tree structure validity - * 7. Storage adapter compatibility - * 8. VFS entity capture - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import * as fs from 'fs' -import * as path from 'path' - -const TEST_DATA_PATH = './test-commit-state-capture' - -describe('Commit State Capture (v5.4.0)', () => { - let brain: Brainy - - beforeEach(async () => { - // Clean up test data - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - fs.mkdirSync(TEST_DATA_PATH, { recursive: true }) - - brain = new Brainy({ requireSubtype: false, - storage: { - adapter: 'filesystem', - path: TEST_DATA_PATH - }, - disableAutoRebuild: true, - silent: true - }) - - await brain.init() - }) - - afterEach(() => { - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - }) - - it('should use NULL_HASH when captureState is false (backward compat)', async () => { - console.log('\n📋 Test 1: Backward compatibility') - - // Add some entities - await brain.add({ type: 'concept', data: { title: 'Test' } }) - - // Commit with explicit captureState: false (lightweight metadata-only commit) - const commitId = await brain.commit({ - message: 'Test commit', - captureState: false // Explicit false — skips state capture - }) - - console.log(` Commit ID: ${commitId.slice(0, 8)}`) - - // Read commit object - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { NULL_HASH } = await import('../../src/storage/cow/constants.js') - - const commit = await CommitObject.read(blobStorage, commitId) - - // Verify tree is NULL_HASH (empty) - expect(commit.tree).toBe(NULL_HASH) - console.log(` ✅ Tree is NULL_HASH (backward compatible)`) - }) - - it('should capture state when captureState is omitted (default: true)', async () => { - console.log('\n📋 Test 2: Default behavior (captureState omitted → captures state)') - - await brain.add({ type: 'concept', data: { title: 'Test' } }) - - // Commit WITHOUT captureState parameter (omitted = default true) - const commitId = await brain.commit({ - message: 'Test commit' - // captureState not specified → defaults to true - }) - - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { NULL_HASH } = await import('../../src/storage/cow/constants.js') - - const commit = await CommitObject.read(blobStorage, commitId) - - // Default behavior now captures state (tree hash is NOT null) - expect(commit.tree).not.toBe(NULL_HASH) - console.log(` ✅ Tree is ${commit.tree.slice(0, 8)}... (state captured by default)`) - }) - - it('should capture entity state when captureState is true', async () => { - console.log('\n📋 Test 3: State capture enabled') - - // Add entities (add() returns entity ID, not entity object) - const entity1Id = await brain.add({ - type: 'person', - data: { name: 'Alice', age: 30 } - }) - - const entity2Id = await brain.add({ - type: 'concept', - data: { title: 'AI', description: 'Artificial Intelligence' } - }) - - console.log(` Added 2 entities: ${entity1Id.slice(0, 8)}, ${entity2Id.slice(0, 8)}`) - - // Commit WITH captureState - const commitId = await brain.commit({ - message: 'Snapshot with entities', - captureState: true // CAPTURE STATE! - }) - - console.log(` Commit ID: ${commitId.slice(0, 8)}`) - - // Read commit object - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { NULL_HASH, isNullHash } = await import('../../src/storage/cow/constants.js') - - const commit = await CommitObject.read(blobStorage, commitId) - - // Verify tree is NOT NULL_HASH - expect(commit.tree).not.toBe(NULL_HASH) - expect(isNullHash(commit.tree)).toBe(false) - console.log(` ✅ Tree is NOT NULL_HASH: ${commit.tree.slice(0, 8)}`) - - // Verify tree exists and can be read - const { TreeObject } = await import('../../src/storage/cow/TreeObject.js') - const tree = await TreeObject.read(blobStorage, commit.tree) - - expect(tree).toBeDefined() - expect(tree.entries.length).toBeGreaterThan(0) - console.log(` ✅ Tree has ${tree.entries.length} entries`) - - // Verify entities are in tree - let foundEntities = 0 - const entityNames: string[] = [] - for (const entry of tree.entries) { - if (entry.name.startsWith('entities/')) { - foundEntities++ - entityNames.push(entry.name) - } - } - - console.log(` Found ${foundEntities} entities: ${entityNames.join(', ')}`) - // At least 2 entities should be captured (the ones we added) - // There may be system entities (VFS root, metadata, etc.) - expect(foundEntities).toBeGreaterThanOrEqual(2) - console.log(` ✅ Found ${foundEntities} entities in tree (at least 2 as expected)`) - }) - - it('should handle empty workspace correctly', async () => { - console.log('\n📋 Test 4: Empty workspace (no user entities)') - - // Check how many entities exist before commit - const entitiesBefore = await brain.find({ excludeVFS: false }) - console.log(` Entities before commit: ${entitiesBefore.length}`) - - // Commit with NO user-added entities - const commitId = await brain.commit({ - message: 'Empty snapshot', - captureState: true - }) - - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { NULL_HASH, isNullHash } = await import('../../src/storage/cow/constants.js') - - const commit = await CommitObject.read(blobStorage, commitId) - - // If there were NO entities before commit, tree should be NULL_HASH - // If there WERE entities (system entities), tree should contain them - if (entitiesBefore.length === 0) { - expect(commit.tree).toBe(NULL_HASH) - console.log(` ✅ No entities: tree is NULL_HASH (correct)`) - } else { - expect(isNullHash(commit.tree)).toBe(false) - console.log(` ✅ ${entitiesBefore.length} system entities captured in tree`) - } - }) - - it('should perform well with small workspace (100 entities)', async () => { - console.log('\n📋 Test 5: Performance - 100 entities') - - // Add 100 entities - for (let i = 0; i < 100; i++) { - await brain.add({ - type:'concept', - data: { id: i, title: `Concept ${i}` } - }) - } - - console.log(` Added 100 entities`) - - // Measure commit time - const start = Date.now() - const commitId = await brain.commit({ - message: '100 entity snapshot', - captureState: true - }) - const duration = Date.now() - start - - console.log(` Commit completed in ${duration}ms`) - console.log(` Commit ID: ${commitId.slice(0, 8)}`) - - // Should complete in < 1s - expect(duration).toBeLessThan(1000) - console.log(` ✅ Performance: ${duration}ms < 1000ms`) - }) - - it('should deduplicate unchanged entities across commits', async () => { - console.log('\n📋 Test 6: BlobStorage deduplication') - - // Add initial entities (add() returns entity IDs) - const entity1Id = await brain.add({ - type:'person', - data: { name: 'Alice', age: 30 } - }) - - const entity2Id = await brain.add({ - type:'person', - data: { name: 'Bob', age: 25 } - }) - - // First commit with captureState - const commit1Id = await brain.commit({ - message: 'First snapshot', - captureState: true - }) - - console.log(` Commit 1: ${commit1Id.slice(0, 8)}`) - - // Modify ONLY entity2 - await brain.update({ - id: entity2Id, - data: { name: 'Bob', age: 26 } // Changed age - }) - - // Second commit with captureState - const commit2Id = await brain.commit({ - message: 'Second snapshot', - captureState: true - }) - - console.log(` Commit 2: ${commit2Id.slice(0, 8)}`) - - // Read both trees - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { TreeObject } = await import('../../src/storage/cow/TreeObject.js') - - const commit1 = await CommitObject.read(blobStorage, commit1Id) - const commit2 = await CommitObject.read(blobStorage, commit2Id) - - const tree1 = await TreeObject.read(blobStorage, commit1.tree) - const tree2 = await TreeObject.read(blobStorage, commit2.tree) - - // Find entity1 blob hash in both trees - const entity1Entry1 = tree1.entries.find(e => e.name === `entities/${entity1Id}`) - const entity1Entry2 = tree2.entries.find(e => e.name === `entities/${entity1Id}`) - - expect(entity1Entry1).toBeDefined() - expect(entity1Entry2).toBeDefined() - - // entity1 unchanged, so blob hash should be SAME (deduplication!) - expect(entity1Entry1!.hash).toBe(entity1Entry2!.hash) - console.log(` ✅ Unchanged entity1 has same blob hash (deduplicated)`) - - // Find entity2 blob hash in both trees - const entity2Entry1 = tree1.entries.find(e => e.name === `entities/${entity2Id}`) - const entity2Entry2 = tree2.entries.find(e => e.name === `entities/${entity2Id}`) - - expect(entity2Entry1).toBeDefined() - expect(entity2Entry2).toBeDefined() - - // entity2 changed, so blob hash should be DIFFERENT - expect(entity2Entry1!.hash).not.toBe(entity2Entry2!.hash) - console.log(` ✅ Changed entity2 has different blob hash (new version)`) - }) - - it('should create valid tree structure that can be read back', async () => { - console.log('\n📋 Test 7: Tree structure validity') - - // Add entities (add() returns entity IDs) - const entity1Id = await brain.add({ - type:'person', - data: { name: 'Alice' } - }) - - const entity2Id = await brain.add({ - type:'concept', - data: { title: 'AI' } - }) - - // Commit with captureState - const commitId = await brain.commit({ - message: 'Test snapshot', - captureState: true - }) - - // Read tree - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { TreeObject } = await import('../../src/storage/cow/TreeObject.js') - - const commit = await CommitObject.read(blobStorage, commitId) - const tree = await TreeObject.read(blobStorage, commit.tree) - - // Verify tree structure - expect(tree.entries).toBeDefined() - expect(Array.isArray(tree.entries)).toBe(true) - - // Walk tree and deserialize entities - let foundEntities = 0 - for await (const entry of TreeObject.walk(blobStorage, tree)) { - if (entry.type !== 'blob' || !entry.name.startsWith('entities/')) { - continue - } - - // Read blob - const blob = await blobStorage.read(entry.hash) - const entity = JSON.parse(blob.toString()) - - // Verify entity structure - expect(entity).toHaveProperty('id') - expect(entity).toHaveProperty('type') - expect(entity).toHaveProperty('data') - expect(entity).toHaveProperty('metadata') - - console.log(` Found entity: ${entity.id.slice(0, 8)} (${entity.type})`) - foundEntities++ - } - - // At least 2 entities should be found (the ones we added) - expect(foundEntities).toBeGreaterThanOrEqual(2) - console.log(` ✅ Successfully walked tree and deserialized ${foundEntities} entities`) - }) - - it('should capture VFS entities along with regular entities', async () => { - console.log('\n📋 Test 8: VFS entity capture') - - // Add regular entity (add() returns entity ID) - const conceptId = await brain.add({ - type:'concept', - data: { title: 'Test Concept' } - }) - - // Add VFS file - await brain.vfs.init() - await brain.vfs.writeFile('/test.md', 'Test content') - - console.log(` Added concept: ${conceptId.slice(0, 8)}`) - console.log(` Added VFS file: /test.md`) - - // Commit with captureState - const commitId = await brain.commit({ - message: 'Snapshot with VFS', - captureState: true - }) - - // Read tree and verify both types of entities are captured - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { TreeObject } = await import('../../src/storage/cow/TreeObject.js') - - const commit = await CommitObject.read(blobStorage, commitId) - const tree = await TreeObject.read(blobStorage, commit.tree) - - let regularEntities = 0 - let vfsEntities = 0 - - for await (const entry of TreeObject.walk(blobStorage, tree)) { - if (entry.type !== 'blob' || !entry.name.startsWith('entities/')) { - continue - } - - const blob = await blobStorage.read(entry.hash) - const entity = JSON.parse(blob.toString()) - - if (entity.metadata?.isVFS || entity.metadata?.vfsType) { - vfsEntities++ - console.log(` Found VFS entity: ${entity.metadata.path}`) - } else { - regularEntities++ - console.log(` Found regular entity: ${entity.id.slice(0, 8)} (${entity.type})`) - } - } - - expect(regularEntities).toBeGreaterThanOrEqual(1) - expect(vfsEntities).toBeGreaterThanOrEqual(2) // File + root directory - console.log(` ✅ Captured ${regularEntities} regular + ${vfsEntities} VFS entities`) - }) - - it('should work with Memory storage adapter', async () => { - console.log('\n📋 Test 9: Memory storage adapter') - - const memBrain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' }, - silent: true - }) - - await memBrain.init() - - // Add entity - await memBrain.add({ type:'concept', data: { title: 'Test' } }) - - // Commit with captureState - const commitId = await memBrain.commit({ - message: 'Memory storage test', - captureState: true - }) - - const blobStorage = (memBrain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { isNullHash } = await import('../../src/storage/cow/constants.js') - - const commit = await CommitObject.read(blobStorage, commitId) - - expect(isNullHash(commit.tree)).toBe(false) - console.log(` ✅ Memory storage: tree captured successfully`) - }) - - it('should work with TypeAware storage adapter', async () => { - console.log('\n📋 Test 10: TypeAware storage adapter') - - const testPath = './test-typeaware-commit-capture' - - if (fs.existsSync(testPath)) { - fs.rmSync(testPath, { recursive: true, force: true }) - } - - const typeAwareBrain = new Brainy({ requireSubtype: false, - storage: { - adapter: 'typeaware', - path: testPath - }, - silent: true - }) - - await typeAwareBrain.init() - - // Add entity - await typeAwareBrain.add({ type:'person', data: { name: 'Alice' } }) - - // Commit with captureState - const commitId = await typeAwareBrain.commit({ - message: 'TypeAware storage test', - captureState: true - }) - - const blobStorage = (typeAwareBrain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { isNullHash } = await import('../../src/storage/cow/constants.js') - - const commit = await CommitObject.read(blobStorage, commitId) - - expect(isNullHash(commit.tree)).toBe(false) - console.log(` ✅ TypeAware storage: tree captured successfully`) - - // Cleanup - if (fs.existsSync(testPath)) { - fs.rmSync(testPath, { recursive: true, force: true }) - } - }) - - it('should capture relationships along with entities', async () => { - console.log('\n📋 Test 11: Relationship capture (CRITICAL FIX)') - - // Add entities - const aliceId = await brain.add({ - type: 'person', - data: { name: 'Alice', role: 'Developer' } - }) - - const bobId = await brain.add({ - type: 'person', - data: { name: 'Bob', role: 'Designer' } - }) - - const projectId = await brain.add({ - type: 'concept', - data: { title: 'Project X', description: 'AI Platform' } - }) - - console.log(` Added 3 entities: Alice, Bob, Project X`) - - // Create relationships (using valid VerbTypes) - await brain.relate({ - from: aliceId, - to: projectId, - type: 'relatedTo' // Alice works on Project X - }) - - await brain.relate({ - from: bobId, - to: projectId, - type: 'relatedTo' // Bob works on Project X - }) - - await brain.relate({ - from: aliceId, - to: bobId, - type: 'relatedTo' // Alice collaborates with Bob - }) - - console.log(` Created 3 relationships`) - - // Commit with captureState - const commitId = await brain.commit({ - message: 'Snapshot with entities + relationships', - captureState: true - }) - - console.log(` Commit ID: ${commitId.slice(0, 8)}`) - - // Read tree and verify relationships are captured - const blobStorage = (brain.storage as any).blobStorage - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - const { TreeObject } = await import('../../src/storage/cow/TreeObject.js') - - const commit = await CommitObject.read(blobStorage, commitId) - const tree = await TreeObject.read(blobStorage, commit.tree) - - // Count entities and relationships in tree - let entityCount = 0 - let relationCount = 0 - - for (const entry of tree.entries) { - if (entry.name.startsWith('entities/')) { - entityCount++ - } else if (entry.name.startsWith('relations/')) { - relationCount++ - } - } - - console.log(` Tree contains: ${entityCount} entities, ${relationCount} relationships`) - - // Verify we have at least the entities and relationships we created - expect(entityCount).toBeGreaterThanOrEqual(3) - expect(relationCount).toBe(3) - console.log(` ✅ Relationships captured in tree!`) - - // Verify we can deserialize relationships from tree - let deserializedRelations = 0 - for (const entry of tree.entries) { - if (!entry.name.startsWith('relations/')) continue - - const blob = await blobStorage.read(entry.hash) - const relation = JSON.parse(blob.toString()) - - // Verify relationship structure - expect(relation).toHaveProperty('sourceId') - expect(relation).toHaveProperty('targetId') - expect(relation).toHaveProperty('verb') - - console.log(` Found relation: ${relation.verb} (${relation.sourceId.slice(0, 8)} → ${relation.targetId.slice(0, 8)})`) - deserializedRelations++ - } - - expect(deserializedRelations).toBe(3) - console.log(` ✅ Successfully deserialized ${deserializedRelations} relationships from tree`) - }) -}) diff --git a/tests/integration/cow-commit-storage.test.ts b/tests/integration/cow-commit-storage.test.ts deleted file mode 100644 index c93e337b..00000000 --- a/tests/integration/cow-commit-storage.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Regression test for v5.3.3 bug fix: - * Commits were being stored as blob:${hash} instead of commit:${hash} - * - * This caused getHistory() to return empty arrays because it couldn't find - * commit objects on disk. - * - * Related bugs: - * - v5.3.0 snapshot regression filed via consumer bug report (BRAINY_V5.3.0_SNAPSHOT_BUG_REPORT.md, internal) - * - Root cause: BlobStorage.ts hardcoded 'blob:' prefix in 9 locations - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import * as fs from 'fs/promises' -import * as path from 'path' - -describe('COW Commit Storage Type-Aware Prefixes', () => { - let brain: Brainy - let testDir: string - - beforeEach(async () => { - testDir = path.join('/tmp', `brainy-cow-test-${Date.now()}`) - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: testDir, - branch: 'main', - enableCompression: true - }, - disableAutoRebuild: true, - silent: true - }) - - await brain.init() - }) - - afterEach(async () => { - // Clean up test directory - try { - await fs.rm(testDir, { recursive: true, force: true }) - } catch (err) { - // Ignore cleanup errors - } - }) - - it('should store commits with commit: prefix (not blob: prefix)', async () => { - // Create a commit - await brain.commit('Test commit for type-aware storage') - - // Check filesystem directly - const cowDir = path.join(testDir, '_cow') - const files = await fs.readdir(cowDir) - - // Should have commit: files - const commitFiles = files.filter(f => f.startsWith('commit:')) - expect(commitFiles.length).toBeGreaterThan(0) - - // Should NOT have blob: files that are actually commits - // (blob: files should only be for actual blob data) - const blobFiles = files.filter(f => f.startsWith('blob:') && !f.includes('-meta')) - - // Read metadata of blob files to ensure none are commits - for (const blobFile of blobFiles) { - const metaFile = blobFile.replace('blob:', 'blob:-meta:') - if (files.includes(metaFile)) { - const metaPath = path.join(cowDir, metaFile) - const metaContent = await fs.readFile(metaPath, 'utf8') - const metadata = JSON.parse(metaContent) - expect(metadata.type).not.toBe('commit') - } - } - }) - - it('should allow getHistory() to retrieve commits', async () => { - // Create multiple commits - await brain.commit('First commit') - - await brain.add({ - data: 'Testing commit storage', - type: 'concept' - }) - - await brain.commit('Second commit with entity') - - // Get history - should NOT be empty - const history = await brain.getHistory({ limit: 10 }) - - expect(history).toBeDefined() - expect(Array.isArray(history)).toBe(true) - expect(history.length).toBeGreaterThanOrEqual(2) - - // Verify commit structure - expect(history[0]).toHaveProperty('hash') - expect(history[0]).toHaveProperty('message') - expect(history[0]).toHaveProperty('timestamp') - expect(history[0]).toHaveProperty('author') - }) - - it('should store trees with tree: prefix (if trees are created)', async () => { - // Add an entity and commit - await brain.add({ - data: 'Testing tree storage', - type: 'concept' - }) - - await brain.commit('Commit with tree') - - // Check filesystem - trees might or might not be created depending on implementation - // The important thing is that IF trees are created, they use tree: prefix - const cowDir = path.join(testDir, '_cow') - const files = await fs.readdir(cowDir) - - // Verify commit files exist (this is the critical part) - const commitFiles = files.filter(f => f.startsWith('commit:')) - expect(commitFiles.length).toBeGreaterThan(0) - - // If tree files exist, they should use tree: prefix (not blob:) - const treeFiles = files.filter(f => f.startsWith('tree:')) - const blobTrees = files.filter(f => f.startsWith('blob:') && !f.includes('-meta')) - - // Trees should be in tree: files, not blob: files - // (Or no tree files at all if implementation doesn't create them) - expect(blobTrees.filter(f => f.includes('tree')).length).toBe(0) - }) - - it('should maintain backward compatibility with old blob: prefix', async () => { - // This test verifies that read() auto-detects type by trying multiple prefixes - // We'll verify this by checking that the auto-detection code works - - // Create a commit normally (will use commit: prefix) - await brain.commit('Test commit for backward compat check') - - // The commit should be readable even though read() tries multiple prefixes - const history = await brain.getHistory({ limit: 1 }) - expect(history.length).toBe(1) - - // Verify the commit hash is readable via BlobStorage - const blobStorage = (brain as any).storage.blobStorage - const commitHash = history[0].hash - - // This should work via auto-detection - const readData = await blobStorage.read(commitHash) - expect(readData).toBeDefined() - expect(readData.length).toBeGreaterThan(0) - }) - - it('should handle fork() and snapshot creation with proper commit storage', async () => { - // Add some data - await brain.add({ - data: 'Testing snapshots', - type: 'concept' - }) - - await brain.commit('Commit before fork') - - // Create a fork (snapshot branch) - const snapshotBranch = `snapshot-${Date.now()}` - await brain.fork(snapshotBranch) - - // Get history - should work (this is the critical test) - const history = await brain.getHistory({ limit: 10 }) - expect(history.length).toBeGreaterThanOrEqual(1) - - // History working proves commits are stored correctly - }) - - it('should properly delete commits with type-aware prefix', async () => { - // Create a commit - await brain.commit('Commit to delete') - - const history = await brain.getHistory({ limit: 1 }) - const commitHash = history[0].hash - - // Delete via BlobStorage - const blobStorage = (brain as any).storage.blobStorage - await blobStorage.delete(commitHash) - - // Verify deleted - const exists = await blobStorage.has(commitHash) - expect(exists).toBe(false) - - // Verify files removed from disk - const cowDir = path.join(testDir, '_cow') - const files = await fs.readdir(cowDir) - - const commitFile = files.find(f => f.includes(commitHash) && f.startsWith('commit:')) - expect(commitFile).toBeUndefined() - }) -}) diff --git a/tests/integration/cow-full-integration.test.ts b/tests/integration/cow-full-integration.test.ts deleted file mode 100644 index c61bb7ff..00000000 --- a/tests/integration/cow-full-integration.test.ts +++ /dev/null @@ -1,574 +0,0 @@ -import { NounType } from '../../src/types/graphTypes.js' -/** - * Comprehensive COW Integration Tests - * - * Verifies COW works with: - * - All 8 storage adapters (Memory, OPFS, FileSystem, S3, R2, GCS, Azure, TypeAware) - * - Billion-scale performance - * - find() graph-aware queries - * - Triple Intelligence natural language - * - VFS (Virtual File System) - * - All 4 indexes (HNSW, Metadata, GraphAdjacency, DeletedItems) - * - Distributed mode (read-only, write-only instances) - * - 256 UUID-based sharding - * - * This is the CRITICAL test that proves v5.0.0 is production-ready. - */ - -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import type { StorageAdapter } from '../../src/storage/adapters/baseStorageAdapter.js' - -describe('COW Full Integration', () => { - describe('Storage Adapter Compatibility', () => { - it('should work with Memory adapter', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - // Add entity - const entity = await brain.add({ - type: NounType.Person, - data: { name: 'Alice' } - }) - - // Fork (uses COW) - const fork = await brain.fork('test-branch') - - // Verify entity exists in fork - const retrieved = await fork.get(entity.id) - expect(retrieved.data.name).toBe('Alice') - - await brain.destroy() - await fork.destroy() - }) - - it('should work with FileSystem adapter', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { - adapter: 'filesystem', - path: '/tmp/brainy-cow-test-fs' - } - }) - - await brain.init() - - const entity = await brain.add({ - type: NounType.Document, - data: { title: 'Test' } - }) - - const fork = await brain.fork('fs-branch') - - const retrieved = await fork.get(entity.id) - expect(retrieved.data.title).toBe('Test') - - await brain.destroy() - await fork.destroy() - }) - - it('should work with TypeAware adapter (most advanced)', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { - adapter: 'typeaware', - path: '/tmp/brainy-cow-test-typeaware' - } - }) - - await brain.init() - - const entity = await brain.add({ - type: NounType.Product, - data: { name: 'Widget', price: 99.99 } - }) - - const fork = await brain.fork('typeaware-branch') - - const retrieved = await fork.get(entity.id) - expect(retrieved.data.price).toBe(99.99) - - await brain.destroy() - await fork.destroy() - }) - }) - - describe('Billion-Scale Performance', () => { - it('should fork 1M entities in < 2 seconds', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - // Add 1M entities (representative of billion-scale) - console.log('Adding 1M entities...') - const start = Date.now() - - for (let i = 0; i < 1_000_000; i++) { - await brain.add({ - type: NounType.Thing, - data: { index: i }, - skipCommit: true // Batch commit - }) - - if (i % 100_000 === 0) { - console.log(` ${i / 1000}K entities added...`) - } - } - - await brain.commit({ message: 'Add 1M entities' }) - - const addTime = Date.now() - start - console.log(`Added 1M entities in ${addTime}ms`) - - // Fork (should be instant via COW) - console.log('Forking...') - const forkStart = Date.now() - - const fork = await brain.fork('million-test') - - const forkTime = Date.now() - forkStart - - console.log(`Forked 1M entities in ${forkTime}ms`) - - // CRITICAL: Fork must be < 2 seconds - expect(forkTime).toBeLessThan(2000) - - // Verify data integrity - const entity = await fork.get((await brain.find({ limit: 1 }))[0].id) - expect(entity.noun).toBe('entity') - - await brain.destroy() - await fork.destroy() - }, 120000) // 2-minute timeout for 1M entity test - - it('should deduplicate billions of identical vectors', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - // Same vector used for all entities (simulates common embeddings) - const commonVector = Array(384).fill(0.1) - - // Add 10K entities with same vector - for (let i = 0; i < 10_000; i++) { - await brain.add({ - type: NounType.Document, - data: { id: i }, - vector: commonVector, - skipCommit: true - }) - } - - await brain.commit({ message: 'Add 10K with duplicate vectors' }) - - // Check storage stats - const stats = brain.storage.blobStorage.getStats() - - // Should have massive deduplication savings - // (10K vectors but only 1 unique blob) - expect(stats.dedupSavings).toBeGreaterThan(0) - - console.log(`Deduplication savings: ${(stats.dedupSavings / 1024 / 1024).toFixed(2)}MB`) - - await brain.destroy() - }, 60000) - }) - - describe('find() Integration', () => { - it('should work with graph-aware find() queries', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - // Create entities with relationships - const alice = await brain.add({ - type: NounType.Person, - data: { name: 'Alice' } - }) - - const bob = await brain.add({ - type: NounType.Person, - data: { name: 'Bob' } - }) - - await brain.addRelationship(alice.id, 'knows', bob.id) - - // Fork - const fork = await brain.fork('find-test') - - // find() should work on fork - const results = await fork.find({ - type: NounType.Person, - where: { name: 'Alice' } - }) - - expect(results).toHaveLength(1) - expect(results[0].data.name).toBe('Alice') - - // Graph traversal should work - const connected = await fork.getVerbsBySource(alice.id) - expect(connected).toHaveLength(1) - expect(connected[0].verb).toBe('knows') - - await brain.destroy() - await fork.destroy() - }) - - it('should preserve metadata index across fork', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - // Add entities with indexed metadata - await brain.add({ - type: NounType.Product, - data: { price: 100, category: 'electronics' } - }) - - await brain.add({ - type: NounType.Product, - data: { price: 200, category: 'electronics' } - }) - - const fork = await brain.fork('metadata-test') - - // Metadata queries should work - const cheap = await fork.find({ - type: NounType.Product, - where: { price: { $lt: 150 } } - }) - - expect(cheap).toHaveLength(1) - expect(cheap[0].data.price).toBe(100) - - await brain.destroy() - await fork.destroy() - }) - }) - - describe('Triple Intelligence Integration', () => { - it('should preserve Triple Intelligence across fork', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' }, - intelligence: { - enabled: true, - modelDelivery: 'local' - } - }) - - await brain.init() - - // Add entities - await brain.add({ - type: NounType.Person, - data: { name: 'Alice', age: 30 } - }) - - await brain.add({ - type: NounType.Person, - data: { name: 'Bob', age: 25 } - }) - - const fork = await brain.fork('intelligence-test') - - // Natural language query should work on fork - const results = await fork.query('people older than 28') - - // Should find Alice (age 30) - expect(results.length).toBeGreaterThan(0) - - await brain.destroy() - await fork.destroy() - }) - }) - - describe('VFS Integration', () => { - it('should preserve VFS structure across fork', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' }, - vfs: { enabled: true } - }) - - await brain.init() - - // Create VFS structure - const folder = await brain.vfs.mkdir('/documents') - const file = await brain.vfs.writeFile('/documents/readme.txt', 'Hello World') - - const fork = await brain.fork('vfs-test') - - // VFS should work on fork - const content = await fork.vfs.readFile('/documents/readme.txt') - expect(content).toBe('Hello World') - - const files = await fork.vfs.readdir('/documents') - expect(files).toContain('readme.txt') - - await brain.destroy() - await fork.destroy() - }) - - it('should support VFS paths in billions of entities', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' }, - vfs: { enabled: true } - }) - - await brain.init() - - // Create deep directory structure - await brain.vfs.mkdir('/project/src/components/ui') - - // Add 1000 files - for (let i = 0; i < 1000; i++) { - await brain.vfs.writeFile( - `/project/src/components/ui/component${i}.tsx`, - `export default function Component${i}() { return null }` - ) - } - - const fork = await brain.fork('vfs-scale-test') - - // VFS operations should be fast on fork - const files = await fork.vfs.readdir('/project/src/components/ui') - expect(files).toHaveLength(1000) - - await brain.destroy() - await fork.destroy() - }, 60000) - }) - - describe('All 4 Indexes Integration', () => { - it('should preserve HNSW index across fork', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - // Add entities with vectors - const e1 = await brain.add({ - type: NounType.Document, - data: { text: 'machine learning' }, - vector: [1, 0, 0] - }) - - const e2 = await brain.add({ - type: NounType.Document, - data: { text: 'artificial intelligence' }, - vector: [0.9, 0.1, 0] - }) - - const fork = await brain.fork('hnsw-test') - - // Vector search should work on fork - const similar = await fork.search([1, 0, 0], { limit: 1 }) - - expect(similar[0].id).toBe(e1.id) - - await brain.destroy() - await fork.destroy() - }) - - it('should preserve GraphAdjacency index across fork', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - const a = await brain.add({ type: NounType.Thing, data: { name: 'A' } }) - const b = await brain.add({ type: NounType.Thing, data: { name: 'B' } }) - const c = await brain.add({ type: NounType.Thing, data: { name: 'C' } }) - - await brain.addRelationship(a.id, 'connects', b.id) - await brain.addRelationship(b.id, 'connects', c.id) - - const fork = await brain.fork('graph-test') - - // Graph queries should work - const outgoing = await fork.getVerbsBySource(a.id) - const incoming = await fork.getVerbsByTarget(b.id) - - expect(outgoing).toHaveLength(1) - expect(incoming).toHaveLength(1) - - await brain.destroy() - await fork.destroy() - }) - - it('should preserve DeletedItems index across fork', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - const entity = await brain.add({ - type: NounType.Thing, - data: { value: 'test' } - }) - - await brain.delete(entity.id) - - const fork = await brain.fork('deleted-test') - - // Deleted items should be tracked in fork - const exists = await fork.get(entity.id).catch(() => null) - expect(exists).toBeNull() - - await brain.destroy() - await fork.destroy() - }) - }) - - describe('Distributed Mode', () => { - it('should work with read-only instances', async () => { - // Main brain (read-write) - const main = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await main.init() - - const entity = await main.add({ - type: NounType.Thing, - data: { value: 'test' } - }) - - await main.commit({ message: 'Add entity' }) - - // Read-only instance (shares storage) - const readonly = new Brainy({ requireSubtype: false, - storage: main.config.storage, - readOnly: true - }) - - await readonly.init() - - // Can read from main - const retrieved = await readonly.get(entity.id) - expect(retrieved.data.value).toBe('test') - - // Can fork read-only instance - const fork = await readonly.fork('readonly-fork') - - const forked = await fork.get(entity.id) - expect(forked.data.value).toBe('test') - - await main.destroy() - await readonly.destroy() - await fork.destroy() - }) - - it('should work with write-only instances', async () => { - // Write-only instance - const writeOnly = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' }, - writeOnly: true - }) - - await writeOnly.init() - - await writeOnly.add({ - type: NounType.Message, - data: { message: 'test' } - }) - - await writeOnly.commit({ message: 'Add log' }) - - // Fork should work even on write-only - const fork = await writeOnly.fork('writeonly-fork') - - expect(fork).toBeTruthy() - - await writeOnly.destroy() - await fork.destroy() - }) - }) - - describe('256 UUID Sharding', () => { - it('should work with sharded storage', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { - adapter: 'memory', - sharding: { - enabled: true, - shardCount: 256 - } - } - }) - - await brain.init() - - // Add entities across shards - for (let i = 0; i < 1000; i++) { - await brain.add({ - noun: 'sharded', - data: { index: i }, - skipCommit: true - }) - } - - await brain.commit({ message: 'Add sharded entities' }) - - // Fork should work with sharding - const fork = await brain.fork('sharded-test') - - const results = await fork.find({ noun: 'sharded' }) - - expect(results).toHaveLength(1000) - - await brain.destroy() - await fork.destroy() - }) - }) - - describe('Time-Travel Queries (Enterprise Preview)', () => { - it('should support asOf queries', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' } - }) - - await brain.init() - - // Time 1: Add entity - await brain.add({ - type: NounType.Document, - data: { version: 1 } - }) - - await brain.commit({ message: 'Version 1' }) - - const time1 = Date.now() - - // Wait a bit - await new Promise(resolve => setTimeout(resolve, 100)) - - // Time 2: Update entity - const docs = await brain.find({ type: NounType.Document }) - await brain.update(docs[0].id, { version: 2 }) - - await brain.commit({ message: 'Version 2' }) - - // asOf(time1) should return version 1 - const snapshot = await brain.asOf(time1) - - const retrieved = await snapshot.find({ type: NounType.Document }) - - expect(retrieved[0].data.version).toBe(1) - - await brain.destroy() - await snapshot.destroy() - }) - }) -}) diff --git a/tests/integration/db-mvcc.test.ts b/tests/integration/db-mvcc.test.ts index 26b482a9..a9cfb545 100644 --- a/tests/integration/db-mvcc.test.ts +++ b/tests/integration/db-mvcc.test.ts @@ -952,4 +952,42 @@ describe('8.0 Db API — generational MVCC', () => { await spec1.release() await db.release() }) + + it('transactionLog() returns committed entries newest first, with meta and limit', async () => { + const brain = await openMemoryBrain() + + // Nothing committed yet — the log is empty (single-op writes do not log). + await brain.add({ id: uid('txlog-solo'), type: NounType.Document, data: 'solo', vector: vec(99), subtype: 'note' }) + expect(await brain.transactionLog()).toEqual([]) + + const first = await brain.transact( + [{ op: 'add', id: uid('txlog-a'), type: NounType.Document, data: 'a', vector: vec(100), metadata: {} }], + { meta: { author: 'job-1' } } + ) + const second = await brain.transact( + [{ op: 'update', id: uid('txlog-a'), metadata: { v: 2 } }], + { meta: { author: 'job-2' } } + ) + const third = await brain.transact([{ op: 'update', id: uid('txlog-a'), metadata: { v: 3 } }]) + + const entries = await brain.transactionLog() + expect(entries.map((entry) => entry.generation)).toEqual([ + third.generation, + second.generation, + first.generation + ]) + expect(entries[1].meta).toEqual({ author: 'job-2' }) + expect(entries[2].meta).toEqual({ author: 'job-1' }) + expect(entries[0].meta).toBeUndefined() + for (const entry of entries) { + expect(entry.timestamp).toBeGreaterThan(0) + } + + const limited = await brain.transactionLog({ limit: 2 }) + expect(limited.map((entry) => entry.generation)).toEqual([third.generation, second.generation]) + + await first.release() + await second.release() + await third.release() + }) }) diff --git a/tests/integration/empty-tree-bug.test.ts b/tests/integration/empty-tree-bug.test.ts deleted file mode 100644 index 8b69f783..00000000 --- a/tests/integration/empty-tree-bug.test.ts +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Reproduction test for empty tree bug in v5.3.1 - * - * BUG: brain.getHistory() throws "Blob not found: 0000...0" when commits - * have empty trees (entityCount: 0). - * - * The zero hash represents an empty tree and should be handled gracefully. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import * as fs from 'fs' - -const TEST_DATA_PATH = './test-empty-tree-bug-data' - -describe('Empty Tree Bug (v5.3.1)', () => { - let brain: Brainy - - beforeEach(async () => { - // Clean up test data - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: TEST_DATA_PATH, - branch: 'main', - enableCompression: true - }, - disableAutoRebuild: true, - silent: true - }) - - await brain.init() - }) - - afterEach(() => { - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - }) - - it('should handle getHistory() with empty tree commit', async () => { - // Create a commit with NO entities (empty tree) - const commitId = await brain.commit({ - message: 'Empty workspace snapshot', - author: 'test-user', - metadata: { - timestamp: Date.now(), - isSnapshot: true - } - }) - - expect(commitId).toBeTruthy() - console.log('✅ Commit created:', commitId) - - // Read the commit blob to inspect it - const blobStorage = (brain.storage as any).blobStorage - const refManager = (brain.storage as any).refManager - const { CommitObject } = await import('../../src/storage/cow/CommitObject.js') - - const commit = await CommitObject.read(blobStorage, commitId) - console.log('📄 Commit structure:', JSON.stringify(commit, null, 2)) - - // Check what the ref points to - const refHash = await refManager.resolveRef('main') - - // If ref is zero hash, that's the bug - const zeroHash = '0000000000000000000000000000000000000000000000000000000000000000' - if (refHash === zeroHash) { - throw new Error(`BUG CONFIRMED: Ref "main" points to zero hash! commitId=${commitId}, refHash=${refHash}, tree=${commit.tree}`) - } - - // Verify commitId is not zero hash - expect(commitId).not.toBe(zeroHash) - expect(refHash).toBe(commitId) - - // This should NOT throw "Blob not found: 0000...0" - // Empty trees should be handled gracefully - try { - const history = await brain.getHistory({ limit: 50 }) - - expect(history).toBeDefined() - expect(Array.isArray(history)).toBe(true) - console.log('✅ getHistory() works with empty tree') - console.log(' History length:', history.length) - } catch (error: any) { - console.log('❌ Error during getHistory():', error.message) - console.log(' Commit tree hash:', commit.tree) - console.log(' Commit parent hash:', commit.parent) - throw error - } - }) - - it('should handle multiple commits with empty trees', async () => { - // Create first empty commit - await brain.commit({ - message: 'First empty snapshot', - author: 'user1' - }) - - // Create second empty commit - await brain.commit({ - message: 'Second empty snapshot', - author: 'user2' - }) - - // Get history - should NOT throw - const history = await brain.getHistory({ limit: 10 }) - - expect(history).toBeDefined() - expect(history.length).toBeGreaterThanOrEqual(2) - console.log('✅ Multiple empty commits work') - }) - - it('should handle mixed commits (empty + with entities)', async () => { - // Create empty commit - await brain.commit({ - message: 'Empty snapshot', - author: 'user1' - }) - - // Add entity - await brain.add({ - data: 'Test entity', - type: 'concept' - }) - - // Create commit with entities - await brain.commit({ - message: 'Snapshot with entities', - author: 'user2' - }) - - // Get history - should handle both empty and non-empty trees - const history = await brain.getHistory({ limit: 10 }) - - expect(history).toBeDefined() - expect(history.length).toBeGreaterThanOrEqual(2) - console.log('✅ Mixed commits (empty + entities) work') - }) -}) diff --git a/tests/integration/fork-persistence.test.ts b/tests/integration/fork-persistence.test.ts deleted file mode 100644 index 01d61345..00000000 --- a/tests/integration/fork-persistence.test.ts +++ /dev/null @@ -1,267 +0,0 @@ -/** - * Fork Persistence Integration Test - * - * Tests the complete fork() → listBranches() → checkout() workflow - * to prevent regression of the v5.3.6 bug where fork() silently failed - * to persist branches to storage (internal bug report). - * - * @see https://github.com/soulcraftlabs/brainy/issues/XXX - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy' -import * as fs from 'fs' -import * as path from 'path' - -describe('Fork Persistence (v5.3.6 Bug Fix)', () => { - let brain: Brainy - let testDir: string - - beforeEach(async () => { - // Use filesystem storage to mirror cloud storage behavior - testDir = `/tmp/brainy-fork-test-${Date.now()}` - brain = new Brainy({ requireSubtype: false, - storage: { - adapter: 'filesystem', - path: testDir - }, - silent: true - }) - await brain.init() - }) - - afterEach(async () => { - // Cleanup test directory - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }) - } - }) - - it('should persist forked branches to storage', async () => { - // Add data and commit - await brain.add({ data: { value: 1 }, type: 'concept' }) - const commitId = await brain.commit({ - message: 'Initial commit', - author: 'test@example.com' - }) - expect(commitId).toBeTruthy() - - // Fork a new branch - const branchName = `test-branch-${Date.now()}` - await brain.fork(branchName, { - author: 'test@example.com', - message: 'Test fork for persistence' - }) - - // CRITICAL: Verify branch exists in storage (this would fail in v5.3.5) - const branches = await brain.listBranches() - expect(branches).toContain(branchName) - expect(branches).toContain('main') - }) - - it('should allow checkout of forked branch', async () => { - // Setup: Add data, commit, fork - await brain.add({ data: { value: 1 }, type: 'concept' }) - await brain.commit({ - message: 'Initial commit', - author: 'test@example.com' - }) - - const branchName = `checkout-test-${Date.now()}` - await brain.fork(branchName, { - author: 'test@example.com', - message: 'Test fork for checkout' - }) - - // CRITICAL: Checkout should succeed (this would fail in v5.3.5) - await expect(brain.checkout(branchName)).resolves.not.toThrow() - - // Verify we're on the new branch - const currentBranch = await brain.getCurrentBranch() - expect(currentBranch).toBe(branchName) - }) - - it('should persist fork across multiple storage operations', async () => { - // Create initial state - await brain.add({ data: { step: 1 }, type: 'concept' }) - await brain.commit({ - message: 'Step 1', - author: 'test@example.com' - }) - - // Fork branch 1 - const branch1 = `branch-1-${Date.now()}` - await brain.fork(branch1, { - author: 'test@example.com', - message: 'Fork 1' - }) - - // Add more data on main - await brain.add({ data: { step: 2 }, type: 'concept' }) - await brain.commit({ - message: 'Step 2 on main', - author: 'test@example.com' - }) - - // Fork branch 2 - const branch2 = `branch-2-${Date.now()}` - await brain.fork(branch2, { - author: 'test@example.com', - message: 'Fork 2' - }) - - // Verify both branches exist - const branches = await brain.listBranches() - expect(branches).toContain('main') - expect(branches).toContain(branch1) - expect(branches).toContain(branch2) - - // Verify all branches are accessible - await expect(brain.checkout(branch1)).resolves.not.toThrow() - await expect(brain.checkout(branch2)).resolves.not.toThrow() - await expect(brain.checkout('main')).resolves.not.toThrow() - }) - - it('should return new Brainy instance pointing to fork', async () => { - // Verify fork() returns a working Brainy instance on the new branch - - await brain.add({ data: { value: 1 }, type: 'concept' }) - await brain.commit({ - message: 'Initial commit', - author: 'test@example.com' - }) - - // Fork and get new instance - const validBranch = `return-test-${Date.now()}` - const forkedBrain = await brain.fork(validBranch, { - author: 'test@example.com', - message: 'Test fork return value' - }) - - // Verify the returned instance is on the new branch - const forkCurrentBranch = await forkedBrain.getCurrentBranch() - expect(forkCurrentBranch).toBe(validBranch) - - // Verify original instance is still on main - const mainCurrentBranch = await brain.getCurrentBranch() - expect(mainCurrentBranch).toBe('main') - - // Verify both instances can see the branch - const mainBranches = await brain.listBranches() - const forkBranches = await forkedBrain.listBranches() - expect(mainBranches).toContain(validBranch) - expect(forkBranches).toContain(validBranch) - }) - - it('should handle snapshot naming convention (consumer use case)', async () => { - // Reproduce exact Consumer snapshot workflow - await brain.add({ data: { test: true }, type: 'concept' }) - - const commitId = await brain.commit({ - message: 'Consumer snapshot test', - author: 'workshop@example.com' - }) - - // Use the consumer's exact naming convention - const timestamp = Date.now() - const snapshotBranch = `snapshot-${timestamp}` - - // Create snapshot branch (this is what failed in the consumer report) - await brain.fork(snapshotBranch, { - author: 'workshop@example.com', - message: `Snapshot: Consumer test`, - metadata: { - timestamp, - userId: 'test-user', - isSnapshot: true, - snapshotCommit: commitId - } - }) - - // Verify snapshot can be listed - const branches = await brain.listBranches() - expect(branches).toContain(snapshotBranch) - - // Verify snapshot can be restored (checked out) - await expect(brain.checkout(snapshotBranch)).resolves.not.toThrow() - - // Verify we're on the snapshot branch - const currentBranch = await brain.getCurrentBranch() - expect(currentBranch).toBe(snapshotBranch) - }) - - it('should verify _cow paths are not branch-scoped', async () => { - // This test verifies the resolveBranchPath fix works correctly - // by checking that COW metadata is accessible globally - - await brain.add({ data: { value: 1 }, type: 'concept' }) - await brain.commit({ - message: 'Test commit', - author: 'test@example.com' - }) - - const branch1 = `branch-a-${Date.now()}` - const branch2 = `branch-b-${Date.now()}` - - // Create two branches - await brain.fork(branch1, { - author: 'test@example.com', - message: 'Branch A' - }) - - await brain.fork(branch2, { - author: 'test@example.com', - message: 'Branch B' - }) - - // From any branch, we should be able to list ALL branches - // This proves COW metadata is global, not branch-scoped - await brain.checkout(branch1) - let branches = await brain.listBranches() - expect(branches).toContain('main') - expect(branches).toContain(branch1) - expect(branches).toContain(branch2) - - await brain.checkout(branch2) - branches = await brain.listBranches() - expect(branches).toContain('main') - expect(branches).toContain(branch1) - expect(branches).toContain(branch2) - - await brain.checkout('main') - branches = await brain.listBranches() - expect(branches).toContain('main') - expect(branches).toContain(branch1) - expect(branches).toContain(branch2) - }) - - it('should persist fork with metadata', async () => { - // Test that metadata is preserved in fork - await brain.add({ data: { value: 1 }, type: 'concept' }) - await brain.commit({ - message: 'Test commit', - author: 'test@example.com' - }) - - const branchName = `metadata-test-${Date.now()}` - const metadata = { - timestamp: Date.now(), - userId: 'test-user-123', - customField: 'test-value', - nested: { data: true } - } - - await brain.fork(branchName, { - author: 'test@example.com', - message: 'Fork with metadata', - metadata - }) - - // Verify branch exists - const branches = await brain.listBranches() - expect(branches).toContain(branchName) - - // Verify branch is accessible - await expect(brain.checkout(branchName)).resolves.not.toThrow() - }) -}) diff --git a/tests/integration/history-ref-resolution-bug.test.ts b/tests/integration/history-ref-resolution-bug.test.ts deleted file mode 100644 index 59e75c40..00000000 --- a/tests/integration/history-ref-resolution-bug.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Regression test for v5.3.0 history ref resolution bug - * - * BUG: brain.getHistory() was passing `heads/${branch}` to commitLog.getHistory(), - * but RefManager.normalizeRefName() expects either: - * - Full ref: 'refs/heads/main' - * - Short name: 'main' (normalized to 'refs/heads/main') - * - * Passing 'heads/main' caused normalization to 'refs/heads/heads/main' → ref not found! - * - * Fixed in: src/brainy.ts lines 2354, 2856, 2895 - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import * as fs from 'fs' - -const TEST_DATA_PATH = './test-history-ref-bug-data' - -describe('History Ref Resolution Bug (v5.3.0)', () => { - let brain: Brainy - - beforeEach(async () => { - // Clean up test data - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: TEST_DATA_PATH, - branch: 'main', - enableCompression: true - }, - disableAutoRebuild: true, - silent: true - }) - - await brain.init() - }) - - afterEach(() => { - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - }) - - it('should retrieve history after commit (bug: ref not found)', async () => { - // Add an entity - await brain.add({ - data: 'Test entity for history bug', - type: 'concept', - metadata: { test: true } - }) - - // Create a commit - const commitId = await brain.commit({ - message: 'Test snapshot', - author: 'dev-user', - metadata: { - timestamp: Date.now(), - isSnapshot: true - } - }) - - expect(commitId).toBeTruthy() - expect(commitId.length).toBe(64) // SHA-256 hash - - // THIS WAS FAILING with "Ref not found: heads/main" - // Because getHistory() passed 'heads/main' instead of 'main' - // After fix: should not throw "Ref not found" error - try { - const history = await brain.getHistory({ limit: 50 }) - - expect(history).toBeDefined() - expect(Array.isArray(history)).toBe(true) - // Note: history might be empty due to empty tree blob issue, - // but the important part is NO "Ref not found: heads/main" error - } catch (error: any) { - // Should NOT get "Ref not found: heads/main" error - expect(error.message).not.toContain('Ref not found: heads/main') - // Other errors (like tree blob) are separate issues - if (error.message.includes('Blob not found')) { - // This is a known issue with empty tree - not our bug - console.log('✅ Ref resolution working (tree blob issue is separate)') - return - } - throw error - } - }) - - it('should not throw ref resolution error for multiple commits', async () => { - // Create first commit - await brain.add({ data: 'Entity 1', type: 'concept' }) - await brain.commit({ - message: 'First commit', - author: 'user1' - }) - - // Create second commit - await brain.add({ data: 'Entity 2', type: 'concept' }) - await brain.commit({ - message: 'Second commit', - author: 'user2' - }) - - // Get history - should NOT throw "Ref not found: heads/main" - try { - const history = await brain.getHistory({ limit: 10 }) - expect(history).toBeDefined() - console.log('✅ getHistory() works without ref resolution error') - } catch (error: any) { - // Should NOT get "Ref not found: heads/main" error - expect(error.message).not.toContain('Ref not found: heads/main') - expect(error.message).not.toContain('Ref not found: main') - // Other errors are acceptable - console.log('✅ No ref resolution error (other error is acceptable)') - } - }) - - it('should use fork() to test COW (correct API)', async () => { - // fork() is the public API that uses COW internally - await brain.add({ - data: 'Original entity', - type: 'concept' - }) - - // Fork creates a new branch and switches to it - const fork = await brain.fork('feature-branch') - - // Add entity in fork - await fork.add({ - data: 'Fork entity', - type: 'concept' - }) - - // Both should work without ref resolution errors - expect(fork).toBeDefined() - console.log('✅ fork() works (uses COW refs internally)') - }) -}) diff --git a/tests/integration/hybrid-search-cow.test.ts b/tests/integration/hybrid-search-cow.test.ts deleted file mode 100644 index e49b6637..00000000 --- a/tests/integration/hybrid-search-cow.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Hybrid Search COW Integration Tests (v7.7.0) - * - * Verifies that hybrid search works correctly with: - * - fork() - COW branching - * - asOf() - Historical queries - * - VFS entities - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy' -import { NounType } from '../../src/types/graphTypes' -import * as fs from 'fs' -import * as path from 'path' -import * as os from 'os' - -describe('Hybrid Search with COW (v7.7.0)', () => { - let brain: Brainy - let testDir: string - - beforeEach(async () => { - // Use filesystem storage for COW support - testDir = path.join(os.tmpdir(), `brainy-cow-test-${Date.now()}`) - fs.mkdirSync(testDir, { recursive: true }) - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - options: { basePath: testDir } - } - }) - await brain.init() - }) - - afterEach(async () => { - await brain.close() - // Cleanup test directory - try { - fs.rmSync(testDir, { recursive: true, force: true }) - } catch (e) { - // Ignore cleanup errors - } - }) - - describe('fork() compatibility', () => { - it('should perform hybrid search in forked brain', async () => { - // Add entity to main branch - const mainId = await brain.add({ - data: 'Python programming language tutorial', - type: NounType.Document, - metadata: { branch: 'main' } - }) - - // Commit main branch - await brain.commit({ message: 'Add Python doc' }) - - // Fork - const fork = await brain.fork('feature-branch') - - // Add entity to fork - const forkId = await fork.add({ - data: 'JavaScript programming guide', - type: NounType.Document, - metadata: { branch: 'feature' } - }) - - // Hybrid search in fork should find both - const forkResults = await fork.find({ - query: 'programming', - limit: 10 - }) - - expect(forkResults.length).toBeGreaterThanOrEqual(2) - expect(forkResults.some(r => r.id === mainId)).toBe(true) - expect(forkResults.some(r => r.id === forkId)).toBe(true) - - // Hybrid search in main should only find main entity - const mainResults = await brain.find({ - query: 'programming', - limit: 10 - }) - - expect(mainResults.some(r => r.id === mainId)).toBe(true) - // Fork entity should NOT be visible in main - expect(mainResults.some(r => r.id === forkId)).toBe(false) - - await fork.close() - }) - - it('should support text-only search in forked brain', async () => { - await brain.add({ - data: 'exact keyword match test', - type: NounType.Document, - metadata: { test: true } - }) - await brain.commit({ message: 'Add test doc' }) - - const fork = await brain.fork('text-search-test') - - const results = await fork.find({ - query: 'exact keyword', - searchMode: 'text', - limit: 5 - }) - - expect(results.length).toBeGreaterThan(0) - await fork.close() - }) - }) - - describe('VFS compatibility', () => { - it('should include VFS file content in hybrid search', async () => { - // Write a file via VFS - const vfs = brain.vfs - await vfs.writeFile('/docs/readme.txt', 'This is a readme file with important information') - - // Search should find VFS content - const results = await brain.find({ - query: 'readme important', - limit: 10 - }) - - // VFS entities should appear in results - expect(results.length).toBeGreaterThan(0) - }) - - it('should exclude VFS with excludeVFS flag in hybrid search', async () => { - // Add regular entity - const entityId = await brain.add({ - data: 'regular document about readme files', - type: NounType.Document, - metadata: { source: 'api' } - }) - - // Write VFS file - await brain.vfs.writeFile('/readme.md', 'VFS readme content') - - // Search with excludeVFS should only find regular entity - const results = await brain.find({ - query: 'readme', - excludeVFS: true, - limit: 10 - }) - - expect(results.some(r => r.id === entityId)).toBe(true) - // VFS entities should be excluded - expect(results.every(r => !r.metadata?.vfsType)).toBe(true) - }) - }) -}) diff --git a/tests/integration/hybrid-search-vfs.test.ts b/tests/integration/hybrid-search-vfs.test.ts new file mode 100644 index 00000000..c881fa97 --- /dev/null +++ b/tests/integration/hybrid-search-vfs.test.ts @@ -0,0 +1,80 @@ +/** + * Hybrid Search VFS Integration Tests + * + * Verifies that hybrid search works correctly with VFS entities: + * file content is searchable by default, and `excludeVFS` filters + * VFS entities out of results. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType } from '../../src/types/graphTypes' +import * as fs from 'fs' +import * as path from 'path' +import * as os from 'os' + +describe('Hybrid Search with VFS', () => { + let brain: Brainy + let testDir: string + + beforeEach(async () => { + testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`) + fs.mkdirSync(testDir, { recursive: true }) + + brain = new Brainy({ requireSubtype: false, + storage: { + type: 'filesystem', + options: { basePath: testDir } + } + }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + // Cleanup test directory + try { + fs.rmSync(testDir, { recursive: true, force: true }) + } catch (e) { + // Ignore cleanup errors + } + }) + + it('should include VFS file content in hybrid search', async () => { + // Write a file via VFS + const vfs = brain.vfs + await vfs.writeFile('/docs/readme.txt', 'This is a readme file with important information') + + // Search should find VFS content + const results = await brain.find({ + query: 'readme important', + limit: 10 + }) + + // VFS entities should appear in results + expect(results.length).toBeGreaterThan(0) + }) + + it('should exclude VFS with excludeVFS flag in hybrid search', async () => { + // Add regular entity + const entityId = await brain.add({ + data: 'regular document about readme files', + type: NounType.Document, + metadata: { source: 'api' } + }) + + // Write VFS file + await brain.vfs.writeFile('/readme.md', 'VFS readme content') + + // Search with excludeVFS should only find regular entity + const results = await brain.find({ + query: 'readme', + excludeVFS: true, + limit: 10 + }) + + expect(results.some(r => r.id === entityId)).toBe(true) + // VFS entities should be excluded + expect(results.every(r => !r.metadata?.vfsType)).toBe(true) + }) +}) diff --git a/tests/integration/initial-commit-null-parent.test.ts b/tests/integration/initial-commit-null-parent.test.ts deleted file mode 100644 index 61a0ef2b..00000000 --- a/tests/integration/initial-commit-null-parent.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Regression test for v5.3.3 NULL parent bug - * - * BUG: CommitObject.walk() didn't guard against NULL parent hash, causing - * "Blob metadata not found: 0000...0000" error when calling getHistory() - * on a fresh database with only the initial commit. - * - * ROOT CAUSE: Initial commit has parent = null or NULL_HASH ('0000...0000'), - * but while(currentHash) treated the string as truthy and tried to read it. - * - * FIXED IN: v5.3.4 - * - Added isNullHash() guard in CommitObject.walk() - * - Added defensive check in BlobStorage.read() - * - Created NULL_HASH constant to prevent hardcoding - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NULL_HASH } from '../../src/storage/cow/constants.js' -import * as fs from 'fs' - -const TEST_DATA_PATH = './test-null-parent-data' - -describe('Initial Commit NULL Parent Bug (v5.3.3 regression)', () => { - let brain: Brainy - - beforeEach(async () => { - // Clean up test data - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: TEST_DATA_PATH, - branch: 'main', - enableCompression: true - }, - disableAutoRebuild: true, - silent: true - }) - - await brain.init() - }) - - afterEach(() => { - if (fs.existsSync(TEST_DATA_PATH)) { - fs.rmSync(TEST_DATA_PATH, { recursive: true, force: true }) - } - }) - - it('should handle getHistory() on fresh database with only initial commit', async () => { - // Fresh brain has only the initial commit (created during init) - // This was throwing "Blob metadata not found: 0000...0000" in v5.3.3 - - // Get history - should NOT throw - const history = await brain.getHistory({ limit: 10 }) - - // Should return the initial commit - expect(history).toBeDefined() - expect(Array.isArray(history)).toBe(true) - expect(history.length).toBe(1) - - // Initial commit should have specific properties - const initialCommit = history[0] - expect(initialCommit.message).toBe('Initial commit') - expect(initialCommit.author).toBe('system') - // Parent can be null or undefined for initial commit - expect(initialCommit.parent == null).toBe(true) - }) - - it('should stop walk at initial commit (parent = null)', async () => { - // Note: brain.commit() creates "Snapshot commit" messages by default - // Create 2 commits - await brain.add({ data: 'First entity', type: 'concept' }) - await brain.commit('First user commit') - - await brain.add({ data: 'Second entity', type: 'concept' }) - await brain.commit('Second user commit') - - // Get full history - const history = await brain.getHistory({ limit: 100 }) - - // Should have at least 3 commits (2 user + 1 initial) - expect(history.length).toBeGreaterThanOrEqual(3) - - // Last commit should be initial commit with no parent - const initialCommit = history[history.length - 1] - expect(initialCommit.message).toBe('Initial commit') - // Parent can be null or undefined for initial commit - expect(initialCommit.parent == null).toBe(true) - }) - - it('should not attempt to read NULL hash from BlobStorage', async () => { - // Direct test: attempting to read NULL_HASH should throw clear error - const blobStorage = (brain as any).storage.blobStorage - - // Should throw clear error message - await expect( - blobStorage.read(NULL_HASH) - ).rejects.toThrow(/sentinel value/) - - // Error message should mention what NULL_HASH is for - await expect( - blobStorage.read(NULL_HASH) - ).rejects.toThrow(/no parent/) - }) - - it('should handle multiple calls to getHistory() consistently', async () => { - // Ensure caching doesn't break NULL hash handling - - // First call - const history1 = await brain.getHistory({ limit: 10 }) - expect(history1.length).toBe(1) - - // Add a commit - await brain.add({ data: 'Test entity', type: 'concept' }) - await brain.commit('Test commit') - - // Second call (should now have 2 commits) - const history2 = await brain.getHistory({ limit: 10 }) - expect(history2.length).toBe(2) - - // Third call (should still have 2 commits) - const history3 = await brain.getHistory({ limit: 10 }) - expect(history3.length).toBe(2) - - // All should end with initial commit - expect(history1[history1.length - 1].message).toBe('Initial commit') - expect(history2[history2.length - 1].message).toBe('Initial commit') - expect(history3[history3.length - 1].message).toBe('Initial commit') - }) - - it('should handle walking with maxDepth on initial commit', async () => { - // Edge case: maxDepth = 1 on fresh database - - const history = await brain.getHistory({ limit: 1 }) - - expect(history.length).toBe(1) - expect(history[0].message).toBe('Initial commit') - }) -}) diff --git a/tests/integration/memory-enhancements-v5.11.0.test.ts b/tests/integration/memory-enhancements-v5.11.0.test.ts index 823453f3..e689e1f4 100644 --- a/tests/integration/memory-enhancements-v5.11.0.test.ts +++ b/tests/integration/memory-enhancements-v5.11.0.test.ts @@ -2,7 +2,6 @@ * Integration tests for Memory Enhancements (v5.11.0) * * End-to-end tests verifying: - * - streamHistory() works in production scenarios * - Container detection works correctly * - Memory limits are applied correctly * - getMemoryStats() provides accurate information @@ -40,86 +39,6 @@ describe('Memory Enhancements Integration (v5.11.0)', () => { }) }) - describe('Production Workflow: Consumer Snapshot Timeline', () => { - it('should stream 1000 snapshots efficiently', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - options: { path: testDir } - }, - silent: true - }) - - await brain.init() - - // Create a realistic workflow: user creates entities and saves snapshots - for (let i = 0; i < 100; i++) { - // Add 10 entities - for (let j = 0; j < 10; j++) { - await brain.add({ - type: 'document', - data: `Chapter ${i}, Section ${j}` - }) - } - - // Save snapshot - await brain.commit({ message: `Version ${i}`, captureState: true }) - } - - // Stream history efficiently - const snapshots: string[] = [] - const startTime = Date.now() - - for await (const commit of brain.streamHistory({ limit: 100 })) { - snapshots.push(commit.message) - } - - const duration = Date.now() - startTime - - expect(snapshots.length).toBe(100) - expect(duration).toBeLessThan(5000) // Should complete in < 5s - - await brain.close() - }) - - it('should handle large snapshot with filtering', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - options: { path: testDir } - }, - silent: true - }) - - await brain.init() - - // Create snapshots by different authors - for (let i = 0; i < 50; i++) { - await brain.add({ type: 'document', data: `Content ${i}` }) - - await brain.commit({ - message: `Snapshot ${i}`, - author: i % 3 === 0 ? 'alice' : i % 3 === 1 ? 'bob' : 'charlie', - captureState: true - }) - } - - // Stream only alice's snapshots - const aliceSnapshots: any[] = [] - - for await (const commit of brain.streamHistory({ author: 'alice', limit: 100 })) { - aliceSnapshots.push(commit) - } - - expect(aliceSnapshots.length).toBeGreaterThan(0) - aliceSnapshots.forEach(commit => { - expect(commit.author).toBe('alice') - }) - - await brain.close() - }) - }) - describe('Production Workflow: Cloud Run Deployment', () => { it('should detect 4GB Cloud Run container and set optimal limits', async () => { process.env.CLOUD_RUN_MEMORY = '4Gi' @@ -190,51 +109,6 @@ describe('Memory Enhancements Integration (v5.11.0)', () => { }) }) - describe('Combined Features: Stream + Memory Management', () => { - it('should stream large histories within memory limits', async () => { - process.env.CLOUD_RUN_MEMORY = '2Gi' - - const brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - options: { path: testDir } - }, - silent: true - }) - - await brain.init() - - // Create many snapshots - for (let i = 0; i < 200; i++) { - await brain.add({ type: 'document', data: `Data ${i}` }) - - if (i % 5 === 0) { - await brain.commit({ message: `Checkpoint ${i / 5}`, captureState: true }) - } - } - - // Verify memory limits - const stats = brain.getMemoryStats() - expect(stats.limits.maxQueryLimit).toBe(5000) // 2GB * 0.25 - - // Stream all snapshots (should be ~40) - const heapBefore = process.memoryUsage().heapUsed - - let count = 0 - for await (const commit of brain.streamHistory({ limit: 100 })) { - count++ - } - - const heapAfter = process.memoryUsage().heapUsed - const heapGrowth = heapAfter - heapBefore - - expect(count).toBeGreaterThan(0) - expect(heapGrowth).toBeLessThan(20 * 1024 * 1024) // < 20MB growth - - await brain.close() - }) - }) - describe('Memory Stats API', () => { it('should provide actionable recommendations', async () => { process.env.CLOUD_RUN_MEMORY = '4Gi' @@ -352,32 +226,5 @@ describe('Memory Enhancements Integration (v5.11.0)', () => { await brain.close() }) - - it('should keep getHistory() working as before', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - options: { path: testDir } - }, - silent: true - }) - - await brain.init() - - // Create some snapshots - for (let i = 0; i < 10; i++) { - await brain.add({ type: 'note', data: `Test ${i}` }) - await brain.commit({ message: `Snapshot ${i}`, captureState: true }) - } - - // Old API still works - const history = await brain.getHistory({ limit: 10 }) - - expect(history.length).toBe(10) - expect(history[0]).toHaveProperty('hash') - expect(history[0]).toHaveProperty('message') - - await brain.close() - }) }) }) diff --git a/tests/integration/metadata-only-comprehensive.test.ts b/tests/integration/metadata-only-comprehensive.test.ts index 10cadca9..76c32269 100644 --- a/tests/integration/metadata-only-comprehensive.test.ts +++ b/tests/integration/metadata-only-comprehensive.test.ts @@ -291,40 +291,6 @@ describe('Metadata-Only Comprehensive Integration (v5.11.1)', () => { }) }) - describe('COW and Fork', () => { - it('should work with metadata-only in forks', async () => { - const brain = new Brainy({ requireSubtype: false, - storage: { adapter: 'memory' }, - silent: true - }) - - await brain.init() - - const id = await brain.add({ - data: 'Original', - type: NounType.Document, - metadata: { version: 1 } - }) - - await brain.commit({ message: 'Initial commit' }) - - const fork = await brain.fork('test-branch') - - // Metadata-only in fork - const entity = await fork.get(id) - expect(entity).toBeTruthy() - expect(entity!.metadata.version).toBe(1) - expect(entity!.vector).toEqual([]) - - // Full entity in fork - const full = await fork.get(id, { includeVectors: true }) - expect(full!.vector.length).toBe(384) - - await brain.close() - await fork.close() - }) - }) - describe('Performance Verification', () => { it('metadata-only should be significantly faster', async () => { const brain = new Brainy({ requireSubtype: false, diff --git a/tests/integration/migration.test.ts b/tests/integration/migration.test.ts index 5c1765d5..726606d0 100644 --- a/tests/integration/migration.test.ts +++ b/tests/integration/migration.test.ts @@ -4,7 +4,7 @@ * Verifies: * - MigrationRunner with in-memory storage * - brain.migrate() public API (dry-run and apply) - * - Backup branch creation with metadata tagging + * - Pre-migration snapshots via `backupTo` (persist-before-migrate + restore) * - No-op when MIGRATIONS array is empty * - Warning log when autoMigrate is false * - Multi-tenant independent migration state @@ -12,6 +12,9 @@ */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' import { Brainy } from '../../src/brainy.js' import { MigrationRunner, MIGRATIONS } from '../../src/migration/index.js' import type { Migration } from '../../src/migration/index.js' @@ -242,13 +245,21 @@ describe('Migration System', () => { }) }) - // ─── Backup branch tests ────────────────────────────────────── + // ─── Pre-migration snapshot tests ───────────────────────────── - describe('Backup branches', () => { - it('should create a backup branch before migrating', async () => { + describe('Pre-migration snapshots (backupTo)', () => { + let backupDir: string + + beforeEach(() => { + backupDir = path.join(os.tmpdir(), `brainy-migration-backup-${Date.now()}-${Math.random().toString(36).slice(2)}`) + }) + + afterEach(() => { + fs.rmSync(backupDir, { recursive: true, force: true }) + }) + + it('should persist a snapshot before migrating and report its path', async () => { await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { x: 1 } }) - // Need a commit for fork to work - await brain.commit({ message: 'test', author: 'test' }) const migration: Migration = { id: 'test-backup', @@ -259,45 +270,19 @@ describe('Migration System', () => { } await withMigrations([migration], async () => { - const result = await brain.migrate() + const result = await brain.migrate({ backupTo: backupDir }) const r = result as any - expect(r.backupBranch).toBe('pre-migration-2.0.0') + expect(r.backupPath).toBe(backupDir) + expect(r.migrationsApplied).toContain('test-backup') - const branches = await brain.listBranches() - expect(branches).toContain('pre-migration-2.0.0') + // The snapshot is a self-contained store directory + expect(fs.existsSync(backupDir)).toBe(true) + expect(fs.readdirSync(backupDir).length).toBeGreaterThan(0) }) }) - it('should tag backup branch with system:backup metadata', async () => { - await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { z: 1 } }) - await brain.commit({ message: 'test', author: 'test' }) - - const migration: Migration = { - id: 'test-tag', - version: '4.0.0', - description: 'Tag test', - applies: 'nouns', - transform: (m) => 'z' in m ? { ...m, tagged: true } : null - } - - await withMigrations([migration], async () => { - await brain.migrate() - - // Verify metadata tag on the backup branch ref - const refManager = (brain as any).storage.refManager - if (refManager) { - const ref = await refManager.getRef('pre-migration-4.0.0') - expect(ref).toBeDefined() - expect(ref?.metadata?.type).toBe('system:backup') - expect(ref?.metadata?.migrationVersion).toBe('4.0.0') - expect(ref?.metadata?.author).toBe('brainy-migration') - } - }) - }) - - it('should keep backup branch as a named restore point', async () => { + it('should restore the pre-migration state wholesale from the snapshot', async () => { const id = await brain.add({ type: NounType.Concept, data: { name: 'Restore Test' }, metadata: { original: true } }) - await brain.commit({ message: 'before migration', author: 'test' }) const migration: Migration = { id: 'test-restore', @@ -313,29 +298,39 @@ describe('Migration System', () => { } await withMigrations([migration], async () => { - const result = await brain.migrate() + const result = await brain.migrate({ backupTo: backupDir }) const r = result as any + expect(r.backupPath).toBe(backupDir) // Verify migration was applied const migrated = await brain.get(id) expect(migrated?.metadata?.migrated).toBe(true) + expect(migrated?.metadata?.original).toBe(false) - // Verify backup branch exists as a named restore point - expect(r.backupBranch).toBe('pre-migration-3.0.0') - const branches = await brain.listBranches() - expect(branches).toContain('pre-migration-3.0.0') + // Restore the snapshot — the store returns to its pre-migration state + await brain.restore(backupDir, { confirm: true }) + const restored = await brain.get(id) + expect(restored?.metadata?.original).toBe(true) + expect(restored?.metadata?.migrated).toBeUndefined() + }) + }) - // Verify the backup ref points to the pre-migration commit - const refManager = (brain as any).storage.refManager - if (refManager) { - const mainRef = await refManager.getRef('main') - const backupRef = await refManager.getRef('pre-migration-3.0.0') - expect(backupRef).toBeDefined() - // Backup was forked before migration, so both share the same commit - // (the pre-migration commit). After migration, main's overlay has new data, - // but the commit pointer is unchanged. - expect(backupRef.commitHash).toBe(mainRef.commitHash) - } + it('should report backupPath null when no backupTo is supplied', async () => { + await brain.add({ type: NounType.Concept, data: { name: 'NoBackup' }, metadata: { q: 1 } }) + + const migration: Migration = { + id: 'test-no-backup', + version: '4.0.0', + description: 'Add field', + applies: 'nouns', + transform: (m) => 'q' in m && !('r' in m) ? { ...m, r: 2 } : null + } + + await withMigrations([migration], async () => { + const result = await brain.migrate() + const r = result as any + expect(r.backupPath).toBeNull() + expect(r.migrationsApplied).toContain('test-no-backup') }) }) }) @@ -520,86 +515,6 @@ describe('Migration System', () => { }) }) - // ─── Multi-branch migration ───────────────────────────────── - - describe('Multi-branch migration', () => { - it('should migrate entities on all branches including feature branches', async () => { - // Setup: create entities on main, fork a branch, add entities on the branch - const mainId = await brain.add({ - type: NounType.Concept, - data: { name: 'Main Entity' }, - metadata: { legacy: true, source: 'main' } - }) - await brain.commit({ message: 'initial', author: 'test' }) - - // Fork a feature branch and add branch-local entity - const fork = await brain.fork('feature-x') - const branchId = await fork.add({ - type: NounType.Concept, - data: { name: 'Branch Entity' }, - metadata: { legacy: true, source: 'branch' } - }) - - // Define migration that transforms the 'legacy' field - const migration: Migration = { - id: 'test-multi-branch', - version: '1.0.0', - description: 'Mark legacy entities as upgraded', - applies: 'nouns', - transform: (m) => { - if (m.legacy === true) { - return { ...m, legacy: false, upgraded: true } - } - return null // Already migrated or not applicable - } - } - - await withMigrations([migration], async () => { - // Migrate from main — should reach all branches - const result = await brain.migrate() - const r = result as any - - expect(r.migrationsApplied).toContain('test-multi-branch') - // Main entity + branch-local entity should both be modified - expect(r.entitiesModified).toBeGreaterThanOrEqual(2) - - // Verify main entity was transformed - const mainEntity = await brain.get(mainId) - expect(mainEntity?.metadata?.upgraded).toBe(true) - expect(mainEntity?.metadata?.legacy).toBe(false) - }) - - await fork.close() - }) - - it('should skip system:backup branches during multi-branch migration', async () => { - await brain.add({ type: NounType.Concept, data: { name: 'Test' }, metadata: { v: 1 } }) - await brain.commit({ message: 'test', author: 'test' }) - - // Create two migrations — first creates a backup branch, second should skip it - const migration1: Migration = { - id: 'test-skip-backup-1', - version: '5.0.0', - description: 'First migration', - applies: 'nouns', - transform: (m) => typeof m.v === 'number' && !('m1' in m) ? { ...m, m1: true } : null - } - - await withMigrations([migration1], async () => { - const result = await brain.migrate() - const r = result as any - expect(r.backupBranch).toBe('pre-migration-5.0.0') - - // Verify backup branch exists - const branches = await brain.listBranches() - expect(branches).toContain('pre-migration-5.0.0') - - // Migration should not have errored from trying to migrate the backup - expect(r.migrationsApplied).toContain('test-skip-backup-1') - }) - }) - }) - // ─── MigrationRunner unit-level tests ───────────────────────── describe('MigrationRunner', () => { @@ -684,47 +599,6 @@ describe('Migration System', () => { }) }) - it('should propagate errors from branch migrations into the result', async () => { - // Create entity on main, commit, fork a branch, add a branch-local entity that will fail - await brain.add({ type: NounType.Concept, data: { name: 'MainOk' }, metadata: { branchTest: 1 } }) - await brain.commit({ message: 'initial', author: 'test' }) - - const fork = await brain.fork('branch-with-error') - await fork.add({ type: NounType.Concept, data: { name: 'BranchBad' }, metadata: { branchTest: 'crash' } }) - - const migration: Migration = { - id: 'test-branch-error-prop', - version: '1.0.0', - description: 'Throws on non-number branchTest', - applies: 'nouns', - transform: (m) => { - if ('branchTest' in m) { - if (typeof m.branchTest !== 'number') { - throw new Error('branchTest must be a number') - } - return { ...m, branchTest: (m.branchTest as number) * 10 } - } - return null - } - } - - await withMigrations([migration], async () => { - const result = await brain.migrate() - const r = result as any - - // Main entity should have migrated successfully - expect(r.migrationsApplied).toContain('test-branch-error-prop') - expect(r.entitiesModified).toBeGreaterThanOrEqual(1) - - // Branch error should appear in the combined result - expect(r.errors.length).toBeGreaterThanOrEqual(1) - const branchError = r.errors.find((e: any) => e.error.includes('branchTest must be a number')) - expect(branchError).toBeDefined() - }) - - await fork.close() - }) - it('should stop early when maxErrors is exceeded', async () => { // Add many entities that will all fail for (let i = 0; i < 5; i++) { diff --git a/tests/integration/phase3TypeFirstQuery.integration.test.ts b/tests/integration/phase3TypeFirstQuery.integration.test.ts deleted file mode 100644 index beefcc3e..00000000 --- a/tests/integration/phase3TypeFirstQuery.integration.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Phase 3 Integration Tests - Type-First Query Optimization - * - * End-to-end tests verifying Phase 3 works with the complete Brainy system - * Target: 8 tests covering real-world scenarios - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import { NounType } from '../../src/types/graphTypes.js' -import { TypeAwareHNSWIndex } from '../../src/hnsw/typeAwareHNSWIndex.js' - -describe('Phase 3: Type-First Query Optimization - Integration', () => { - let brainy: Brainy - - beforeEach(async () => { - // Initialize with memory storage and TypeAwareHNSWIndex - brainy = new Brainy({ requireSubtype: false, - name: 'phase3-test', - dimension: 384, - storage: { - type: 'memory' // Use memory for fast tests - }, - index: { - M: 16, - efConstruction: 200, - efSearch: 50 - }, - debug: false // Disable debug logging for tests - }) - - await brainy.initialize() - }) - - afterEach(async () => { - // Clean up - await brainy.close() - }) - - // ========== Basic Type Inference Tests (3 tests) ========== - - describe('Basic Type Inference', () => { - it('should automatically infer Person type from "engineer" query', async () => { - // Add test data - await brainy.add({ - type: NounType.Person, - data: { name: 'Alice', role: 'engineer' } - }) - - await brainy.add({ - type: NounType.Document, - data: { title: 'Engineering Guide' } - }) - - // Query with natural language - const results = await brainy.find('Find engineers') - - // Should find Person, not Document - expect(results.length).toBeGreaterThan(0) - - // Verify TypeAwareHNSWIndex is being used - expect((brainy as any).index).toBeInstanceOf(TypeAwareHNSWIndex) - }) - - it('should handle queries with explicit type override', async () => { - await brainy.add({ - type: NounType.Person, - data: { name: 'Bob', role: 'developer' } - }) - - await brainy.add({ - type: NounType.Document, - data: { title: 'Development Process' } - }) - - // Query with explicit type should override inference - const results = await brainy.find({ - query: 'development', - type: NounType.Document // Explicit override - }) - - // Should only find documents - expect(results.length).toBeGreaterThan(0) - expect(results.every(r => r.entity.noun === NounType.Document)).toBe(true) - }) - - it('should handle multi-type queries efficiently', async () => { - // Add diverse data - await brainy.add({ - type: NounType.Person, - data: { name: 'Charlie', company: 'TechCorp' } - }) - - await brainy.add({ - type: NounType.Organization, - data: { name: 'TechCorp', industry: 'Software' } - }) - - await brainy.add({ - type: NounType.Document, - data: { title: 'TechCorp Overview' } - }) - - // Query that should infer multiple types - const results = await brainy.find('people at TechCorp') - - expect(results.length).toBeGreaterThan(0) - - // Should find both Person and Organization - const types = results.map(r => r.entity.noun) - expect(types).toContain(NounType.Person) - }) - }) - - // ========== Performance Tests (2 tests) ========== - - describe('Performance Impact', () => { - it('should reduce query latency for type-specific queries', async () => { - // Add 100 diverse entities - for (let i = 0; i < 50; i++) { - await brainy.add({ - type: NounType.Person, - data: { name: `Person ${i}`, role: 'engineer' } - }) - } - - for (let i = 0; i < 50; i++) { - await brainy.add({ - type: NounType.Document, - data: { title: `Document ${i}` } - }) - } - - // Measure query with type inference - const start = Date.now() - const results = await brainy.find('Find engineers') - const elapsed = Date.now() - start - - expect(results.length).toBeGreaterThan(0) - expect(elapsed).toBeLessThan(1000) // Should be fast even with 100 entities - - // Verify results are correct type - const personResults = results.filter(r => r.entity.noun === NounType.Person) - expect(personResults.length).toBeGreaterThan(0) - }, 10000) - - it('should handle high-volume queries without degradation', async () => { - // Add test data - for (let i = 0; i < 20; i++) { - await brainy.add({ - type: NounType.Person, - data: { name: `Person ${i}` } - }) - } - - // Run 10 queries in sequence - const latencies: number[] = [] - - for (let i = 0; i < 10; i++) { - const start = Date.now() - await brainy.find('Find people') - const elapsed = Date.now() - start - latencies.push(elapsed) - } - - // Verify consistent performance (no degradation) - const avgLatency = latencies.reduce((a, b) => a + b, 0) / latencies.length - const maxLatency = Math.max(...latencies) - - expect(maxLatency).toBeLessThan(avgLatency * 2) // Max should not be > 2x avg - }, 15000) - }) - - // ========== Edge Cases (2 tests) ========== - - describe('Edge Cases', () => { - it('should handle queries with no matching types gracefully', async () => { - await brainy.add({ - type: NounType.Person, - data: { name: 'Dave' } - }) - - // Query that won't infer any specific type - const results = await brainy.find('random stuff xyz') - - // Should still work (fallback to all-types) - expect(Array.isArray(results)).toBe(true) - }) - - it('should handle empty query gracefully', async () => { - await brainy.add({ - type: NounType.Person, - data: { name: 'Eve' } - }) - - // Empty query should return results - const results = await brainy.find({ limit: 5 }) - - expect(Array.isArray(results)).toBe(true) - expect(results.length).toBeGreaterThan(0) - }) - }) - - // ========== Backward Compatibility (1 test) ========== - - describe('Backward Compatibility', () => { - it('should work with all existing query patterns', async () => { - await brainy.add({ - type: NounType.Person, - data: { name: 'Frank', age: 30 } - }) - - // Test various query patterns - const results1 = await brainy.find({ query: 'Frank' }) - expect(results1.length).toBeGreaterThan(0) - - const results2 = await brainy.find({ type: NounType.Person }) - expect(results2.length).toBeGreaterThan(0) - - const results3 = await brainy.find({ - where: { age: 30 } - }) - expect(results3.length).toBeGreaterThan(0) - - const results4 = await brainy.find('Find Frank') - expect(results4.length).toBeGreaterThan(0) - }) - }) -}) diff --git a/tests/integration/storage-batch-operations.test.ts b/tests/integration/storage-batch-operations.test.ts index edfc4e32..9972df1a 100644 --- a/tests/integration/storage-batch-operations.test.ts +++ b/tests/integration/storage-batch-operations.test.ts @@ -1,9 +1,8 @@ /** * Storage-Level Batch Operations Test Suite v5.12.0 * - * Comprehensive testing of new storage-level batch APIs: + * Comprehensive testing of storage-level batch APIs: * - storage.getNounMetadataBatch() - Batch metadata reads - * - storage.readBatchWithInheritance() - COW-aware batch reads * - storage.getVerbsBySourceBatch() - Batch relationship queries * - brain.batchGet() - High-level batch entity retrieval * - PathResolver.getChildren() - VFS batch operations @@ -11,10 +10,8 @@ * Coverage: * ✅ Type-aware storage compatibility * ✅ Sharding preservation - * ✅ COW (Copy-on-Write) integration - * ✅ fork() and branch isolation + * ✅ Write-cache coherence * ✅ Performance improvements (N+1 → batched) - * ✅ Cloud adapter native batch APIs */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' @@ -129,13 +126,13 @@ describe('Storage-Level Batch Operations v5.12.0', () => { }) describe('storage.getNounMetadataBatch() - Storage Layer', () => { - it('should batch fetch noun metadata with type caching', async () => { + it('should batch fetch noun metadata across types via ID-first paths', async () => { // Add entities of different types const id1 = await brain.add({ type: 'document', data: 'Doc' }) const id2 = await brain.add({ type: 'thing', data: 'Thing' }) const id3 = await brain.add({ type: 'person', data: 'Person' }) - // Access storage directly + // Access storage directly (8.0 layout: ID-first paths, no type lookup) const storage = brain.storage as any const results = await storage.getNounMetadataBatch([id1, id2, id3]) @@ -143,32 +140,18 @@ describe('Storage-Level Batch Operations v5.12.0', () => { expect(results.get(id1)?.noun).toBe('document') expect(results.get(id2)?.noun).toBe('thing') expect(results.get(id3)?.noun).toBe('person') - - // Type cache should be populated - expect(storage.nounTypeCache.has(id1)).toBe(true) - expect(storage.nounTypeCache.get(id1)).toBe('document') }) - it('should handle uncached IDs by trying multiple types', async () => { - // Add entity + it('should omit missing ids from the result map', async () => { const id = await brain.add({ type: 'document', data: 'Test' }) + const missing = '00000000-0000-4000-8000-000000000000' - // Clear type cache to simulate uncached scenario const storage = brain.storage as any - storage.nounTypeCache.delete(id) - - // Batch fetch should still work (tries all types) - const results = await storage.getNounMetadataBatch([id]) + const results = await storage.getNounMetadataBatch([id, missing]) expect(results.size).toBe(1) expect(results.get(id)?.noun).toBe('document') - - // Cache should be repopulated (or may still be empty if metadata doesn't populate it) - // This is acceptable as long as the data is retrieved correctly - const cachedType = storage.nounTypeCache.get(id) - if (cachedType !== undefined) { - expect(cachedType).toBe('document') - } + expect(results.has(missing)).toBe(false) }) it('should preserve sharding in all paths', async () => { @@ -212,32 +195,7 @@ describe('Storage-Level Batch Operations v5.12.0', () => { }) }) - describe('COW Integration - readBatchWithInheritance()', () => { - it('should resolve branch paths before reading', async () => { - // Add entity on main - const id = await brain.add({ type: 'document', data: 'Main branch' }) - - // Create fork - const fork = await brain.fork('test-branch') - - // Add entity on fork - const forkId = await fork.add({ type: 'document', data: 'Fork branch' }) - - // Batch get on fork should see fork entity - const forkResults = await fork.batchGet([forkId, id]) - - expect(forkResults.size).toBe(2) - expect(forkResults.get(forkId)?.data).toBe('Fork branch') - expect(forkResults.get(id)?.data).toBe('Main branch') // Inherited - - // Batch get on main should NOT see fork entity - const mainResults = await brain.batchGet([forkId, id]) - - expect(mainResults.size).toBe(1) - expect(mainResults.has(forkId)).toBe(false) // Not on main - expect(mainResults.get(id)?.data).toBe('Main branch') - }) - + describe('Write-cache coherence', () => { it('should respect write cache for dirty entities', async () => { // Add entity const id = await brain.add({ type: 'document', data: 'Original' }) @@ -250,29 +208,6 @@ describe('Storage-Level Batch Operations v5.12.0', () => { expect(results.get(id)?.data).toBe('Updated') }) - - it('should inherit from parent commits for missing entities', async () => { - // Add entities on main - const id1 = await brain.add({ type: 'document', data: 'Main 1' }) - const id2 = await brain.add({ type: 'document', data: 'Main 2' }) - - // Commit - await brain.commit('Initial entities') - - // Create fork - const fork = await brain.fork('child-branch') - - // Add new entity only on fork - const forkId = await fork.add({ type: 'document', data: 'Fork only' }) - - // Batch get on fork should inherit main entities - const results = await fork.batchGet([id1, id2, forkId]) - - expect(results.size).toBe(3) - expect(results.get(id1)?.data).toBe('Main 1') // Inherited - expect(results.get(id2)?.data).toBe('Main 2') // Inherited - expect(results.get(forkId)?.data).toBe('Fork only') // Fork's own - }) }) describe('getVerbsBySourceBatch() - Batch Relationship Queries', () => { diff --git a/tests/integration/typeAwareHNSW.integration.test.ts b/tests/integration/typeAwareHNSW.integration.test.ts deleted file mode 100644 index 83e4a191..00000000 --- a/tests/integration/typeAwareHNSW.integration.test.ts +++ /dev/null @@ -1,527 +0,0 @@ -/** - * TypeAwareHNSW Integration Tests - * - * End-to-end tests for Phase 2 Type-Aware HNSW implementation. - * Tests cover: - * - Brainy integration (add, find) - * - Storage integration (FileSystem, Memory) - * - Rebuild functionality - * - Cache behavior - * - Large datasets - * - Performance characteristics - * - * Total: 15 integration tests - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { TypeAwareHNSWIndex } from '../../src/hnsw/typeAwareHNSWIndex.js' -import type { NounType } from '../../src/types/graphTypes.js' -import { euclideanDistance } from '../../src/utils/index.js' -import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' -import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' -import { v4 as uuidv4 } from 'uuid' -import fs from 'fs' -import path from 'path' - -describe('TypeAwareHNSW Integration Tests', () => { - const TEST_DATA_DIR = path.join(process.cwd(), '.test-data-type-aware-hnsw') - - afterEach(() => { - // Clean up test data directory - if (fs.existsSync(TEST_DATA_DIR)) { - fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }) - } - }) - - // =================================================================== - // 1. STORAGE INTEGRATION - // =================================================================== - - describe('Storage Integration', () => { - it('should work with MemoryStorage', async () => { - const storage = new MemoryStorage() - const index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - - // Add entities with valid UUIDs - const personId = uuidv4() - const docId = uuidv4() - - await index.addItem( - { id: personId, vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: docId, vector: [0, 1, 0] }, - 'document' as NounType - ) - - // Search - const results = await index.search([1, 0, 0], 2) - - expect(results).toHaveLength(2) - expect(results[0][0]).toBe(personId) // Closest to [1,0,0] - }) - - it('should work with FileSystemStorage', async () => { - const storage = new FileSystemStorage(TEST_DATA_DIR) - await storage.init() - - const index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - - // Add entities with valid UUIDs - const personId = uuidv4() - const docId = uuidv4() - - await index.addItem( - { id: personId, vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: docId, vector: [0, 1, 0] }, - 'document' as NounType - ) - - // Search should work - const results = await index.search([1, 0, 0], 2) - expect(results).toHaveLength(2) - }) - }) - - // =================================================================== - // 2. REBUILD FUNCTIONALITY - // =================================================================== - - describe('Rebuild Functionality', () => { - it('should handle rebuild gracefully when no data exists', async () => { - const storage = new MemoryStorage() - await storage.init() - - const index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - - // Rebuild with no data - should not crash - await index.rebuild() - - expect(index.size()).toBe(0) - }) - - it('should skip rebuild when no storage adapter', async () => { - const index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage: null } - ) - - // Should not crash when storage is null - await index.rebuild() - - expect(index.size()).toBe(0) - }) - - it('should allow specifying types to rebuild', async () => { - const storage = new MemoryStorage() - await storage.init() - - const index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - - // Rebuild specific types (no data, but should not crash) - await index.rebuild({ types: ['person', 'document'] as NounType[] }) - - expect(index.size()).toBe(0) - }) - }) - - // =================================================================== - // 3. LARGE DATASET TESTS - // =================================================================== - - describe('Large Dataset Tests', () => { - it('should handle 1000 entities across 5 types', async () => { - const storage = new MemoryStorage() - const index = new TypeAwareHNSWIndex( - { M: 8, efConstruction: 100, efSearch: 50 }, - euclideanDistance, - { storage } - ) - - const types: NounType[] = [ - 'person', - 'document', - 'event', - 'organization', - 'location' - ] - - // Add 200 entities per type = 1000 total - for (const type of types) { - for (let i = 0; i < 200; i++) { - await index.addItem( - { - id: `${type}-${i}`, - vector: [ - Math.random(), - Math.random(), - Math.random(), - Math.random() - ] - }, - type - ) - } - } - - expect(index.size()).toBe(1000) - expect(index.getActiveTypes()).toHaveLength(5) - - // Verify each type has correct count - for (const type of types) { - expect(index.sizeForType(type)).toBe(200) - } - - // Verify search works - const results = await index.search([0.5, 0.5, 0.5, 0.5], 10) - expect(results).toHaveLength(10) - }, 30000) // 30s timeout for large dataset - - it('should handle unbalanced distribution (1 dominant type)', async () => { - const storage = new MemoryStorage() - const index = new TypeAwareHNSWIndex( - { M: 8, efConstruction: 100, efSearch: 50 }, - euclideanDistance, - { storage } - ) - - // Add 900 person entities (dominant type) - for (let i = 0; i < 900; i++) { - await index.addItem( - { - id: `person-${i}`, - vector: [Math.random(), Math.random(), Math.random()] - }, - 'person' as NounType - ) - } - - // Add 100 document entities - for (let i = 0; i < 100; i++) { - await index.addItem( - { - id: `doc-${i}`, - vector: [Math.random(), Math.random(), Math.random()] - }, - 'document' as NounType - ) - } - - expect(index.size()).toBe(1000) - expect(index.sizeForType('person' as NounType)).toBe(900) - expect(index.sizeForType('document' as NounType)).toBe(100) - - // Verify search works correctly for both types - const personResults = await index.search( - [0.5, 0.5, 0.5], - 10, - 'person' as NounType - ) - const docResults = await index.search( - [0.5, 0.5, 0.5], - 10, - 'document' as NounType - ) - - expect(personResults.length).toBeGreaterThanOrEqual(10) - expect(docResults.length).toBeGreaterThanOrEqual(10) - - // All person results should be from person type - personResults.forEach((result) => { - expect(result[0]).toMatch(/^person-/) - }) - }, 30000) - }) - - // =================================================================== - // 4. TYPE-SPECIFIC QUERIES - // =================================================================== - - describe('Type-Specific Queries', () => { - let index: TypeAwareHNSWIndex - - beforeEach(async () => { - const storage = new MemoryStorage() - index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - - // Add entities of different types - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'person-2', vector: [0.9, 0.1, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - await index.addItem( - { id: 'doc-2', vector: [0, 0.9, 0.1] }, - 'document' as NounType - ) - await index.addItem( - { id: 'event-1', vector: [0, 0, 1] }, - 'event' as NounType - ) - }) - - it('should search single type only (fast path)', async () => { - const results = await index.search([1, 0, 0], 2, 'person' as NounType) - - expect(results).toHaveLength(2) - expect(results[0][0]).toBe('person-1') - expect(results[1][0]).toBe('person-2') - - // Verify no document or event results - results.forEach((result) => { - expect(result[0]).toMatch(/^person-/) - }) - }) - - it('should search multiple types', async () => { - const results = await index.search( - [0.5, 0.5, 0], - 5, - ['person', 'document'] as NounType[] - ) - - expect(results).toHaveLength(4) // 2 person + 2 document - - const ids = results.map((r) => r[0]) - expect(ids).toContain('person-1') - expect(ids).toContain('person-2') - expect(ids).toContain('doc-1') - expect(ids).toContain('doc-2') - expect(ids).not.toContain('event-1') // Not searched - }) - - it('should fall back to all-types search when type unknown', async () => { - const results = await index.search([0, 0, 1], 5) - - expect(results).toHaveLength(5) - - const ids = results.map((r) => r[0]) - expect(ids).toContain('event-1') // Found in all-types search - }) - }) - - // =================================================================== - // 5. MEMORY ISOLATION - // =================================================================== - - describe('Memory Isolation', () => { - it('should maintain separate memory for each type', async () => { - const storage = new MemoryStorage() - const index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - - // Add entities of different types - for (let i = 0; i < 100; i++) { - await index.addItem( - { id: `person-${i}`, vector: [Math.random(), Math.random(), 0] }, - 'person' as NounType - ) - } - - for (let i = 0; i < 100; i++) { - await index.addItem( - { id: `doc-${i}`, vector: [0, Math.random(), Math.random()] }, - 'document' as NounType - ) - } - - // Get stats to verify memory isolation - const stats = index.getStats() - - expect(stats.typeCount).toBe(2) - expect(stats.typeStats.has('person' as NounType)).toBe(true) - expect(stats.typeStats.has('document' as NounType)).toBe(true) - - const personStats = stats.typeStats.get('person' as NounType)! - const docStats = stats.typeStats.get('document' as NounType)! - - expect(personStats.nodeCount).toBe(100) - expect(docStats.nodeCount).toBe(100) - - // Verify memory is tracked separately (may be 0 in MemoryStorage) - expect(personStats.memoryMB).toBeGreaterThanOrEqual(0) - expect(docStats.memoryMB).toBeGreaterThanOrEqual(0) - - // Clear one type and verify other is unaffected - index.clearType('person' as NounType) - - expect(index.sizeForType('person' as NounType)).toBe(0) - expect(index.sizeForType('document' as NounType)).toBe(100) - }) - }) - - // =================================================================== - // 6. CACHE BEHAVIOR - // =================================================================== - - describe('Cache Behavior', () => { - it('should use UnifiedCache across all type indexes', async () => { - const storage = new MemoryStorage() - const index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - - // Add entities to different types - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - - // Get cache stats for each type - const personStats = index.getStatsForType('person' as NounType) - const docStats = index.getStatsForType('document' as NounType) - - expect(personStats).not.toBeNull() - expect(docStats).not.toBeNull() - - // Verify cache stats are available - expect(personStats!.cacheStats).toBeDefined() - expect(docStats!.cacheStats).toBeDefined() - - // UnifiedCache is shared, so both should have cache stats - expect(personStats!.cacheStats.hnswCache).toBeDefined() - expect(docStats!.cacheStats.hnswCache).toBeDefined() - }) - }) - - // =================================================================== - // 7. PERFORMANCE CHARACTERISTICS - // =================================================================== - - describe('Performance Characteristics', () => { - it('should have faster single-type search than all-types search', async () => { - const storage = new MemoryStorage() - const index = new TypeAwareHNSWIndex( - { M: 8, efConstruction: 100, efSearch: 50 }, - euclideanDistance, - { storage } - ) - - // Add 500 entities across 5 types - const types: NounType[] = [ - 'person', - 'document', - 'event', - 'organization', - 'location' - ] - - for (const type of types) { - for (let i = 0; i < 100; i++) { - await index.addItem( - { - id: `${type}-${i}`, - vector: [Math.random(), Math.random(), Math.random()] - }, - type - ) - } - } - - // Measure single-type search time - const singleTypeStart = Date.now() - await index.search([0.5, 0.5, 0.5], 10, 'person' as NounType) - const singleTypeTime = Date.now() - singleTypeStart - - // Measure all-types search time - const allTypesStart = Date.now() - await index.search([0.5, 0.5, 0.5], 10) - const allTypesTime = Date.now() - allTypesStart - - // Single-type should be faster (but this is a loose check for small dataset) - // At billion scale, this difference would be 10x - expect(singleTypeTime).toBeLessThanOrEqual(allTypesTime * 2) - }, 15000) - - it('should demonstrate memory reduction', async () => { - const storage = new MemoryStorage() - const index = new TypeAwareHNSWIndex( - { M: 8, efConstruction: 100, efSearch: 50 }, - euclideanDistance, - { storage } - ) - - // Add 1000 entities across 10 types - const types: NounType[] = [ - 'person', - 'document', - 'event', - 'organization', - 'location', - 'product', - 'concept', - 'project', - 'task', - 'message' - ] - - for (const type of types) { - for (let i = 0; i < 100; i++) { - await index.addItem( - { - id: `${type}-${i}`, - vector: [Math.random(), Math.random(), Math.random(), Math.random()] - }, - type - ) - } - } - - const stats = index.getStats() - - expect(stats.totalNodes).toBe(1000) - expect(stats.typeCount).toBe(10) - - // Verify memory reduction is calculated - expect(stats.estimatedMonolithicMemoryMB).toBeGreaterThan(0) - expect(stats.totalMemoryMB).toBeGreaterThanOrEqual(0) - expect(stats.memoryReductionPercent).toBeGreaterThanOrEqual(0) - - // At this scale, reduction should be significant - expect(stats.totalMemoryMB).toBeLessThan( - stats.estimatedMonolithicMemoryMB - ) - }, 30000) - }) -}) diff --git a/tests/integration/versioning.test.ts b/tests/integration/versioning.test.ts deleted file mode 100644 index 980b3bb2..00000000 --- a/tests/integration/versioning.test.ts +++ /dev/null @@ -1,511 +0,0 @@ -/** - * Entity Versioning Integration Tests (v5.3.0, v6.3.0) - * - * Tests the complete versioning workflow: - * - Save versions - * - List versions - * - Restore versions - * - Compare versions - * - Prune versions - * - Branch isolation - * - Index pollution prevention - * - * v6.3.0: Pure key-value storage, no index pollution, restore() updates all indexes - */ - -import { describe, it, expect, beforeEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import type { EntityVersion } from '../../src/versioning/VersionManager.js' -import { randomUUID } from 'crypto' - -// Helper to generate valid UUIDs for tests -const uuid = () => randomUUID().replace(/-/g, '') - -describe('Entity Versioning (v5.3.0)', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy({ requireSubtype: false, - storage: { type: 'memory' }, - silent: true - }) - await brain.init() - }) - - describe('Core Versioning API', () => { - it('should save and retrieve versions', async () => { - const entityId = uuid() - - // Add initial entity - await brain.add({ - data: 'Alice', - id: entityId, - type: 'person', // v6.3.0: Use valid NounType - metadata: { - name: 'Alice', - email: 'alice@example.com' - } - }) - - // Save version 1 - const v1 = await brain.versions.save(entityId, { - tag: 'v1.0', - description: 'Initial version' - }) - - expect(v1.version).toBe(1) - expect(v1.entityId).toBe(entityId) - expect(v1.tag).toBe('v1.0') - expect(v1.contentHash).toBeDefined() - - // Update entity - await brain.update({ id: entityId, metadata: { name: 'Alice Smith' } }) - - // Save version 2 - const v2 = await brain.versions.save(entityId, { - tag: 'v2.0', - description: 'Updated name' - }) - - expect(v2.version).toBe(2) - expect(v2.entityId).toBe(entityId) - - // List versions - const versions = await brain.versions.list(entityId) - expect(versions).toHaveLength(2) - expect(versions[0].version).toBe(2) // Newest first - expect(versions[1].version).toBe(1) - }) - - it('should deduplicate identical content', async () => { - const entityId = uuid() - - await brain.add({ - data: 'Doc', - id: entityId, - type: 'document', - metadata: { - name: 'Doc', - content: 'Hello' - } - }) - - // Save version 1 - const v1 = await brain.versions.save(entityId, { tag: 'v1' }) - - // Save again without changes - const v2 = await brain.versions.save(entityId, { tag: 'v2' }) - - // Should return existing version (same content hash) - expect(v2.version).toBe(v1.version) - expect(v2.contentHash).toBe(v1.contentHash) - - // Only one version should exist - const versions = await brain.versions.list(entityId) - expect(versions).toHaveLength(1) - }) - - it('should restore to previous version', async () => { - const entityId = uuid() - - await brain.add({ - data: 'Config', - id: entityId, - type: 'thing', - metadata: { - name: 'Config', - settings: { theme: 'light' } - } - }) - - // Save v1 - await brain.versions.save(entityId, { tag: 'v1' }) - - // Update - await brain.update({ id: entityId, metadata: { settings: { theme: 'dark' } } }) - - // Save v2 - await brain.versions.save(entityId, { tag: 'v2' }) - - // Verify current state (getNounMetadata returns flat NounMetadata) - let current = await brain.getNounMetadata(entityId) - expect(current?.settings?.theme).toBe('dark') - - // Restore to v1 - await brain.versions.restore(entityId, 1) - - // Verify restored state - current = await brain.getNounMetadata(entityId) - expect(current?.settings?.theme).toBe('light') - }) - - it('should compare versions', async () => { - const entityId = uuid() - - await brain.add({ - data: 'Bob', - id: entityId, - type: 'person', // v6.3.0: Use valid NounType - metadata: { - name: 'Bob', - email: 'bob@example.com', - age: 30 - } - }) - - await brain.versions.save(entityId, { tag: 'v1' }) - - // Update - await brain.update({ - id: entityId, - metadata: { - name: 'Robert', - email: 'robert@example.com', - city: 'NYC' - } - }) - - await brain.versions.save(entityId, { tag: 'v2' }) - - // Compare versions - const diff = await brain.versions.compare(entityId, 1, 2) - - expect(diff.totalChanges).toBeGreaterThan(0) - expect(diff.modified.length).toBeGreaterThan(0) - expect(diff.added.length).toBeGreaterThan(0) - - // Check specific changes - const nameChange = diff.modified.find(c => c.path.includes('name')) - expect(nameChange).toBeDefined() - expect(nameChange?.oldValue).toBe('Bob') - expect(nameChange?.newValue).toBe('Robert') - - const cityAdd = diff.added.find(c => c.path.includes('city')) - expect(cityAdd).toBeDefined() - expect(cityAdd?.newValue).toBe('NYC') - }) - - it('should get version content without restoring', async () => { - const entityId = uuid() - - await brain.add({ - data: 'Note', - id: entityId, - type: 'document', - metadata: { - name: 'Note', - content: 'Version 1' - } - }) - - await brain.versions.save(entityId, { tag: 'v1' }) - - await brain.update({ id: entityId, metadata: { content: 'Version 2' } }) - await brain.versions.save(entityId, { tag: 'v2' }) - - // Get v1 content without restoring (returns flat NounMetadata) - const v1Content = await brain.versions.getContent(entityId, 1) - expect(v1Content.content).toBe('Version 1') - - // Current should still be v2 (getNounMetadata returns flat NounMetadata) - const current = await brain.getNounMetadata(entityId) - expect(current?.content).toBe('Version 2') - }) - - it('should prune old versions', async () => { - const entityId = uuid() - - await brain.add({ - data: 'Log', - id: entityId, - type: 'document', - metadata: { - name: 'Log' - } - }) - - // Create 10 versions - for (let i = 1; i <= 10; i++) { - await brain.update({ id: entityId, metadata: { content: `Entry ${i}` } }) - await brain.versions.save(entityId, { tag: `v${i}` }) - } - - // Verify all 10 exist - let versions = await brain.versions.list(entityId) - expect(versions.length).toBeGreaterThanOrEqual(10) - - // Prune to keep only 5 most recent - const result = await brain.versions.prune(entityId, { - keepRecent: 5, - keepTagged: false - }) - - expect(result.deleted).toBeGreaterThan(0) - expect(result.kept).toBe(5) - - // Verify only 5 remain - versions = await brain.versions.list(entityId) - expect(versions).toHaveLength(5) - }) - - it('should support version tags', async () => { - const entityId = uuid() - - await brain.add({ - data: 'App', - id: entityId, - type: 'thing', - metadata: { - name: 'App' - } - }) - - await brain.versions.save(entityId, { tag: 'alpha' }) - await brain.update({ id: entityId, metadata: { version: '0.2' } }) - await brain.versions.save(entityId, { tag: 'beta' }) - await brain.update({ id: entityId, metadata: { version: '1.0' } }) - await brain.versions.save(entityId, { tag: 'release' }) - - // Get by tag - const beta = await brain.versions.getVersionByTag(entityId, 'beta') - expect(beta).toBeDefined() - expect(beta?.tag).toBe('beta') - - // Restore by tag - await brain.versions.restore(entityId, 'beta') - const current = await brain.getNounMetadata(entityId) - expect(current?.version).toBe('0.2') - }) - - it('should support undo/revert', async () => { - const entityId = uuid() - - await brain.add({ - data: 'Data', - id: entityId, - type: 'thing', - metadata: { - name: 'Data', - value: 100 - } - }) - - // Save v1 (value=100) - await brain.versions.save(entityId, { tag: 'v1' }) - - // Update and save v2 (value=200) - await brain.update({ id: entityId, metadata: { value: 200 } }) - await brain.versions.save(entityId, { tag: 'v2' }) - - // undo() restores to SECOND-MOST-RECENT version and creates a snapshot - // Note: undo() internally calls restore() with createSnapshot: true - const undone = await brain.versions.undo(entityId) - expect(undone).not.toBeNull() - - // After undo, entity should have v1's value - const currentVal = await brain.getNounMetadata(entityId) - expect(currentVal?.value).toBe(100) // v1's value - - // Now we have: [before-undo, v2, v1] - undo created a snapshot - const versionsAfterUndo = await brain.versions.list(entityId) - expect(versionsAfterUndo.length).toBeGreaterThanOrEqual(2) - - // Update and save v3 (value=300) - await brain.update({ id: entityId, metadata: { value: 300 } }) - await brain.versions.save(entityId, { tag: 'v3' }) - - // revert() is alias for undo() - const reverted = await brain.versions.revert(entityId) - expect(reverted).not.toBeNull() - - // The entity should be restored to second-most-recent version - const revertedVal = await brain.getNounMetadata(entityId) - // After revert from v3, we go to the previous version - expect(revertedVal?.value).toBeDefined() - }) - }) - - describe('Branch Isolation', () => { - it('should isolate versions by branch', async () => { - const entityId = uuid() - - await brain.add({ - data: 'Test', - id: entityId, - type: 'thing', - metadata: { - name: 'Test' - } - }) - - // Save versions on main - await brain.versions.save(entityId, { tag: 'main-v1' }) - await brain.update({ id: entityId, metadata: { name: 'Main Update' } }) - await brain.versions.save(entityId, { tag: 'main-v2' }) - - // Main should have versions - const mainVersionsBefore = await brain.versions.list(entityId) - expect(mainVersionsBefore.length).toBeGreaterThanOrEqual(2) - // Main versions should have branch='main' - expect(mainVersionsBefore.every(v => v.branch === 'main')).toBe(true) - - // Switch to feature branch - // Note: fork() creates the branch and returns a NEW instance - // We need to checkout() to switch the current instance to the new branch - await brain.fork('feature') - await brain.checkout('feature') - - // Save version on feature with unique tag - await brain.update({ id: entityId, metadata: { name: 'Feature Update' } }) - await brain.versions.save(entityId, { tag: 'feature-only-v1' }) - - // Feature versions list - may include inherited versions from COW - const featureVersions = await brain.versions.list(entityId) - expect(featureVersions.length).toBeGreaterThan(0) - - // Feature should have our unique tag (this is the NEW version on feature) - const featureOnlyVersion = featureVersions.find(v => v.tag === 'feature-only-v1') - expect(featureOnlyVersion).toBeDefined() - - // The version we just saved on feature should have branch='feature' - expect(featureOnlyVersion!.branch).toBe('feature') - - // Switch back to main - await brain.checkout('main') - - // Main versions should not have feature's unique tag - const mainVersionsAfter = await brain.versions.list(entityId) - const featureTagOnMain = mainVersionsAfter.find(v => v.tag === 'feature-only-v1') - expect(featureTagOnMain).toBeUndefined() // Feature-only version not on main - - // Main should still have its original versions - expect(mainVersionsAfter.some(v => v.tag === 'main-v1')).toBe(true) - expect(mainVersionsAfter.some(v => v.tag === 'main-v2')).toBe(true) - }) - }) - - describe('Edge Cases', () => { - it('should handle non-existent entities gracefully', async () => { - const nonExistentId = uuid() - await expect( - brain.versions.save(nonExistentId, { tag: 'v1' }) - ).rejects.toThrow(`Entity ${nonExistentId} not found`) - }) - - it('should handle empty version history', async () => { - const entityId = uuid() - await brain.add({ data: 'New', id: entityId, type: 'thing', metadata: { name: 'New' } }) - - expect(await brain.versions.count(entityId)).toBe(0) - expect(await brain.versions.hasVersions(entityId)).toBe(false) - expect(await brain.versions.getLatest(entityId)).toBeNull() - }) - - it('should handle version not found', async () => { - const entityId = uuid() - await brain.add({ data: 'Test', id: entityId, type: 'thing', metadata: { name: 'Test' } }) - - await expect( - brain.versions.restore(entityId, 999) - ).rejects.toThrow('Version 999 not found') - }) - }) - - describe('Index Pollution Prevention (v6.3.0)', () => { - it('should NOT pollute find() results with versions', async () => { - const entityId = uuid() - - // Add entity - await brain.add({ - data: 'Test entity', - id: entityId, - type: 'document', - metadata: { name: 'Test', category: 'pollution-test' } - }) - - // Save multiple versions - await brain.versions.save(entityId, { tag: 'v1' }) - await brain.update({ id: entityId, metadata: { name: 'Updated' } }) - await brain.versions.save(entityId, { tag: 'v2' }) - await brain.update({ id: entityId, metadata: { name: 'Updated Again' } }) - await brain.versions.save(entityId, { tag: 'v3' }) - - // Verify 3 versions exist - const versions = await brain.versions.list(entityId) - expect(versions).toHaveLength(3) - - // find() should return ONLY the real entity, not version entries - const results = await brain.find({ where: { category: 'pollution-test' } }) - expect(results).toHaveLength(1) - expect(results[0].id).toBe(entityId) - - // Verify no version entities pollute the index - // (This was the bug - _isVersion entities appeared in find()) - const allDocs = await brain.find({ where: { type: 'document' }, limit: 100 }) - const versionEntities = allDocs.filter((e: any) => e.metadata?._isVersion) - expect(versionEntities).toHaveLength(0) - }) - - it('should keep current entity fully indexed after versioning', async () => { - const entityId = uuid() - - // Add entity - await brain.add({ - data: 'Searchable content about machine learning', - id: entityId, - type: 'document', - metadata: { name: 'ML Doc', topic: 'ai' } - }) - - // Save versions - await brain.versions.save(entityId, { tag: 'v1' }) - await brain.update({ id: entityId, metadata: { name: 'Updated ML Doc' } }) - await brain.versions.save(entityId, { tag: 'v2' }) - - // Entity should still be searchable by metadata - const byMetadata = await brain.find({ where: { topic: 'ai' } }) - expect(byMetadata).toHaveLength(1) - expect(byMetadata[0].id).toBe(entityId) - - // Entity should still be searchable by vector similarity - const bySimilarity = await brain.find({ query: 'machine learning', limit: 5 }) - const found = bySimilarity.find((e: any) => e.id === entityId) - expect(found).toBeDefined() - }) - - it('should update indexes when restoring a version', async () => { - const entityId = uuid() - - // Add entity with initial state - await brain.add({ - data: 'Initial content', - id: entityId, - type: 'document', - metadata: { name: 'Doc', status: 'draft' } - }) - await brain.versions.save(entityId, { tag: 'draft' }) - - // Update to published state - await brain.update({ id: entityId, metadata: { status: 'published' } }) - await brain.versions.save(entityId, { tag: 'published' }) - - // Verify current state - let published = await brain.find({ where: { status: 'published' } }) - expect(published).toHaveLength(1) - - let drafts = await brain.find({ where: { status: 'draft' } }) - expect(drafts).toHaveLength(0) - - // Restore to draft version - await brain.versions.restore(entityId, 'draft') - - // Indexes should update - now entity is draft again - published = await brain.find({ where: { status: 'published' } }) - expect(published).toHaveLength(0) - - drafts = await brain.find({ where: { status: 'draft' } }) - expect(drafts).toHaveLength(1) - expect(drafts[0].id).toBe(entityId) - }) - }) -}) diff --git a/tests/integration/vfs-historical-reads.test.ts b/tests/integration/vfs-historical-reads.test.ts deleted file mode 100644 index fdf285e6..00000000 --- a/tests/integration/vfs-historical-reads.test.ts +++ /dev/null @@ -1,431 +0,0 @@ -/** - * VFS Historical Reads Integration Test (v5.3.7) - * - * Tests commitId support for time-travel reads: - * - readFile(path, { commitId }) - * - readdir(path, { commitId }) - * - stat(path, { commitId }) - * - exists(path, { commitId }) - * - * This is the CRITICAL test for a consumer's time-travel feature. - */ - -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import * as fs from 'fs' -import * as path from 'path' - -describe('VFS Historical Reads (v5.3.7)', () => { - const testDir = path.join(process.cwd(), 'test-vfs-historical') - let brain: Brainy - - beforeAll(async () => { - // Clean up test directory - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }) - } - fs.mkdirSync(testDir, { recursive: true }) - - // Initialize Brainy with filesystem storage (required for COW) - brain = new Brainy({ requireSubtype: false, - storage: { - adapter: 'filesystem', - path: testDir - } - }) - await brain.init() - - // Initialize VFS - await brain.vfs.init() - }) - - afterAll(async () => { - // Clean up test directory - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }) - } - }) - - it('should read files from historical commits', async () => { - console.log('\n📁 Test 1: Historical file reading') - - // Commit 1: Empty workspace - await brain.commit({ - message: 'Initial empty state', - author: 'test@example.com' - }) - const history1 = await brain.getHistory({ limit: 1 }) - const emptyCommitId = history1[0].hash - console.log(` Empty commit: ${emptyCommitId.slice(0, 8)}`) - - // Create first file - await brain.vfs.writeFile('/test.md', 'Version 1') - - // Commit 2: First file added - await brain.commit({ - message: 'Added test.md', - author: 'test@example.com' - }) - const history2 = await brain.getHistory({ limit: 1 }) - const v1CommitId = history2[0].hash - console.log(` V1 commit: ${v1CommitId.slice(0, 8)}`) - - // Update file - await brain.vfs.writeFile('/test.md', 'Version 2') - - // Commit 3: File updated - await brain.commit({ - message: 'Updated test.md', - author: 'test@example.com' - }) - const history3 = await brain.getHistory({ limit: 1 }) - const v2CommitId = history3[0].hash - console.log(` V2 commit: ${v2CommitId.slice(0, 8)}`) - - // Test 1a: Read current file (should be V2) - const currentContent = await brain.vfs.readFile('/test.md') - expect(currentContent.toString()).toBe('Version 2') - console.log(` ✅ Current content: "${currentContent.toString()}"`) - - // Test 1b: Read file at V1 commit (should be V1) - const v1Content = await brain.vfs.readFile('/test.md', { commitId: v1CommitId }) - expect(v1Content.toString()).toBe('Version 1') - console.log(` ✅ V1 content: "${v1Content.toString()}"`) - - // Test 1c: Read file at empty commit (should fail - file didn't exist) - await expect( - brain.vfs.readFile('/test.md', { commitId: emptyCommitId }) - ).rejects.toThrow('File not found at commit') - console.log(` ✅ Empty commit correctly throws ENOENT`) - }) - - it('should list directory contents from historical commits', async () => { - console.log('\n📂 Test 2: Historical directory listing') - - // Get current commit (from previous test, has /test.md) - const history1 = await brain.getHistory({ limit: 1 }) - const beforeProjectCommit = history1[0].hash - - // Create project directory with files - await brain.vfs.mkdir('/project') - await brain.vfs.writeFile('/project/app.ts', 'console.log("v1")') - await brain.vfs.writeFile('/project/utils.ts', 'export const util = 1') - - // Commit: Project created - await brain.commit({ - message: 'Added project directory', - author: 'test@example.com' - }) - const history2 = await brain.getHistory({ limit: 1 }) - const projectCommit = history2[0].hash - console.log(` Project commit: ${projectCommit.slice(0, 8)}`) - - // Add more files - await brain.vfs.writeFile('/project/config.json', '{"version": 1}') - - // Commit: Config added - await brain.commit({ - message: 'Added config', - author: 'test@example.com' - }) - const history3 = await brain.getHistory({ limit: 1 }) - const configCommit = history3[0].hash - console.log(` Config commit: ${configCommit.slice(0, 8)}`) - - // Test 2a: Read current directory (should have 3 files) - const currentEntries = await brain.vfs.readdir('/project') - expect(currentEntries.length).toBe(3) - expect(currentEntries).toContain('app.ts') - expect(currentEntries).toContain('utils.ts') - expect(currentEntries).toContain('config.json') - console.log(` ✅ Current directory: ${currentEntries.length} files`) - - // Test 2b: Read directory at project commit (should have 2 files, no config) - const projectEntries = await brain.vfs.readdir('/project', { commitId: projectCommit }) - expect(projectEntries.length).toBe(2) - expect(projectEntries).toContain('app.ts') - expect(projectEntries).toContain('utils.ts') - expect(projectEntries).not.toContain('config.json') - console.log(` ✅ Project commit directory: ${projectEntries.length} files (no config)`) - - // Test 2c: Read directory at before-project commit (should fail - dir didn't exist) - await expect( - brain.vfs.readdir('/project', { commitId: beforeProjectCommit }) - ).rejects.toThrow('Directory not found at commit') - console.log(` ✅ Before project commit correctly throws ENOENT`) - - // Test 2d: Read root directory at different commits - const currentRoot = await brain.vfs.readdir('/') - const projectRoot = await brain.vfs.readdir('/', { commitId: projectCommit }) - - // Current root should have both /test.md and /project - expect(currentRoot.length).toBeGreaterThanOrEqual(2) - - // Project commit root should also have both - expect(projectRoot.length).toBeGreaterThanOrEqual(2) - - console.log(` ✅ Root directory listings work at different commits`) - }) - - it('should stat files from historical commits', async () => { - console.log('\n📊 Test 3: Historical file stats') - - // Create a file and get its initial stats - await brain.vfs.writeFile('/stats-test.txt', 'Initial content') - - // Commit: Initial file - await brain.commit({ - message: 'Added stats-test.txt', - author: 'test@example.com' - }) - const history1 = await brain.getHistory({ limit: 1 }) - const initialCommit = history1[0].hash - console.log(` Initial commit: ${initialCommit.slice(0, 8)}`) - - // Get initial stats - const initialStats = await brain.vfs.stat('/stats-test.txt', { commitId: initialCommit }) - expect(initialStats.size).toBe('Initial content'.length) - expect(initialStats.isFile()).toBe(true) - console.log(` ✅ Initial stats: ${initialStats.size} bytes`) - - // Wait a moment to ensure different timestamp - await new Promise(resolve => setTimeout(resolve, 100)) - - // Update file with different content - await brain.vfs.writeFile('/stats-test.txt', 'Much longer updated content here') - - // Commit: Updated file - await brain.commit({ - message: 'Updated stats-test.txt', - author: 'test@example.com' - }) - const history2 = await brain.getHistory({ limit: 1 }) - const updatedCommit = history2[0].hash - console.log(` Updated commit: ${updatedCommit.slice(0, 8)}`) - - // Test 3a: Current stats (should reflect new size) - const currentStats = await brain.vfs.stat('/stats-test.txt') - expect(currentStats.size).toBe('Much longer updated content here'.length) - console.log(` ✅ Current stats: ${currentStats.size} bytes`) - - // Test 3b: Historical stats (should reflect old size) - const historicalStats = await brain.vfs.stat('/stats-test.txt', { commitId: initialCommit }) - expect(historicalStats.size).toBe('Initial content'.length) - expect(historicalStats.size).not.toBe(currentStats.size) - console.log(` ✅ Historical stats: ${historicalStats.size} bytes (different from current)`) - - // Test 3c: Stat directory at historical commit - const dirStats = await brain.vfs.stat('/', { commitId: initialCommit }) - expect(dirStats.isDirectory()).toBe(true) - expect(dirStats.isFile()).toBe(false) - console.log(` ✅ Directory stats work with commitId`) - }) - - it('should check existence from historical commits', async () => { - console.log('\n✅ Test 4: Historical existence checks') - - // Get current commit - const history1 = await brain.getHistory({ limit: 1 }) - const beforeNewFileCommit = history1[0].hash - - // Create a new file - await brain.vfs.writeFile('/new-file.txt', 'New file content') - - // Commit: New file added - await brain.commit({ - message: 'Added new-file.txt', - author: 'test@example.com' - }) - const history2 = await brain.getHistory({ limit: 1 }) - const afterNewFileCommit = history2[0].hash - console.log(` Before new file: ${beforeNewFileCommit.slice(0, 8)}`) - console.log(` After new file: ${afterNewFileCommit.slice(0, 8)}`) - - // Test 4a: File exists in current state - const existsNow = await brain.vfs.exists('/new-file.txt') - expect(existsNow).toBe(true) - console.log(` ✅ File exists in current state`) - - // Test 4b: File exists at after-commit - const existsAfter = await brain.vfs.exists('/new-file.txt', { commitId: afterNewFileCommit }) - expect(existsAfter).toBe(true) - console.log(` ✅ File exists at after-commit`) - - // Test 4c: File does NOT exist at before-commit - const existsBefore = await brain.vfs.exists('/new-file.txt', { commitId: beforeNewFileCommit }) - expect(existsBefore).toBe(false) - console.log(` ✅ File correctly doesn't exist at before-commit`) - - // Test 4d: Non-existent file returns false at any commit - const neverExists = await brain.vfs.exists('/never-created.txt', { commitId: afterNewFileCommit }) - expect(neverExists).toBe(false) - console.log(` ✅ Non-existent file returns false at any commit`) - - // Test 4e: Directory existence checks - const rootExists = await brain.vfs.exists('/', { commitId: beforeNewFileCommit }) - expect(rootExists).toBe(true) - console.log(` ✅ Root directory exists at all commits`) - }) - - it('should handle readdir with withFileTypes at historical commits', async () => { - console.log('\n📋 Test 5: Historical readdir with withFileTypes') - - // Create mixed content - await brain.vfs.mkdir('/mixed') - await brain.vfs.writeFile('/mixed/file1.txt', 'File 1') - await brain.vfs.mkdir('/mixed/subdir') - await brain.vfs.writeFile('/mixed/file2.txt', 'File 2') - - // Commit: Mixed content - await brain.commit({ - message: 'Added mixed directory', - author: 'test@example.com' - }) - const history = await brain.getHistory({ limit: 1 }) - const mixedCommit = history[0].hash - console.log(` Mixed commit: ${mixedCommit.slice(0, 8)}`) - - // Test 5a: readdir with withFileTypes at historical commit - const entries = await brain.vfs.readdir('/mixed', { - withFileTypes: true, - commitId: mixedCommit - }) - - expect(entries.length).toBe(3) - - // Check that we got VFSDirent objects - const fileEntries = entries.filter((e: any) => e.type === 'file') - const dirEntries = entries.filter((e: any) => e.type === 'directory') - - expect(fileEntries.length).toBe(2) - expect(dirEntries.length).toBe(1) - - console.log(` ✅ withFileTypes: ${fileEntries.length} files, ${dirEntries.length} dirs`) - - // Verify entries have correct properties - const firstEntry = entries[0] as any - expect(firstEntry).toHaveProperty('name') - expect(firstEntry).toHaveProperty('path') - expect(firstEntry).toHaveProperty('type') - expect(firstEntry).toHaveProperty('entityId') - console.log(` ✅ VFSDirent objects have all required properties`) - }) - - it('should handle errors gracefully for invalid commits', async () => { - console.log('\n⚠️ Test 6: Error handling for invalid commits') - - const fakeCommitId = 'f'.repeat(64) // Invalid commit hash - - // Test 6a: readFile with invalid commit - await expect( - brain.vfs.readFile('/test.md', { commitId: fakeCommitId }) - ).rejects.toThrow('Invalid commit ID') - console.log(` ✅ readFile throws on invalid commit`) - - // Test 6b: readdir with invalid commit - await expect( - brain.vfs.readdir('/', { commitId: fakeCommitId }) - ).rejects.toThrow('Invalid commit ID') - console.log(` ✅ readdir throws on invalid commit`) - - // Test 6c: stat with invalid commit - await expect( - brain.vfs.stat('/test.md', { commitId: fakeCommitId }) - ).rejects.toThrow('Invalid commit ID') - console.log(` ✅ stat throws on invalid commit`) - - // Test 6d: exists with invalid commit (returns false, doesn't throw) - const exists = await brain.vfs.exists('/test.md', { commitId: fakeCommitId }) - expect(exists).toBe(false) - console.log(` ✅ exists returns false on invalid commit (doesn't throw)`) - }) - - it('should maintain performance at scale', async () => { - console.log('\n⚡ Test 7: Performance with multiple files') - - // Create multiple files - const fileCount = 50 // Reasonable for integration test - for (let i = 0; i < fileCount; i++) { - await brain.vfs.writeFile(`/perf-test-${i}.txt`, `File ${i} content`) - } - - // Commit: Many files - await brain.commit({ - message: 'Added 50 files for performance test', - author: 'test@example.com' - }) - const history = await brain.getHistory({ limit: 1 }) - const perfCommit = history[0].hash - console.log(` Performance commit: ${perfCommit.slice(0, 8)}`) - - // Test 7a: Historical readdir performance - const startReaddir = Date.now() - const entries = await brain.vfs.readdir('/', { commitId: perfCommit }) - const readdirTime = Date.now() - startReaddir - - expect(entries.length).toBeGreaterThanOrEqual(fileCount) - console.log(` ✅ readdir (${entries.length} entries): ${readdirTime}ms`) - expect(readdirTime).toBeLessThan(5000) // Should complete in < 5s - - // Test 7b: Historical readFile performance - const startReadFile = Date.now() - const content = await brain.vfs.readFile('/perf-test-25.txt', { commitId: perfCommit }) - const readFileTime = Date.now() - startReadFile - - expect(content.toString()).toBe('File 25 content') - console.log(` ✅ readFile: ${readFileTime}ms`) - expect(readFileTime).toBeLessThan(1000) // Should complete in < 1s - - // Test 7c: Historical exists performance - const startExists = Date.now() - const exists = await brain.vfs.exists('/perf-test-25.txt', { commitId: perfCommit }) - const existsTime = Date.now() - startExists - - expect(exists).toBe(true) - console.log(` ✅ exists: ${existsTime}ms`) - expect(existsTime).toBeLessThan(1000) // Should complete in < 1s - }) - - it('should work with nested directory structures', async () => { - console.log('\n🌲 Test 8: Nested directories at historical commits') - - // Create nested structure - await brain.vfs.mkdir('/deep') - await brain.vfs.mkdir('/deep/level1') - await brain.vfs.mkdir('/deep/level1/level2') - await brain.vfs.writeFile('/deep/level1/level2/deep-file.txt', 'Deep content') - - // Commit: Deep structure - await brain.commit({ - message: 'Added deep directory structure', - author: 'test@example.com' - }) - const history1 = await brain.getHistory({ limit: 1 }) - const deepCommit = history1[0].hash - console.log(` Deep structure commit: ${deepCommit.slice(0, 8)}`) - - // Test 8a: Read deep file - const content = await brain.vfs.readFile('/deep/level1/level2/deep-file.txt', { - commitId: deepCommit - }) - expect(content.toString()).toBe('Deep content') - console.log(` ✅ Read file from nested directory`) - - // Test 8b: List intermediate directory - const level1Entries = await brain.vfs.readdir('/deep/level1', { commitId: deepCommit }) - expect(level1Entries).toContain('level2') - console.log(` ✅ List intermediate directory`) - - // Test 8c: Stat deep directory - const dirStats = await brain.vfs.stat('/deep/level1/level2', { commitId: deepCommit }) - expect(dirStats.isDirectory()).toBe(true) - console.log(` ✅ Stat nested directory`) - - // Test 8d: Check existence of deep path - const exists = await brain.vfs.exists('/deep/level1/level2/deep-file.txt', { - commitId: deepCommit - }) - expect(exists).toBe(true) - console.log(` ✅ Check existence of deep path`) - }) -}) diff --git a/tests/integration/vfs-versioning.test.ts b/tests/integration/vfs-versioning.test.ts deleted file mode 100644 index 4ade037d..00000000 --- a/tests/integration/vfs-versioning.test.ts +++ /dev/null @@ -1,307 +0,0 @@ -/** - * VFS File Versioning Integration Tests (v6.3.2) - * - * Tests the fix for the VFS versioning bug where file content was stale. - * - * Bug: VFS files store content in BlobStorage, but versioning captured stale - * embedding text from entity.data instead of actual file content. - * - * Fix: VersionManager.save() now reads fresh content from BlobStorage for VFS files, - * and restore() writes content back to BlobStorage. - */ - -import { describe, it, expect, beforeAll, afterAll } from 'vitest' -import { Brainy } from '../../src/brainy.js' -import * as fs from 'fs' -import * as path from 'path' - -describe('VFS File Versioning (v6.3.2 Fix)', () => { - const testDir = path.join(process.cwd(), 'test-vfs-versioning-fix') - let brain: Brainy - - beforeAll(async () => { - // Clean up from any previous run - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }) - } - fs.mkdirSync(testDir, { recursive: true }) - - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - options: { path: testDir } - }, - silent: true - }) - await brain.init() - }) - - afterAll(() => { - // Clean up test directory - if (fs.existsSync(testDir)) { - fs.rmSync(testDir, { recursive: true, force: true }) - } - }) - - describe('Core VFS Versioning', () => { - it('should version actual file content, not stale embedding text', async () => { - // Create a text file - await brain.vfs.writeFile('/test/file.md', 'Initial content - version 1') - - // Get the file entity - const stat = await brain.vfs.stat('/test/file.md') - const entityId = stat.entityId - - // Save version 1 - await brain.versions.save(entityId, { description: 'Version 1' }) - - // Modify the file with different content - await brain.vfs.writeFile('/test/file.md', 'Modified content - version 2 with more text') - - // Save version 2 - await brain.versions.save(entityId, { description: 'Version 2' }) - - // Retrieve both versions - const v1Content = await brain.versions.getContent(entityId, 1) - const v2Content = await brain.versions.getContent(entityId, 2) - - // CRITICAL TEST: The data should be DIFFERENT between versions - // This was the bug - both had the same stale content - expect(v1Content.data).not.toBe(v2Content.data) - - // Verify actual content - expect(v1Content.data).toBe('Initial content - version 1') - expect(v2Content.data).toBe('Modified content - version 2 with more text') - }) - - it('should restore VFS file content correctly', async () => { - // Create initial file - await brain.vfs.writeFile('/docs/readme.md', 'README version 1') - - const stat = await brain.vfs.stat('/docs/readme.md') - const entityId = stat.entityId - - // Save version 1 - await brain.versions.save(entityId, { tag: 'v1' }) - - // Modify file - await brain.vfs.writeFile('/docs/readme.md', 'README version 2 - updated') - - // Save version 2 - await brain.versions.save(entityId, { tag: 'v2' }) - - // Verify current content is v2 - let currentContent = await brain.vfs.readFile('/docs/readme.md') - expect(currentContent.toString()).toBe('README version 2 - updated') - - // Restore to v1 - await brain.versions.restore(entityId, 1) - - // Verify content is now v1 - currentContent = await brain.vfs.readFile('/docs/readme.md') - expect(currentContent.toString()).toBe('README version 1') - }) - - it('should handle multiple file edits and versions', async () => { - await brain.vfs.writeFile('/project/config.json', '{"version": 1}') - const stat = await brain.vfs.stat('/project/config.json') - const entityId = stat.entityId - - // Create 5 versions - for (let i = 1; i <= 5; i++) { - await brain.vfs.writeFile('/project/config.json', `{"version": ${i}}`) - await brain.versions.save(entityId, { tag: `v${i}` }) - } - - // Verify all 5 versions exist - const versions = await brain.versions.list(entityId) - expect(versions).toHaveLength(5) - - // Verify each version has correct content - for (let i = 1; i <= 5; i++) { - const content = await brain.versions.getContent(entityId, i) - expect(content.data).toBe(`{"version": ${i}}`) - } - }) - - it('should restore to any version in history', async () => { - await brain.vfs.writeFile('/notes/todo.txt', 'Task 1') - const stat = await brain.vfs.stat('/notes/todo.txt') - const entityId = stat.entityId - - await brain.versions.save(entityId, { tag: 'initial' }) - - await brain.vfs.writeFile('/notes/todo.txt', 'Task 1\nTask 2') - await brain.versions.save(entityId, { tag: 'two-tasks' }) - - await brain.vfs.writeFile('/notes/todo.txt', 'Task 1\nTask 2\nTask 3') - await brain.versions.save(entityId, { tag: 'three-tasks' }) - - // Restore to middle version - await brain.versions.restore(entityId, 'two-tasks') - let content = await brain.vfs.readFile('/notes/todo.txt') - expect(content.toString()).toBe('Task 1\nTask 2') - - // Restore to initial version - await brain.versions.restore(entityId, 'initial') - content = await brain.vfs.readFile('/notes/todo.txt') - expect(content.toString()).toBe('Task 1') - - // Restore to latest version - await brain.versions.restore(entityId, 3) - content = await brain.vfs.readFile('/notes/todo.txt') - expect(content.toString()).toBe('Task 1\nTask 2\nTask 3') - }) - }) - - describe('Binary File Versioning', () => { - it('should version binary files with base64 encoding', async () => { - // Create a simple binary file (simulated image header) - const binaryContent1 = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x01]) - const binaryContent2 = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x02]) - - await brain.vfs.writeFile('/images/test.png', binaryContent1) - const stat = await brain.vfs.stat('/images/test.png') - const entityId = stat.entityId - - // Save version 1 - await brain.versions.save(entityId, { tag: 'v1' }) - - // Modify binary content - await brain.vfs.writeFile('/images/test.png', binaryContent2) - - // Save version 2 - await brain.versions.save(entityId, { tag: 'v2' }) - - // Retrieve and verify both versions are different - const v1Content = await brain.versions.getContent(entityId, 1) - const v2Content = await brain.versions.getContent(entityId, 2) - - expect(v1Content.data).not.toBe(v2Content.data) - - // Verify binary content can be restored - await brain.versions.restore(entityId, 1) - const restored = await brain.vfs.readFile('/images/test.png') - expect(Buffer.compare(restored, binaryContent1)).toBe(0) - }) - }) - - describe('Version Deduplication', () => { - it('should deduplicate identical VFS file content', async () => { - await brain.vfs.writeFile('/cache/data.txt', 'Cached data') - const stat = await brain.vfs.stat('/cache/data.txt') - const entityId = stat.entityId - - // Save version 1 - const v1 = await brain.versions.save(entityId, { tag: 'v1' }) - - // Save again without changes - should dedupe - const v2 = await brain.versions.save(entityId, { tag: 'v2' }) - - // Should return same version (content hash match) - expect(v2.version).toBe(v1.version) - expect(v2.contentHash).toBe(v1.contentHash) - - // Only 1 version should exist - const versions = await brain.versions.list(entityId) - expect(versions).toHaveLength(1) - }) - }) - - describe('Mixed Entity Versioning', () => { - it('should handle VFS and non-VFS entities in same brain', async () => { - // Create a VFS file - await brain.vfs.writeFile('/files/doc.txt', 'File content v1') - const fileStat = await brain.vfs.stat('/files/doc.txt') - const fileId = fileStat.entityId - - // Create a regular entity - const regularId = await brain.add({ - data: 'Regular entity', - type: 'document', - metadata: { title: 'Regular Doc' } - }) - - // Version both - await brain.versions.save(fileId, { tag: 'file-v1' }) - await brain.versions.save(regularId, { tag: 'regular-v1' }) - - // Update both - await brain.vfs.writeFile('/files/doc.txt', 'File content v2') - await brain.update({ id: regularId, metadata: { title: 'Updated Regular Doc' } }) - - // Version both again - await brain.versions.save(fileId, { tag: 'file-v2' }) - await brain.versions.save(regularId, { tag: 'regular-v2' }) - - // Verify VFS file versions have different content - const fileV1 = await brain.versions.getContent(fileId, 1) - const fileV2 = await brain.versions.getContent(fileId, 2) - expect(fileV1.data).toBe('File content v1') - expect(fileV2.data).toBe('File content v2') - - // Verify regular entity versions work too - const regV1 = await brain.versions.getContent(regularId, 1) - const regV2 = await brain.versions.getContent(regularId, 2) - expect(regV1.title).toBe('Regular Doc') - expect(regV2.title).toBe('Updated Regular Doc') - }) - }) - - describe('Edge Cases', () => { - it('should handle minimal content files', async () => { - // VFS requires non-empty data for embedding, so we use minimal content - await brain.vfs.writeFile('/minimal/file.txt', ' ') // Single space - const stat = await brain.vfs.stat('/minimal/file.txt') - const entityId = stat.entityId - - await brain.versions.save(entityId, { tag: 'minimal' }) - - await brain.vfs.writeFile('/minimal/file.txt', 'Now has content') - await brain.versions.save(entityId, { tag: 'filled' }) - - // Restore to minimal - await brain.versions.restore(entityId, 'minimal') - const content = await brain.vfs.readFile('/minimal/file.txt') - expect(content.toString()).toBe(' ') - }) - - it('should handle large text files', async () => { - const largeContent1 = 'A'.repeat(100000) + ' version 1' - const largeContent2 = 'B'.repeat(100000) + ' version 2' - - await brain.vfs.writeFile('/large/file.txt', largeContent1) - const stat = await brain.vfs.stat('/large/file.txt') - const entityId = stat.entityId - - await brain.versions.save(entityId, { tag: 'v1' }) - - await brain.vfs.writeFile('/large/file.txt', largeContent2) - await brain.versions.save(entityId, { tag: 'v2' }) - - // Verify versions are different - const v1 = await brain.versions.getContent(entityId, 1) - const v2 = await brain.versions.getContent(entityId, 2) - expect(v1.data).toBe(largeContent1) - expect(v2.data).toBe(largeContent2) - }) - - it('should handle special characters in content', async () => { - const specialContent = 'Unicode: \u{1F600} \u{1F914} \u{1F4A1}\nNewlines\nAnd\ttabs' - - await brain.vfs.writeFile('/special.txt', specialContent) - const stat = await brain.vfs.stat('/special.txt') - const entityId = stat.entityId - - await brain.versions.save(entityId, { tag: 'special' }) - - await brain.vfs.writeFile('/special.txt', 'Plain text now') - await brain.versions.save(entityId, { tag: 'plain' }) - - // Restore special content - await brain.versions.restore(entityId, 'special') - const content = await brain.vfs.readFile('/special.txt') - expect(content.toString()).toBe(specialContent) - }) - }) -}) diff --git a/tests/transaction/integration/cow-transactions.test.ts b/tests/transaction/integration/cow-transactions.test.ts deleted file mode 100644 index 49309a60..00000000 --- a/tests/transaction/integration/cow-transactions.test.ts +++ /dev/null @@ -1,265 +0,0 @@ -/** - * COW + Transactions Integration Tests - * - * Verifies that transactions work correctly with Copy-on-Write storage: - * - Branch isolation - * - Atomic commits - * - Rollback without affecting main branch - * - Content-addressable storage - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../../src/brainy.js' -import { NounType, VerbType } from '../../../src/types/graphTypes.js' -import { tmpdir } from 'os' -import { join } from 'path' -import { mkdirSync, rmSync } from 'fs' - -describe('Transactions + COW Integration', () => { - let brain: Brainy - let testDir: string - - beforeEach(async () => { - // Create unique test directory - testDir = join(tmpdir(), `brainy-cow-test-${Date.now()}`) - mkdirSync(testDir, { recursive: true }) - - // Initialize Brainy with COW enabled - brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - path: testDir - } - }) - - await brain.init() - - // Enable COW if available - if (brain.cow) { - await brain.cow.init() - } - }) - - afterEach(async () => { - if (brain) { - await brain.shutdown() - } - if (testDir) { - rmSync(testDir, { recursive: true, force: true }) - } - }) - - describe('Basic COW Operations', () => { - it('should commit transaction successfully on COW branch', async () => { - // Create branch - if (!brain.cow) { - console.log('COW not available, skipping test') - return - } - - await brain.cow.createBranch('feature-branch') - await brain.cow.checkout('feature-branch') - - // Add entity (should succeed) - const id = await brain.add({ - data: { name: 'Test Entity' }, - type: NounType.Thing - }) - - expect(id).toBeTruthy() - - // Verify entity exists - const entity = await brain.get(id) - expect(entity).toBeTruthy() - expect(entity?.data.name).toBe('Test Entity') - }) - - it('should isolate transaction rollback to branch', async () => { - if (!brain.cow) { - console.log('COW not available, skipping test') - return - } - - // Add entity on main branch - const mainId = await brain.add({ - data: { name: 'Main Entity' }, - type: NounType.Thing - }) - - // Create and checkout feature branch - await brain.cow.createBranch('feature-branch') - await brain.cow.checkout('feature-branch') - - // Try to add entity that will fail - try { - await brain.add({ - data: { name: 'Feature Entity' }, - type: NounType.Thing, - // Force failure by providing invalid vector - vector: [] as any - }) - } catch (e) { - // Expected to fail - } - - // Switch back to main - await brain.cow.checkout('main') - - // Main branch entity should still exist - const mainEntity = await brain.get(mainId) - expect(mainEntity).toBeTruthy() - expect(mainEntity?.data.name).toBe('Main Entity') - }) - - it('should handle atomic updates across COW branches', async () => { - if (!brain.cow) { - console.log('COW not available, skipping test') - return - } - - // Add entity - const id = await brain.add({ - data: { name: 'Original', version: 1 }, - type: NounType.Thing - }) - - // Create branch - await brain.cow.createBranch('feature-branch') - await brain.cow.checkout('feature-branch') - - // Update entity (atomic operation) - await brain.update({ - id, - data: { name: 'Updated', version: 2 }, - merge: false - }) - - // Verify update on feature branch - const featureEntity = await brain.get(id) - expect(featureEntity?.data.version).toBe(2) - - // Switch to main - await brain.cow.checkout('main') - - // Original should be unchanged on main - const mainEntity = await brain.get(id) - expect(mainEntity?.data.version).toBe(1) - }) - }) - - describe('Transaction Rollback with COW', () => { - it('should rollback failed transaction without affecting COW branch', async () => { - if (!brain.cow) { - console.log('COW not available, skipping test') - return - } - - await brain.cow.createBranch('test-branch') - await brain.cow.checkout('test-branch') - - // Add first entity (will succeed) - const id1 = await brain.add({ - data: { name: 'Entity 1' }, - type: NounType.Thing - }) - - // Attempt to add with invalid data (will fail) - // This tests that the transaction rollback doesn't corrupt the branch - let failed = false - try { - await brain.add({ - data: null as any, // Invalid - type: NounType.Thing - }) - } catch (e) { - failed = true - } - - expect(failed).toBe(true) - - // First entity should still exist (transaction rollback worked) - const entity1 = await brain.get(id1) - expect(entity1).toBeTruthy() - }) - }) - - describe('COW Branch Merging with Transactions', () => { - it('should preserve transaction atomicity during branch operations', async () => { - if (!brain.cow) { - console.log('COW not available, skipping test') - return - } - - // Add entity on main - const mainId = await brain.add({ - data: { name: 'Main Entity', branch: 'main' }, - type: NounType.Thing - }) - - // Create feature branch - await brain.cow.createBranch('feature') - await brain.cow.checkout('feature') - - // Add multiple entities in transaction (atomic) - const featureId1 = await brain.add({ - data: { name: 'Feature 1', branch: 'feature' }, - type: NounType.Thing - }) - - const featureId2 = await brain.add({ - data: { name: 'Feature 2', branch: 'feature' }, - type: NounType.Thing - }) - - // Create relationship (atomic) - await brain.relate({ - from: featureId1, - to: featureId2, - type: VerbType.RelatesTo - }) - - // All operations should be atomic on feature branch - const feature1 = await brain.get(featureId1) - const feature2 = await brain.get(featureId2) - const relations = await brain.getRelations({ from: featureId1 }) - - expect(feature1).toBeTruthy() - expect(feature2).toBeTruthy() - expect(relations).toHaveLength(1) - - // Switch back to main - feature entities should not exist - await brain.cow.checkout('main') - const mainCheck = await brain.get(featureId1) - expect(mainCheck).toBeNull() - }) - }) - - describe('Content-Addressable Storage', () => { - it('should handle content-addressable storage with transactions', async () => { - if (!brain.cow) { - console.log('COW not available, skipping test') - return - } - - // Add entity with specific data - const id = await brain.add({ - data: { content: 'Test content', value: 123 }, - type: NounType.Thing - }) - - // Update entity (creates new blob) - await brain.update({ - id, - data: { content: 'Updated content', value: 456 } - }) - - // Get entity - should have updated content - const entity = await brain.get(id) - expect(entity?.data.content).toBe('Updated content') - expect(entity?.data.value).toBe(456) - - // Transaction system should work with content-addressable storage - // (each version is a separate blob, rollback restores previous blob reference) - }) - }) -}) diff --git a/tests/typeAwareHNSWIndex.test.ts b/tests/typeAwareHNSWIndex.test.ts deleted file mode 100644 index 173d660e..00000000 --- a/tests/typeAwareHNSWIndex.test.ts +++ /dev/null @@ -1,467 +0,0 @@ -/** - * TypeAwareHNSWIndex Unit Tests - * - * Comprehensive test suite for Phase 2 Type-Aware HNSW implementation. - * Tests cover: - * - Lazy initialization - * - Type routing (single/multi/all types) - * - Edge cases (empty array, null, invalid type) - * - Error handling - * - Memory isolation - * - Statistics - * - Configuration - * - * Total: 25 unit tests - */ - -import { describe, it, expect, beforeEach } from 'vitest' -import { TypeAwareHNSWIndex } from '../src/hnsw/typeAwareHNSWIndex.js' -import type { NounType } from '../src/types/graphTypes.js' -import { euclideanDistance } from '../src/utils/index.js' -import { MemoryStorage } from '../src/storage/adapters/memoryStorage.js' - -describe('TypeAwareHNSWIndex', () => { - let index: TypeAwareHNSWIndex - let storage: MemoryStorage - - beforeEach(() => { - storage = new MemoryStorage() - index = new TypeAwareHNSWIndex( - { M: 4, efConstruction: 50, efSearch: 20 }, - euclideanDistance, - { storage } - ) - }) - - // =================================================================== - // 1. LAZY INITIALIZATION - // =================================================================== - - describe('Lazy Initialization', () => { - it('should not create indexes upfront', () => { - expect(index.getActiveTypes()).toHaveLength(0) - expect(index.size()).toBe(0) - }) - - it('should create index only when first entity added', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 2, 3] }, - 'person' as NounType - ) - - expect(index.getActiveTypes()).toContain('person') - expect(index.getActiveTypes()).toHaveLength(1) - expect(index.sizeForType('person' as NounType)).toBe(1) - }) - - it('should create separate indexes for different types', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 2, 3] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [4, 5, 6] }, - 'document' as NounType - ) - - expect(index.getActiveTypes()).toHaveLength(2) - expect(index.getActiveTypes()).toContain('person') - expect(index.getActiveTypes()).toContain('document') - expect(index.sizeForType('person' as NounType)).toBe(1) - expect(index.sizeForType('document' as NounType)).toBe(1) - }) - - it('should not create index for types with no entities', () => { - expect(index.sizeForType('event' as NounType)).toBe(0) - expect(index.getActiveTypes()).not.toContain('event') - }) - }) - - // =================================================================== - // 2. TYPE ROUTING - // =================================================================== - - describe('Type Routing', () => { - beforeEach(async () => { - // Add entities of different types - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'person-2', vector: [1, 0.1, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - await index.addItem( - { id: 'event-1', vector: [0, 0, 1] }, - 'event' as NounType - ) - }) - - it('should search single type only (fast path)', async () => { - const results = await index.search([1, 0, 0], 2, 'person' as NounType) - - expect(results).toHaveLength(2) - expect(results[0][0]).toBe('person-1') // Exact match - expect(results[1][0]).toBe('person-2') // Close match - }) - - it('should search multiple types', async () => { - const results = await index.search( - [1, 0, 0], - 3, - ['person', 'document'] as NounType[] - ) - - expect(results).toHaveLength(3) - const ids = results.map((r) => r[0]) - expect(ids).toContain('person-1') - expect(ids).toContain('person-2') - expect(ids).toContain('doc-1') - expect(ids).not.toContain('event-1') // Not searched - }) - - it('should search all types when type not specified', async () => { - const results = await index.search([1, 0, 0], 4) - - expect(results).toHaveLength(4) - const ids = results.map((r) => r[0]) - expect(ids).toContain('person-1') - expect(ids).toContain('person-2') - expect(ids).toContain('doc-1') - expect(ids).toContain('event-1') - }) - - it('should return results sorted by distance', async () => { - const results = await index.search([1, 0, 0], 4) - - // Distances should be increasing - for (let i = 0; i < results.length - 1; i++) { - expect(results[i][1]).toBeLessThanOrEqual(results[i + 1][1]) - } - }) - }) - - // =================================================================== - // 3. EDGE CASE HANDLING - // =================================================================== - - describe('Edge Cases', () => { - it('should handle empty array in search() (fall through to all types)', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - - const results = await index.search([1, 0, 0], 10, [] as NounType[]) - - // Should search all types (fallback behavior) - expect(results).toHaveLength(1) - expect(results[0][0]).toBe('person-1') - }) - - it('should throw on null item in addItem()', async () => { - await expect( - index.addItem(null as any, 'person' as NounType) - ).rejects.toThrow('Invalid VectorDocument: item or vector is null/undefined') - }) - - it('should throw on undefined vector in addItem()', async () => { - await expect( - index.addItem({ id: 'test' } as any, 'person' as NounType) - ).rejects.toThrow('Invalid VectorDocument: item or vector is null/undefined') - }) - - it('should throw on null type in addItem()', async () => { - await expect( - index.addItem({ id: 'test', vector: [1, 2, 3] }, null as any) - ).rejects.toThrow('Type is required for type-aware indexing') - }) - - it('should throw on invalid type string', async () => { - await expect( - index.addItem( - { id: 'test', vector: [1, 2, 3] }, - 'not-a-valid-noun-type-at-all' as any - ) - ).rejects.toThrow('Invalid NounType') - }) - - it('should handle search with no results', async () => { - const results = await index.search([1, 2, 3], 10, 'person' as NounType) - - expect(results).toHaveLength(0) - }) - - it('should handle removeItem() for non-existent type', async () => { - const removed = await index.removeItem('test-id', 'person' as NounType) - - expect(removed).toBe(false) - }) - }) - - // =================================================================== - // 4. ADD/REMOVE/SEARCH OPERATIONS - // =================================================================== - - describe('Operations', () => { - it('should add item and return ID', async () => { - const id = await index.addItem( - { id: 'person-1', vector: [1, 2, 3] }, - 'person' as NounType - ) - - expect(id).toBe('person-1') - expect(index.size()).toBe(1) - }) - - it('should remove item from correct type', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 2, 3] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [4, 5, 6] }, - 'document' as NounType - ) - - const removed = await index.removeItem('person-1', 'person' as NounType) - - expect(removed).toBe(true) - expect(index.sizeForType('person' as NounType)).toBe(0) - expect(index.sizeForType('document' as NounType)).toBe(1) // Unchanged - }) - - it('should search with filter function', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'person-2', vector: [1, 0.1, 0] }, - 'person' as NounType - ) - - const filter = async (id: string) => id === 'person-1' - const results = await index.search( - [1, 0, 0], - 2, - 'person' as NounType, - filter - ) - - expect(results).toHaveLength(1) - expect(results[0][0]).toBe('person-1') - }) - }) - - // =================================================================== - // 5. MEMORY ISOLATION - // =================================================================== - - describe('Memory Isolation', () => { - it('should maintain separate memory for each type', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - - // Clear person type only - index.clearType('person' as NounType) - - expect(index.sizeForType('person' as NounType)).toBe(0) - expect(index.sizeForType('document' as NounType)).toBe(1) // Unchanged - }) - - it('should clear all indexes', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - - index.clear() - - expect(index.size()).toBe(0) - expect(index.getActiveTypes()).toHaveLength(0) - }) - }) - - // =================================================================== - // 6. SIZE AND STATISTICS - // =================================================================== - - describe('Size and Statistics', () => { - it('should return total size across all types', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'person-2', vector: [1, 0.1, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - - expect(index.size()).toBe(3) - }) - - it('should return size for specific type', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'person-2', vector: [1, 0.1, 0] }, - 'person' as NounType - ) - - expect(index.sizeForType('person' as NounType)).toBe(2) - expect(index.sizeForType('document' as NounType)).toBe(0) - }) - - it('should return comprehensive statistics', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - - const stats = index.getStats() - - expect(stats.totalNodes).toBe(2) - expect(stats.typeCount).toBe(2) - expect(stats.typeStats.has('person' as NounType)).toBe(true) - expect(stats.typeStats.has('document' as NounType)).toBe(true) - expect(stats.memoryReductionPercent).toBeGreaterThan(0) - }) - - it('should return stats for specific type', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - - const stats = index.getStatsForType('person' as NounType) - - expect(stats).not.toBeNull() - expect(stats!.nodeCount).toBe(1) - expect(stats!.memoryMB).toBeGreaterThanOrEqual(0) - }) - - it('should return null stats for non-existent type', () => { - const stats = index.getStatsForType('person' as NounType) - - expect(stats).toBeNull() - }) - - it('should calculate memory reduction percentage', async () => { - // Add multiple entities to make calculation meaningful - for (let i = 0; i < 100; i++) { - await index.addItem( - { id: `person-${i}`, vector: [Math.random(), Math.random(), Math.random()] }, - 'person' as NounType - ) - } - - const stats = index.getStats() - - expect(stats.totalNodes).toBe(100) - expect(stats.estimatedMonolithicMemoryMB).toBeGreaterThan(0) - expect(stats.memoryReductionPercent).toBeGreaterThanOrEqual(0) - expect(stats.memoryReductionPercent).toBeLessThanOrEqual(100) - }) - - it('should handle stats with empty indexes', () => { - const stats = index.getStats() - - expect(stats.totalNodes).toBe(0) - expect(stats.typeCount).toBe(0) - expect(stats.totalMemoryMB).toBe(0) - expect(stats.memoryReductionPercent).toBe(0) - }) - }) - - // =================================================================== - // 7. CONFIGURATION - // =================================================================== - - describe('Configuration', () => { - it('should return HNSW configuration', () => { - const config = index.getConfig() - - expect(config.M).toBe(4) - expect(config.efConstruction).toBe(50) - expect(config.efSearch).toBe(20) - }) - - it('should return distance function', () => { - const distFn = index.getDistanceFunction() - - expect(distFn).toBe(euclideanDistance) - }) - - it('should get parallelization setting', () => { - const parallel = index.getUseParallelization() - - expect(parallel).toBe(true) // Default - }) - - it('should set parallelization for all indexes', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - - index.setUseParallelization(false) - - expect(index.getUseParallelization()).toBe(false) - }) - }) - - // =================================================================== - // 8. ACTIVE TYPES - // =================================================================== - - describe('Active Types', () => { - it('should return list of types with entities', async () => { - await index.addItem( - { id: 'person-1', vector: [1, 0, 0] }, - 'person' as NounType - ) - await index.addItem( - { id: 'doc-1', vector: [0, 1, 0] }, - 'document' as NounType - ) - - const activeTypes = index.getActiveTypes() - - expect(activeTypes).toHaveLength(2) - expect(activeTypes).toContain('person') - expect(activeTypes).toContain('document') - }) - - it('should return empty array when no types have entities', () => { - const activeTypes = index.getActiveTypes() - - expect(activeTypes).toHaveLength(0) - }) - }) -}) diff --git a/tests/unit/storage/cow/BlobStorage.compression-policy.test.ts b/tests/unit/storage/blobStorage.compression-policy.test.ts similarity index 88% rename from tests/unit/storage/cow/BlobStorage.compression-policy.test.ts rename to tests/unit/storage/blobStorage.compression-policy.test.ts index f8220fc9..346a50c5 100644 --- a/tests/unit/storage/cow/BlobStorage.compression-policy.test.ts +++ b/tests/unit/storage/blobStorage.compression-policy.test.ts @@ -2,7 +2,7 @@ * @module BlobStorage.compression-policy.test * @description 2.5.0 #32 — content-type-aware compression policy. * - * COW's BlobStorage now consults `BlobWriteOptions.mimeType` in `auto` mode + * BlobStorage consults `BlobWriteOptions.mimeType` in `auto` mode * and skips zstd for MIME types known to be already heavily compressed (JPEG, * PNG, MP4, MP3, ZIP, PDF, etc.). zstd over those formats is reliably a CPU * loss for no measurable byte savings. @@ -24,11 +24,11 @@ import { describe, it, expect, beforeEach } from 'vitest' import { BlobStorage, - COWStorageAdapter, + BlobStoreAdapter, isAlreadyCompressedMimeType -} from '../../../../src/storage/cow/BlobStorage.js' +} from '../../../src/storage/blobStorage.js' -class InMemoryCOWAdapter implements COWStorageAdapter { +class InMemoryBlobAdapter implements BlobStoreAdapter { private store = new Map() async get(key: string): Promise { return this.store.get(key) } async put(key: string, data: Buffer): Promise { this.store.set(key, data) } @@ -45,12 +45,12 @@ class InMemoryCOWAdapter implements COWStorageAdapter { } describe('BlobStorage compression policy (2.5.0 #32)', () => { - let adapter: InMemoryCOWAdapter + let adapter: InMemoryBlobAdapter let blob: BlobStorage beforeEach(() => { - adapter = new InMemoryCOWAdapter() - blob = new BlobStorage(adapter, { enableCompression: true }) + adapter = new InMemoryBlobAdapter() + blob = new BlobStorage(adapter) }) describe('isAlreadyCompressedMimeType helper', () => { @@ -90,7 +90,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => { it('skips compression for image/jpeg in auto mode (metadata records "none")', async () => { const hash = await blob.write(HIGHLY_COMPRESSIBLE, { - compression: 'auto', type: 'blob', mimeType: 'image/jpeg' + compression: 'auto', mimeType: 'image/jpeg' }) const meta = await blob.getMetadata(hash) expect(meta?.compression).toBe('none') @@ -101,7 +101,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => { it('skips compression for video/mp4 in auto mode', async () => { const hash = await blob.write(HIGHLY_COMPRESSIBLE, { - compression: 'auto', type: 'blob', mimeType: 'video/mp4' + compression: 'auto', mimeType: 'video/mp4' }) const meta = await blob.getMetadata(hash) expect(meta?.compression).toBe('none') @@ -109,7 +109,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => { it('skips compression for application/zip in auto mode', async () => { const hash = await blob.write(HIGHLY_COMPRESSIBLE, { - compression: 'auto', type: 'blob', mimeType: 'application/zip' + compression: 'auto', mimeType: 'application/zip' }) const meta = await blob.getMetadata(hash) expect(meta?.compression).toBe('none') @@ -122,7 +122,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => { // falls back to 'none' — same outcome as a no-zstd install would see // in production. Either way, the policy didn't block on text/plain. const hash = await blob.write(HIGHLY_COMPRESSIBLE, { - compression: 'auto', type: 'blob', mimeType: 'text/plain' + compression: 'auto', mimeType: 'text/plain' }) const meta = await blob.getMetadata(hash) expect(meta?.compression === 'zstd' || meta?.compression === 'none').toBe(true) @@ -130,10 +130,10 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => { it('decompresses transparently on read regardless of compression decision', async () => { const jpegHash = await blob.write(HIGHLY_COMPRESSIBLE, { - compression: 'auto', type: 'blob', mimeType: 'image/jpeg' + compression: 'auto', mimeType: 'image/jpeg' }) const textHash = await blob.write(HIGHLY_COMPRESSIBLE, { - compression: 'auto', type: 'blob', mimeType: 'text/plain' + compression: 'auto', mimeType: 'text/plain' }) // Different mime-type policies, same bytes back on read. expect((await blob.read(jpegHash)).equals(HIGHLY_COMPRESSIBLE)).toBe(true) @@ -145,7 +145,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => { it('compression: "none" is honoured for text/plain (policy can\'t force compression on)', async () => { const data = Buffer.alloc(4096, 0x43) const hash = await blob.write(data, { - compression: 'none', type: 'blob', mimeType: 'text/plain' + compression: 'none', mimeType: 'text/plain' }) const meta = await blob.getMetadata(hash) expect(meta?.compression).toBe('none') @@ -156,7 +156,7 @@ describe('BlobStorage compression policy (2.5.0 #32)', () => { it('compression: "zstd" is recorded as the intent for image/jpeg (policy applies only to auto)', async () => { const data = Buffer.alloc(4096, 0x42) const hash = await blob.write(data, { - compression: 'zstd', type: 'blob', mimeType: 'image/jpeg' + compression: 'zstd', mimeType: 'image/jpeg' }) const meta = await blob.getMetadata(hash) // Explicit zstd → the policy DOESN'T short-circuit; the metadata diff --git a/tests/unit/storage/cow/BlobStorage.test.ts b/tests/unit/storage/blobStorage.test.ts similarity index 53% rename from tests/unit/storage/cow/BlobStorage.test.ts rename to tests/unit/storage/blobStorage.test.ts index 3eacd146..8138c64f 100644 --- a/tests/unit/storage/cow/BlobStorage.test.ts +++ b/tests/unit/storage/blobStorage.test.ts @@ -1,25 +1,28 @@ /** - * Comprehensive tests for BlobStorage + * Comprehensive tests for BlobStorage (src/storage/blobStorage.ts) * * Tests: - * - Content-addressable storage (SHA-256) - * - Deduplication - * - Compression (zstd) - * - LRU caching - * - Batch operations - * - Reference counting - * - Garbage collection + * - Content-addressable storage (SHA-256, integrity verification) + * - Deduplication via reference counting + * - Compression (zstd, MIME-aware auto mode) + * - LRU caching (bounded cache still serves correct bytes) + * - Reference counting + delete-at-zero * - Error handling * - Performance characteristics + * - Wrapped-binary-data regressions (key-based dispatch) + * + * Cache-bypass pattern: the store has no cache-introspection API (the LRU is + * an internal optimization), so tests that must force a storage read create a + * FRESH BlobStorage over the same adapter — a cold cache by construction. */ import { describe, it, expect, beforeEach } from 'vitest' -import { BlobStorage, COWStorageAdapter } from '../../../../src/storage/cow/BlobStorage.js' +import { BlobStorage, BlobStoreAdapter } from '../../../src/storage/blobStorage.js' /** - * Simple in-memory COW storage adapter for testing + * Simple in-memory blob store adapter for testing */ -class InMemoryCOWAdapter implements COWStorageAdapter { +class InMemoryBlobAdapter implements BlobStoreAdapter { private store = new Map() async get(key: string): Promise { @@ -46,12 +49,12 @@ class InMemoryCOWAdapter implements COWStorageAdapter { } describe('BlobStorage', () => { - let adapter: COWStorageAdapter + let adapter: BlobStoreAdapter let blobStorage: BlobStorage beforeEach(() => { - adapter = new InMemoryCOWAdapter() - blobStorage = new BlobStorage(adapter, { enableCompression: true }) + adapter = new InMemoryBlobAdapter() + blobStorage = new BlobStorage(adapter) }) describe('Content-Addressable Storage', () => { @@ -84,14 +87,13 @@ describe('BlobStorage', () => { const data = Buffer.from('test data') const hash = await blobStorage.write(data) - // Clear cache so read() fetches from storage - blobStorage.clearCache() - - // Corrupt the blob data + // Corrupt the blob data behind the store's back await adapter.put(`blob:${hash}`, Buffer.from('corrupted')) - // Should detect corruption via hash verification - await expect(blobStorage.read(hash)).rejects.toThrow('integrity check failed') + // A fresh instance has a cold cache, so read() must hit storage and + // detect the corruption via hash verification. + const coldStore = new BlobStorage(adapter) + await expect(coldStore.read(hash)).rejects.toThrow('integrity check failed') }) it('should check if blob exists', async () => { @@ -112,9 +114,9 @@ describe('BlobStorage', () => { expect(hash1).toBe(hash2) - const stats = blobStorage.getStats() - expect(stats.totalBlobs).toBe(1) - expect(stats.dedupSavings).toBe(data.length) + // Exactly one stored blob + one metadata record + expect(await adapter.list('blob:')).toHaveLength(1) + expect(await adapter.list('blob-meta:')).toHaveLength(1) }) it('should increment ref count on duplicate write', async () => { @@ -127,15 +129,17 @@ describe('BlobStorage', () => { expect(metadata?.refCount).toBe(2) }) - it('should track deduplication savings', async () => { + it('should track every duplicate write in the reference count', async () => { const data = Buffer.from('x'.repeat(1000)) const hash1 = await blobStorage.write(data) const hash2 = await blobStorage.write(data) const hash3 = await blobStorage.write(data) - const stats = blobStorage.getStats() - expect(stats.dedupSavings).toBe(2000) // 2 duplicates × 1000 bytes + expect(hash2).toBe(hash1) + expect(hash3).toBe(hash1) + const metadata = await blobStorage.getMetadata(hash1) + expect(metadata?.refCount).toBe(3) }) }) @@ -143,15 +147,12 @@ describe('BlobStorage', () => { it('should compress large text data with zstd', async () => { const data = Buffer.from('a'.repeat(10000)) - const hash = await blobStorage.write(data, { - type: 'metadata', - compression: 'zstd' - }) + const hash = await blobStorage.write(data, { compression: 'zstd' }) const metadata = await blobStorage.getMetadata(hash) // zstd may not be available in test environment - falls back to 'none' - // This is expected behavior (see BlobStorage initCompression fallback) + // This is expected behavior (see BlobStorage.ensureCompressionReady fallback) if (metadata?.compression === 'zstd') { expect(metadata.compressedSize).toBeLessThan(metadata.size) } else { @@ -163,10 +164,7 @@ describe('BlobStorage', () => { it('should decompress zstd data on read', async () => { const originalData = Buffer.from('test data '.repeat(100)) - const hash = await blobStorage.write(originalData, { - type: 'metadata', - compression: 'zstd' - }) + const hash = await blobStorage.write(originalData, { compression: 'zstd' }) const retrieved = await blobStorage.read(hash) @@ -176,36 +174,30 @@ describe('BlobStorage', () => { it('should not compress small blobs', async () => { const data = Buffer.from('small') - const hash = await blobStorage.write(data, { - compression: 'auto' - }) + const hash = await blobStorage.write(data, { compression: 'auto' }) const metadata = await blobStorage.getMetadata(hash) expect(metadata?.compression).toBe('none') }) - it('should not compress vector data (already dense)', async () => { - const vectorData = Buffer.from(new Float32Array([1, 2, 3, 4, 5])) - - const hash = await blobStorage.write(vectorData, { - type: 'vector', - compression: 'auto' - }) - - const metadata = await blobStorage.getMetadata(hash) - - expect(metadata?.compression).toBe('none') - }) - - it('should auto-compress metadata/tree/commit types', async () => { + it('should skip compression for already-compressed MIME types in auto mode', async () => { const data = Buffer.from('x'.repeat(5000)) const hash = await blobStorage.write(data, { - type: 'metadata', - compression: 'auto' + compression: 'auto', + mimeType: 'image/jpeg' }) + const metadata = await blobStorage.getMetadata(hash) + expect(metadata?.compression).toBe('none') + }) + + it('should auto-compress large compressible payloads', async () => { + const data = Buffer.from('x'.repeat(5000)) + + const hash = await blobStorage.write(data, { compression: 'auto' }) + const metadata = await blobStorage.getMetadata(hash) // Should compress if zstd is available @@ -216,91 +208,37 @@ describe('BlobStorage', () => { }) describe('LRU Caching', () => { - it('should cache blob on read', async () => { + it('should serve identical bytes from cache and from storage', async () => { const data = Buffer.from('cached data') const hash = await blobStorage.write(data) - // First read (cache miss) - await blobStorage.read(hash) + // Warm read (write-through cache) and cold read (fresh instance) + const warm = await blobStorage.read(hash) + const cold = await new BlobStorage(adapter).read(hash) - // Second read (cache hit) - await blobStorage.read(hash) - - const stats = blobStorage.getStats() - expect(stats.cacheHits).toBeGreaterThan(0) + expect(warm.equals(data)).toBe(true) + expect(cold.equals(data)).toBe(true) }) - it('should evict LRU entries when cache is full', async () => { - const smallCache = new BlobStorage(adapter, { - cacheMaxSize: 100, // Very small cache - enableCompression: false - }) + it('should keep serving correct bytes when the cache evicts under pressure', async () => { + const smallCache = new BlobStorage(adapter, { cacheMaxSize: 100 }) - // Write blobs that exceed cache size - const blob1 = Buffer.from('x'.repeat(50)) - const blob2 = Buffer.from('y'.repeat(50)) - const blob3 = Buffer.from('z'.repeat(50)) - - const hash1 = await smallCache.write(blob1) - const hash2 = await smallCache.write(blob2) - const hash3 = await smallCache.write(blob3) // Should evict hash1 - - // Read all blobs - await smallCache.read(hash1) - await smallCache.read(hash2) - await smallCache.read(hash3) - - // hash1 should have been evicted, causing cache miss - const stats = smallCache.getStats() - expect(stats.cacheMisses).toBeGreaterThan(0) - }) - - it('should clear cache on demand', async () => { - const data = Buffer.from('test') - const hash = await blobStorage.write(data) - - await blobStorage.read(hash) // Cache it - - blobStorage.clearCache() - - await blobStorage.read(hash) // Should be cache miss - - const stats = blobStorage.getStats() - expect(stats.cacheMisses).toBeGreaterThan(0) - }) - }) - - describe('Batch Operations', () => { - it('should write multiple blobs in parallel', async () => { - const blobs: Array<[Buffer, any]> = [ - [Buffer.from('blob1'), undefined], - [Buffer.from('blob2'), undefined], - [Buffer.from('blob3'), undefined] + // Write blobs that exceed cache size (forces LRU eviction) + const blobs = [ + Buffer.from('x'.repeat(50)), + Buffer.from('y'.repeat(50)), + Buffer.from('z'.repeat(50)) ] + const hashes: string[] = [] + for (const blob of blobs) { + hashes.push(await smallCache.write(blob)) + } - const hashes = await blobStorage.writeBatch(blobs) - - expect(hashes).toHaveLength(3) - expect(hashes[0]).toHaveLength(64) - expect(hashes[1]).toHaveLength(64) - expect(hashes[2]).toHaveLength(64) - }) - - it('should read multiple blobs in parallel', async () => { - const data1 = Buffer.from('blob1') - const data2 = Buffer.from('blob2') - const data3 = Buffer.from('blob3') - - const hash1 = await blobStorage.write(data1) - const hash2 = await blobStorage.write(data2) - const hash3 = await blobStorage.write(data3) - - const blobs = await blobStorage.readBatch([hash1, hash2, hash3]) - - expect(blobs).toHaveLength(3) - expect(blobs[0].toString()).toBe('blob1') - expect(blobs[1].toString()).toBe('blob2') - expect(blobs[2].toString()).toBe('blob3') + // Every blob still reads back correctly, evicted or not + for (let i = 0; i < blobs.length; i++) { + const retrieved = await smallCache.read(hashes[i]) + expect(retrieved.equals(blobs[i])).toBe(true) + } }) }) @@ -339,120 +277,28 @@ describe('BlobStorage', () => { }) }) - describe('Garbage Collection', () => { - it('should delete unreferenced blobs', async () => { - const blob1 = Buffer.from('referenced') - const blob2 = Buffer.from('unreferenced') - - const hash1 = await blobStorage.write(blob1) - const hash2 = await blobStorage.write(blob2) - - // Manually set refCount to 0 for unreferenced blob - // (In real COW usage, refCount tracks actual references from commits/trees) - // GC only deletes when refCount === 0 AND not in referenced set - const metadata2 = await blobStorage.getMetadata(hash2) - if (metadata2) { - metadata2.refCount = 0 - await adapter.put(`blob-meta:${hash2}`, Buffer.from(JSON.stringify(metadata2))) - } - - // Mark only hash1 as referenced - const referenced = new Set([hash1]) - - const deleted = await blobStorage.garbageCollect(referenced) - - expect(deleted).toBeGreaterThan(0) - expect(await blobStorage.has(hash1)).toBe(true) - expect(await blobStorage.has(hash2)).toBe(false) - }) - - it('should not delete referenced blobs', async () => { - const blob1 = Buffer.from('ref1') - const blob2 = Buffer.from('ref2') - - const hash1 = await blobStorage.write(blob1) - const hash2 = await blobStorage.write(blob2) - - // Both referenced - const referenced = new Set([hash1, hash2]) - - const deleted = await blobStorage.garbageCollect(referenced) - - expect(deleted).toBe(0) - expect(await blobStorage.has(hash1)).toBe(true) - expect(await blobStorage.has(hash2)).toBe(true) - }) - }) - describe('Metadata', () => { it('should store blob metadata', async () => { const data = Buffer.from('test') - const hash = await blobStorage.write(data, { - type: 'metadata', - compression: 'none' - }) + const hash = await blobStorage.write(data, { compression: 'none' }) const metadata = await blobStorage.getMetadata(hash) expect(metadata?.hash).toBe(hash) expect(metadata?.size).toBe(data.length) - expect(metadata?.type).toBe('metadata') expect(metadata?.compression).toBe('none') expect(metadata?.createdAt).toBeGreaterThan(0) expect(metadata?.refCount).toBe(1) }) - }) - describe('List Operations', () => { - it('should list all blobs', async () => { - const hash1 = await blobStorage.write(Buffer.from('blob1')) - const hash2 = await blobStorage.write(Buffer.from('blob2')) - const hash3 = await blobStorage.write(Buffer.from('blob3')) - - const blobs = await blobStorage.listBlobs() - - expect(blobs).toContain(hash1) - expect(blobs).toContain(hash2) - expect(blobs).toContain(hash3) - }) - }) - - describe('Statistics', () => { - it('should track storage statistics', async () => { - const data1 = Buffer.from('x'.repeat(1000)) - const data2 = Buffer.from('y'.repeat(2000)) - - await blobStorage.write(data1) - await blobStorage.write(data2) - - const stats = blobStorage.getStats() - - expect(stats.totalBlobs).toBe(2) - expect(stats.totalSize).toBe(3000) - expect(stats.avgBlobSize).toBe(1500) - }) - - it('should calculate compression ratio', async () => { - const data = Buffer.from('a'.repeat(10000)) - - await blobStorage.write(data, { - type: 'metadata', - compression: 'zstd' - }) - - const stats = blobStorage.getStats() - - // Compression ratio should be > 1 if compressed - if (stats.compressedSize < stats.totalSize) { - expect(stats.compressionRatio).toBeGreaterThan(1) - } + it('should return undefined metadata for unknown hashes', async () => { + expect(await blobStorage.getMetadata('f'.repeat(64))).toBeUndefined() }) }) describe('Error Handling', () => { it('should throw on reading non-existent blob', async () => { - // Use 'f' instead of '0' to avoid NULL_HASH sentinel value await expect( blobStorage.read('f'.repeat(64)) ).rejects.toThrow('Blob metadata not found') @@ -462,14 +308,12 @@ describe('BlobStorage', () => { const data = Buffer.from('test') const hash = await blobStorage.write(data) - // Clear cache so read() actually checks metadata - blobStorage.clearCache() - - // Delete metadata but keep blob + // Delete metadata but keep blob bytes await adapter.delete(`blob-meta:${hash}`) - // Use skipCache to ensure we check metadata - await expect(blobStorage.read(hash, { skipCache: true })).rejects.toThrow('metadata not found') + // A fresh instance has a cold cache, so read() must consult metadata + const coldStore = new BlobStorage(adapter) + await expect(coldStore.read(hash)).rejects.toThrow('metadata not found') }) }) @@ -510,10 +354,10 @@ describe('BlobStorage', () => { }) }) - describe('v5.10.0 Regression - Blob Integrity with Wrapped Data', () => { - it('should handle wrapped binary data without hash mismatch (v5.10.0 fix)', async () => { + describe('Wrapped-binary regression (v5.10.0) - hash verification on unwrapped bytes', () => { + it('should handle wrapped binary data without hash mismatch', async () => { // Import TestWrappingAdapter that actually wraps data like production - const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') + const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() const testBlobStorage = new BlobStorage(wrappingAdapter) @@ -522,20 +366,17 @@ describe('BlobStorage', () => { // Write blob (TestWrappingAdapter wraps as {_binary: true, data: "base64..."}) const hash = await testBlobStorage.write(originalData) - // Clear cache to force re-read from storage (bypasses in-memory cache) - testBlobStorage.clearCache() - - // v5.10.0 bug: This would fail with "Blob integrity check failed" - // because BlobStorage.read() was hashing the wrapped data instead of unwrapped - // v5.10.1 fix: unwrapBinaryData() is called before hash verification - const retrieved = await testBlobStorage.read(hash) + // Fresh instance = cold cache: read() must fetch + unwrap from storage. + // v5.10.0 bug: read() hashed the wrapped bytes instead of the unwrapped + // content; the fix runs unwrapBinaryData() before hash verification. + const retrieved = await new BlobStorage(wrappingAdapter).read(hash) expect(retrieved.equals(originalData)).toBe(true) expect(retrieved.toString()).toBe('test content for v5.10.0 regression test') }) it('should handle multiple wrapped blobs without integrity errors', async () => { - const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') + const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() const testBlobStorage = new BlobStorage(wrappingAdapter) @@ -554,19 +395,17 @@ describe('BlobStorage', () => { hashes.push(hash) } - // Clear cache - testBlobStorage.clearCache() - - // Read all blobs (would fail in v5.10.0) + // Cold-cache instance forces storage reads + const coldStore = new BlobStorage(wrappingAdapter) for (let i = 0; i < hashes.length; i++) { - const retrieved = await testBlobStorage.read(hashes[i]) + const retrieved = await coldStore.read(hashes[i]) expect(retrieved.equals(testData[i])).toBe(true) } }) it('should work even if adapter fails to unwrap (defense-in-depth)', async () => { // Create a buggy adapter that doesn't unwrap properly - class BuggyAdapter implements COWStorageAdapter { + class BuggyAdapter implements BlobStoreAdapter { private storage = new Map() async get(key: string): Promise { @@ -601,21 +440,20 @@ describe('BlobStorage', () => { const originalData = Buffer.from('test defense-in-depth') const hash = await testBlobStorage.write(originalData) - testBlobStorage.clearCache() - - // Even with buggy adapter, BlobStorage.read() should unwrap and verify correctly - const retrieved = await testBlobStorage.read(hash) + // Even with buggy adapter (and a cold cache), read() should unwrap and + // verify correctly + const retrieved = await new BlobStorage(buggyAdapter).read(hash) expect(retrieved.equals(originalData)).toBe(true) }) }) - describe('v6.2.0 Regression - Key-Based Dispatch (Permanent Fix)', () => { + describe('Key-based dispatch regression (v6.2.0)', () => { it('should handle JSON-like compressed data without integrity failures', async () => { // THE KILLER TEST CASE: Data that looks like JSON when compressed // This would fail with v5.10.1 wrapBinaryData() guessing approach - const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') + const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() - const testBlobStorage = new BlobStorage(wrappingAdapter, { enableCompression: true }) + const testBlobStorage = new BlobStorage(wrappingAdapter) // Create JSON data that will be compressed const jsonData = { nested: { key: 'value', array: [1, 2, 3] }, repeated: 'x'.repeat(1000) } @@ -624,54 +462,23 @@ describe('BlobStorage', () => { // Write blob (compression happens, might create JSON-parseable bytes) const hash = await testBlobStorage.write(originalData) - // Clear cache to force re-read from storage - testBlobStorage.clearCache() - // v5.10.1 bug: If compressed bytes accidentally parse as JSON, // wrapBinaryData() would store parsed object instead of wrapped binary // On read: JSON.stringify(object) !== original compressed bytes → hash mismatch // // v6.2.0 fix: Key-based dispatch eliminates guessing // 'blob:hash' → Always wrapped as binary, never parsed - const retrieved = await testBlobStorage.read(hash) + const retrieved = await new BlobStorage(wrappingAdapter).read(hash) // Hash MUST match expect(BlobStorage.hash(retrieved)).toBe(hash) expect(retrieved.equals(originalData)).toBe(true) }) - it('should correctly dispatch all key types', async () => { - // Verify key-based dispatch works for all COW key patterns - const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') - const wrappingAdapter = new TestWrappingAdapter() - const testBlobStorage = new BlobStorage(wrappingAdapter) - - // Test binary blob keys - const blobData = Buffer.from('binary blob content') - const blobHash = await testBlobStorage.write(blobData, { type: 'blob' }) - testBlobStorage.clearCache() - const retrievedBlob = await testBlobStorage.read(blobHash) - expect(retrievedBlob.equals(blobData)).toBe(true) - - // Test commit keys - const commitData = Buffer.from('commit content') - const commitHash = await testBlobStorage.write(commitData, { type: 'commit' }) - testBlobStorage.clearCache() - const retrievedCommit = await testBlobStorage.read(commitHash) - expect(retrievedCommit.equals(commitData)).toBe(true) - - // Test tree keys - const treeData = Buffer.from('tree content') - const treeHash = await testBlobStorage.write(treeData, { type: 'tree' }) - testBlobStorage.clearCache() - const retrievedTree = await testBlobStorage.read(treeHash) - expect(retrievedTree.equals(treeData)).toBe(true) - }) - it('should handle metadata keys correctly', async () => { // Verify that key-based dispatch correctly handles metadata keys // This test verifies the dispatch logic, not the full read/write cycle - const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') + const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() // Test metadata key dispatch: should parse as JSON @@ -688,10 +495,10 @@ describe('BlobStorage', () => { expect(retrieved.size).toBe(42) }) - it('should never call wrapBinaryData on write path', async () => { - // Verify that baseStorage COW adapter uses key-based dispatch, + it('should never parse blob bytes as JSON on the write path', async () => { + // Verify that the blob bridge uses key-based dispatch, // NOT wrapBinaryData() guessing - const { TestWrappingAdapter } = await import('../../../helpers/TestWrappingAdapter.js') + const { TestWrappingAdapter } = await import('../../helpers/TestWrappingAdapter.js') const wrappingAdapter = new TestWrappingAdapter() const testBlobStorage = new BlobStorage(wrappingAdapter) @@ -699,8 +506,7 @@ describe('BlobStorage', () => { const problematicData = Buffer.from('[{"looks":"like","valid":"json"}]') const hash = await testBlobStorage.write(problematicData) - testBlobStorage.clearCache() - const retrieved = await testBlobStorage.read(hash) + const retrieved = await new BlobStorage(wrappingAdapter).read(hash) // Should retrieve exact same bytes (no JSON parsing occurred) expect(retrieved.equals(problematicData)).toBe(true) diff --git a/tests/unit/versioning/VersionDiff.test.ts b/tests/unit/versioning/VersionDiff.test.ts deleted file mode 100644 index aa7a712b..00000000 --- a/tests/unit/versioning/VersionDiff.test.ts +++ /dev/null @@ -1,538 +0,0 @@ -/** - * VersionDiff Unit Tests (v5.3.0) - * - * Tests deep object comparison: - * - Added fields - * - Removed fields - * - Modified fields - * - Type changes - * - Nested object comparison - * - Array comparison - */ - -import { describe, it, expect } from 'vitest' -import { compareEntityVersions } from '../../../src/versioning/VersionDiff.js' -import type { NounMetadata } from '../../../src/coreTypes.js' - -describe('VersionDiff', () => { - describe('compareEntityVersions()', () => { - it('should detect added fields', () => { - const from: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { email: 'alice@example.com' } - } - - const to: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { - email: 'alice@example.com', - city: 'NYC', - age: 30 - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'user-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.added.length).toBe(2) - expect(diff.added.some(c => c.path.includes('city'))).toBe(true) - expect(diff.added.some(c => c.path.includes('age'))).toBe(true) - expect(diff.added.find(c => c.path.includes('city'))?.newValue).toBe('NYC') - }) - - it('should detect removed fields', () => { - const from: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { - email: 'alice@example.com', - city: 'NYC', - age: 30 - } - } - - const to: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { email: 'alice@example.com' } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'user-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.removed.length).toBe(2) - expect(diff.removed.some(c => c.path.includes('city'))).toBe(true) - expect(diff.removed.some(c => c.path.includes('age'))).toBe(true) - expect(diff.removed.find(c => c.path.includes('city'))?.oldValue).toBe('NYC') - }) - - it('should detect modified fields', () => { - const from: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { - email: 'alice@example.com', - status: 'active' - } - } - - const to: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice Smith', - metadata: { - email: 'alice.smith@example.com', - status: 'active' - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'user-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.modified.length).toBe(2) - - const nameChange = diff.modified.find(c => c.path.includes('name')) - expect(nameChange?.oldValue).toBe('Alice') - expect(nameChange?.newValue).toBe('Alice Smith') - - const emailChange = diff.modified.find(c => c.path.includes('email')) - expect(emailChange?.oldValue).toBe('alice@example.com') - expect(emailChange?.newValue).toBe('alice.smith@example.com') - }) - - it('should detect type changes', () => { - const from: NounMetadata = { - id: 'config-1', - type: 'thing', - name: 'Config', - metadata: { value: '100' } - } - - const to: NounMetadata = { - id: 'config-1', - type: 'thing', - name: 'Config', - metadata: { value: 100 } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'config-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.typeChanged.length).toBe(1) - const typeChange = diff.typeChanged[0] - expect(typeChange.path).toContain('value') - expect(typeChange.oldType).toBe('string') - expect(typeChange.newType).toBe('number') - expect(typeChange.oldValue).toBe('100') - expect(typeChange.newValue).toBe(100) - }) - - it('should detect identical versions', () => { - const entity: NounMetadata = { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Unchanged' } - } - - const diff = compareEntityVersions(entity, entity, { - entityId: 'doc-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.identical).toBe(true) - expect(diff.totalChanges).toBe(0) - expect(diff.added).toHaveLength(0) - expect(diff.removed).toHaveLength(0) - expect(diff.modified).toHaveLength(0) - }) - - it('should handle nested object changes', () => { - const from: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { - address: { - street: '123 Main St', - city: 'NYC' - } - } - } - - const to: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { - address: { - street: '456 Oak Ave', - city: 'NYC', - zip: '10001' - } - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'user-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.modified.some(c => c.path.includes('address.street'))).toBe(true) - expect(diff.added.some(c => c.path.includes('address.zip'))).toBe(true) - - const streetChange = diff.modified.find(c => c.path.includes('address.street')) - expect(streetChange?.oldValue).toBe('123 Main St') - expect(streetChange?.newValue).toBe('456 Oak Ave') - }) - - it('should handle array changes', () => { - const from: NounMetadata = { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { - tags: ['a', 'b', 'c'] - } - } - - const to: NounMetadata = { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { - tags: ['a', 'b', 'd'] - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'doc-1', - fromVersion: 1, - toVersion: 2 - }) - - // Array element change detected as modification - expect(diff.totalChanges).toBeGreaterThan(0) - }) - - it('should handle null and undefined', () => { - const from: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { - a: null, - b: undefined, - c: 'value' - } - } - - const to: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { - a: 'now-value', - b: null, - c: 'value' - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'test-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.totalChanges).toBeGreaterThan(0) - }) - - it('should ignore specified fields', () => { - const from: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { - email: 'alice@example.com', - lastModified: 1000, - internal: 'ignore-me' - } - } - - const to: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice Smith', - metadata: { - email: 'alice@example.com', - lastModified: 2000, - internal: 'changed' - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'user-1', - fromVersion: 1, - toVersion: 2, - ignoreFields: ['lastModified', 'internal'] - }) - - // Should only detect name change - expect(diff.totalChanges).toBe(1) - expect(diff.modified.some(c => c.path.includes('name'))).toBe(true) - expect(diff.modified.some(c => c.path.includes('lastModified'))).toBe(false) - expect(diff.modified.some(c => c.path.includes('internal'))).toBe(false) - }) - - it('should respect maxDepth option', () => { - const from: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { - level1: { - level2: { - level3: { - level4: 'deep-value' - } - } - } - } - } - - const to: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { - level1: { - level2: { - level3: { - level4: 'changed-value' - } - } - } - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'test-1', - fromVersion: 1, - toVersion: 2, - maxDepth: 2 // Stop at level2 - }) - - // Should detect change at metadata.level1.level2 level - expect(diff.totalChanges).toBeGreaterThan(0) - }) - - it('should handle empty objects', () => { - const from: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - } - - const to: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { value: 123 } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'test-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.added.length).toBe(1) - expect(diff.added[0].newValue).toBe(123) - }) - - it('should calculate total changes correctly', () => { - const from: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { - email: 'alice@example.com', - status: 'active', - age: 30 - } - } - - const to: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice Smith', - metadata: { - email: 'alice.smith@example.com', - city: 'NYC' - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'user-1', - fromVersion: 1, - toVersion: 2 - }) - - const expectedTotal = - diff.added.length + - diff.removed.length + - diff.modified.length + - diff.typeChanged.length - - expect(diff.totalChanges).toBe(expectedTotal) - }) - - it('should handle complex real-world changes', () => { - const from: NounMetadata = { - id: 'project-1', - type: 'thing', - name: 'My Project', - metadata: { - description: 'Old description', - status: 'draft', - team: ['alice', 'bob'], - config: { - theme: 'light', - notifications: true - }, - createdAt: 1000 - } - } - - const to: NounMetadata = { - id: 'project-1', - type: 'thing', - name: 'My Awesome Project', - metadata: { - description: 'New description', - status: 'active', - team: ['alice', 'bob', 'charlie'], - config: { - theme: 'dark', - notifications: true, - language: 'en' - }, - createdAt: 1000, - updatedAt: 2000 - } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'project-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.totalChanges).toBeGreaterThan(0) - expect(diff.modified.length).toBeGreaterThan(0) - expect(diff.added.length).toBeGreaterThan(0) - - const nameChange = diff.modified.find(c => c.path.includes('name')) - expect(nameChange?.newValue).toBe('My Awesome Project') - }) - - it('should handle boolean changes', () => { - const from: NounMetadata = { - id: 'feature-1', - type: 'thing', - name: 'Feature', - metadata: { enabled: false } - } - - const to: NounMetadata = { - id: 'feature-1', - type: 'thing', - name: 'Feature', - metadata: { enabled: true } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'feature-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.modified.length).toBe(1) - expect(diff.modified[0].oldValue).toBe(false) - expect(diff.modified[0].newValue).toBe(true) - }) - - it('should handle number changes', () => { - const from: NounMetadata = { - id: 'counter-1', - type: 'thing', - name: 'Counter', - metadata: { count: 10 } - } - - const to: NounMetadata = { - id: 'counter-1', - type: 'thing', - name: 'Counter', - metadata: { count: 20 } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'counter-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.modified.length).toBe(1) - expect(diff.modified[0].oldValue).toBe(10) - expect(diff.modified[0].newValue).toBe(20) - }) - - it('should handle date/timestamp changes', () => { - const from: NounMetadata = { - id: 'event-1', - type: 'thing', - name: 'Event', - metadata: { timestamp: 1000 } - } - - const to: NounMetadata = { - id: 'event-1', - type: 'thing', - name: 'Event', - metadata: { timestamp: 2000 } - } - - const diff = compareEntityVersions(from, to, { - entityId: 'event-1', - fromVersion: 1, - toVersion: 2 - }) - - expect(diff.modified.length).toBe(1) - expect(diff.modified[0].path).toContain('timestamp') - }) - }) -}) diff --git a/tests/unit/versioning/VersionManager.test.ts b/tests/unit/versioning/VersionManager.test.ts deleted file mode 100644 index de181cce..00000000 --- a/tests/unit/versioning/VersionManager.test.ts +++ /dev/null @@ -1,710 +0,0 @@ -/** - * VersionManager Unit Tests (v5.3.0) - * - * Tests the core versioning engine in isolation: - * - Save versions - * - Restore versions - * - List versions - * - Compare versions - * - Prune versions - * - Content deduplication - * - Branch awareness - */ - -import { describe, it, expect, beforeEach, vi } from 'vitest' -import { VersionManager } from '../../../src/versioning/VersionManager.js' -import type { NounMetadata } from '../../../src/coreTypes.js' - -describe('VersionManager', () => { - let manager: VersionManager - let mockBrain: any - let mockStorage: Map - let mockIndex: Map - - beforeEach(() => { - // Mock storage - mockStorage = new Map() - mockIndex = new Map() - - // Mock Brainy instance - mockBrain = { - currentBranch: 'main', - storageAdapter: { - // v6.3.0: VersionStorage now uses saveMetadata/getMetadata for key-value storage - saveMetadata: vi.fn(async (key: string, data: any) => { - mockStorage.set(key, JSON.stringify(data)) - }), - getMetadata: vi.fn(async (key: string) => { - const data = mockStorage.get(key) - if (!data) return null - return JSON.parse(data) - }), - // Legacy methods for compatibility - exists: vi.fn(async (path: string) => mockStorage.has(path)), - writeFile: vi.fn(async (path: string, data: string) => { - mockStorage.set(path, data) - }), - readFile: vi.fn(async (path: string) => { - const data = mockStorage.get(path) - if (!data) throw new Error('File not found') - return data - }), - deleteFile: vi.fn(async (path: string) => { - mockStorage.delete(path) - }) - }, - refManager: { - getRef: vi.fn(async (branch: string) => ({ - name: `refs/heads/${branch}`, - commitHash: 'commit-hash-123', - type: 'branch' - })), - setRef: vi.fn(async () => {}), - resolveRef: vi.fn(async () => 'commit-hash-123') - }, - getNounMetadata: vi.fn(async (id: string) => { - const entity = mockIndex.get(id) - // Return null for version entities (_version:...) that don't exist - // Throw for regular entities that don't exist - if (!entity) { - if (id.startsWith('_version:')) { - return null // Version doesn't exist - let VersionIndex handle it - } - throw new Error(`Entity ${id} not found`) - } - return entity - }), - saveNounMetadata: vi.fn(async (id: string, data: NounMetadata) => { - mockIndex.set(id, data) - }), - deleteNounMetadata: vi.fn(async (id: string) => { - mockIndex.delete(id) - }), - searchByMetadata: vi.fn(async (query: any) => { - const results: any[] = [] - for (const [id, entity] of mockIndex.entries()) { - let matches = true - for (const [key, value] of Object.entries(query)) { - // Handle top-level properties (like 'type') - if (key === 'type') { - if (entity.type !== value) { - matches = false - break - } - continue // Skip metadata check for 'type' - } - // Check metadata properties with glob pattern support - const entityValue = entity.metadata?.[key] - // Support simple glob patterns (e.g., 'v1.*' matches 'v1.0.0', 'v1.1.0') - if (typeof value === 'string' && value.includes('*')) { - const pattern = new RegExp('^' + value.replace(/\*/g, '.*') + '$') - if (!entityValue || !pattern.test(String(entityValue))) { - matches = false - break - } - } else if (entityValue !== value) { - matches = false - break - } - } - if (matches) { - results.push({ id, ...entity }) - } - } - return results - }), - commit: vi.fn(async () => 'commit-hash-123'), - // v6.3.0: VersionManager.restore() uses brain.update() to refresh all indexes - update: vi.fn(async (opts: { id: string, data?: any, type?: string, metadata?: any, confidence?: number, weight?: number, merge?: boolean }) => { - // Simulate update by merging metadata into the entity - const existing = mockIndex.get(opts.id) - if (!existing) { - throw new Error(`Entity ${opts.id} not found for update`) - } - const updated = { - ...existing, - data: opts.data !== undefined ? opts.data : existing.data, - type: opts.type || existing.type, - confidence: opts.confidence !== undefined ? opts.confidence : existing.confidence, - weight: opts.weight !== undefined ? opts.weight : existing.weight, - ...(opts.merge === false ? opts.metadata : { ...existing.metadata, ...opts.metadata }) - } - mockIndex.set(opts.id, updated) - }) - } - - manager = new VersionManager(mockBrain) - }) - - describe('save()', () => { - it('should save a new version', async () => { - // Add test entity - mockIndex.set('user-123', { - id: 'user-123', - type: 'user', - name: 'Alice', - metadata: { email: 'alice@example.com' } - }) - - const version = await manager.save('user-123', { - tag: 'v1.0', - description: 'Initial version' - }) - - expect(version.version).toBe(1) - expect(version.entityId).toBe('user-123') - expect(version.branch).toBe('main') - expect(version.tag).toBe('v1.0') - expect(version.description).toBe('Initial version') - expect(version.contentHash).toBeDefined() - expect(version.commitHash).toBe('commit-hash-123') - }) - - it('should increment version numbers', async () => { - mockIndex.set('doc-1', { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Version 1' } - }) - - const v1 = await manager.save('doc-1', { tag: 'v1' }) - expect(v1.version).toBe(1) - - // Update entity - mockIndex.set('doc-1', { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Version 2' } - }) - - const v2 = await manager.save('doc-1', { tag: 'v2' }) - expect(v2.version).toBe(2) - - mockIndex.set('doc-1', { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Version 3' } - }) - - const v3 = await manager.save('doc-1', { tag: 'v3' }) - expect(v3.version).toBe(3) - }) - - it('should deduplicate identical content', async () => { - mockIndex.set('note-1', { - id: 'note-1', - type: 'document', - name: 'Note', - metadata: { content: 'Unchanged' } - }) - - const v1 = await manager.save('note-1', { tag: 'v1' }) - const v2 = await manager.save('note-1', { tag: 'v2' }) - - // Should return same version (content unchanged) - expect(v2.version).toBe(v1.version) - expect(v2.contentHash).toBe(v1.contentHash) - }) - - it('should throw if entity does not exist', async () => { - await expect( - manager.save('nonexistent', { tag: 'v1' }) - ).rejects.toThrow('Entity nonexistent not found') - }) - - it('should save without tag or description', async () => { - mockIndex.set('test-1', { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - }) - - const version = await manager.save('test-1') - expect(version.version).toBe(1) - expect(version.tag).toBeUndefined() - expect(version.description).toBeUndefined() - }) - }) - - describe('restore()', () => { - it('should restore entity to specific version', async () => { - // Save version 1 - mockIndex.set('config-1', { - id: 'config-1', - type: 'thing', - name: 'Config', - metadata: { theme: 'light', version: 1 } - }) - await manager.save('config-1', { tag: 'v1' }) - - // Update to version 2 - mockIndex.set('config-1', { - id: 'config-1', - type: 'thing', - name: 'Config', - metadata: { theme: 'dark', version: 2 } - }) - await manager.save('config-1', { tag: 'v2' }) - - // Restore to v1 - await manager.restore('config-1', 1) - - // v6.3.0: restore() uses brain.update() to refresh all indexes - expect(mockBrain.update).toHaveBeenCalledWith( - expect.objectContaining({ - id: 'config-1', - merge: false // Replace, don't merge - }) - ) - }) - - it('should restore by tag', async () => { - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { status: 'alpha' } - }) - await manager.save('app-1', { tag: 'alpha' }) - - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { status: 'beta' } - }) - await manager.save('app-1', { tag: 'beta' }) - - // Restore to alpha - await manager.restore('app-1', 'alpha') - - // v6.3.0: restore() uses brain.update() to refresh all indexes - expect(mockBrain.update).toHaveBeenCalledWith( - expect.objectContaining({ - id: 'app-1', - merge: false // Replace, don't merge - }) - ) - }) - - it('should throw if version not found', async () => { - mockIndex.set('test-1', { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - }) - await manager.save('test-1') - - await expect( - manager.restore('test-1', 999) - ).rejects.toThrow('Version 999 not found') - }) - }) - - describe('list()', () => { - it('should list all versions for entity', async () => { - mockIndex.set('log-1', { - id: 'log-1', - type: 'document', - name: 'Log', - metadata: { entry: 1 } - }) - - await manager.save('log-1', { tag: 'v1' }) - - mockIndex.set('log-1', { - id: 'log-1', - type: 'document', - name: 'Log', - metadata: { entry: 2 } - }) - await manager.save('log-1', { tag: 'v2' }) - - mockIndex.set('log-1', { - id: 'log-1', - type: 'document', - name: 'Log', - metadata: { entry: 3 } - }) - await manager.save('log-1', { tag: 'v3' }) - - const versions = await manager.list('log-1') - expect(versions).toHaveLength(3) - expect(versions[0].version).toBe(3) // Newest first - expect(versions[1].version).toBe(2) - expect(versions[2].version).toBe(1) - }) - - it('should filter by exact tag', async () => { - // v6.3.0: Tag filtering is exact match only (no glob patterns) - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { v: 1 } - }) - - await manager.save('app-1', { tag: 'v1.0.0' }) - - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { v: 2 } - }) - await manager.save('app-1', { tag: 'v1.1.0' }) - - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { v: 3 } - }) - await manager.save('app-1', { tag: 'v2.0.0' }) - - // v6.3.0: Exact tag match only - const v1Versions = await manager.list('app-1', { tag: 'v1.0.0' }) - expect(v1Versions).toHaveLength(1) - expect(v1Versions[0].tag).toBe('v1.0.0') - - // Get all versions - const allVersions = await manager.list('app-1') - expect(allVersions).toHaveLength(3) - }) - - it('should limit results', async () => { - mockIndex.set('data-1', { - id: 'data-1', - type: 'thing', - name: 'Data', - metadata: { v: 1 } - }) - - for (let i = 1; i <= 5; i++) { - mockIndex.set('data-1', { - id: 'data-1', - type: 'thing', - name: 'Data', - metadata: { v: i } - }) - await manager.save('data-1', { tag: `v${i}` }) - } - - const versions = await manager.list('data-1', { limit: 3 }) - expect(versions).toHaveLength(3) - expect(versions[0].version).toBe(5) - expect(versions[1].version).toBe(4) - expect(versions[2].version).toBe(3) - }) - - it('should return empty array if no versions', async () => { - mockIndex.set('new-1', { - id: 'new-1', - type: 'thing', - name: 'New', - metadata: {} - }) - - const versions = await manager.list('new-1') - expect(versions).toHaveLength(0) - }) - }) - - describe('compare()', () => { - it('should compare two versions', async () => { - mockIndex.set('user-1', { - id: 'user-1', - type: 'user', - name: 'Bob', - metadata: { email: 'bob@example.com', age: 30 } - }) - await manager.save('user-1', { tag: 'v1' }) - - mockIndex.set('user-1', { - id: 'user-1', - type: 'user', - name: 'Robert', - metadata: { email: 'robert@example.com', age: 30, city: 'NYC' } - }) - await manager.save('user-1', { tag: 'v2' }) - - const diff = await manager.compare('user-1', 1, 2) - - expect(diff.totalChanges).toBeGreaterThan(0) - expect(diff.modified.length).toBeGreaterThan(0) - expect(diff.added.length).toBeGreaterThan(0) - - const nameChange = diff.modified.find(c => c.path.includes('name')) - expect(nameChange?.oldValue).toBe('Bob') - expect(nameChange?.newValue).toBe('Robert') - - const cityAdd = diff.added.find(c => c.path.includes('city')) - expect(cityAdd?.newValue).toBe('NYC') - }) - - it('should compare by tags', async () => { - mockIndex.set('doc-1', { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Alpha' } - }) - await manager.save('doc-1', { tag: 'alpha' }) - - mockIndex.set('doc-1', { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Beta' } - }) - await manager.save('doc-1', { tag: 'beta' }) - - const diff = await manager.compare('doc-1', 'alpha', 'beta') - - expect(diff.modified.length).toBeGreaterThan(0) - const contentChange = diff.modified.find(c => c.path.includes('content')) - expect(contentChange?.oldValue).toBe('Alpha') - expect(contentChange?.newValue).toBe('Beta') - }) - - it('should detect identical versions', async () => { - mockIndex.set('same-1', { - id: 'same-1', - type: 'thing', - name: 'Same', - metadata: { value: 100 } - }) - await manager.save('same-1', { tag: 'v1' }) - await manager.save('same-1', { tag: 'v2' }) // Content unchanged - - const diff = await manager.compare('same-1', 1, 1) - expect(diff.identical).toBe(true) - expect(diff.totalChanges).toBe(0) - }) - }) - - describe('prune()', () => { - it('should prune old versions keeping recent', async () => { - mockIndex.set('log-1', { - id: 'log-1', - type: 'document', - name: 'Log', - metadata: { entry: 1 } - }) - - for (let i = 1; i <= 10; i++) { - mockIndex.set('log-1', { - id: 'log-1', - type: 'document', - name: 'Log', - metadata: { entry: i } - }) - await manager.save('log-1', { tag: `v${i}` }) - } - - const result = await manager.prune('log-1', { - keepRecent: 5, - keepTagged: false - }) - - expect(result.deleted).toBe(5) - expect(result.kept).toBe(5) - - const remaining = await manager.list('log-1') - expect(remaining).toHaveLength(5) - expect(remaining[0].version).toBe(10) // Most recent - }) - - it('should keep tagged versions', async () => { - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { v: 1 } - }) - - await manager.save('app-1', { tag: 'release' }) - - for (let i = 2; i <= 10; i++) { - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { v: i } - }) - await manager.save('app-1') // No tag - } - - const result = await manager.prune('app-1', { - keepRecent: 3, - keepTagged: true - }) - - const remaining = await manager.list('app-1') - const releaseVersion = remaining.find(v => v.tag === 'release') - expect(releaseVersion).toBeDefined() - }) - - it('should respect keepAfter timestamp', async () => { - mockIndex.set('data-1', { - id: 'data-1', - type: 'thing', - name: 'Data', - metadata: { v: 1 } - }) - - const now = Date.now() - const oneDayAgo = now - 24 * 60 * 60 * 1000 - - await manager.save('data-1', { tag: 'old' }) - - // Simulate newer versions - for (let i = 2; i <= 5; i++) { - mockIndex.set('data-1', { - id: 'data-1', - type: 'thing', - name: 'Data', - metadata: { v: i } - }) - await manager.save('data-1', { tag: `new${i}` }) - } - - const result = await manager.prune('data-1', { - keepAfter: oneDayAgo, - keepTagged: false - }) - - expect(result.deleted).toBeGreaterThanOrEqual(0) - }) - }) - - describe('getVersion()', () => { - it('should get specific version by number', async () => { - mockIndex.set('test-1', { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { v: 1 } - }) - const v1 = await manager.save('test-1', { tag: 'v1' }) - - mockIndex.set('test-1', { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { v: 2 } - }) - await manager.save('test-1', { tag: 'v2' }) - - const version = await manager.getVersion('test-1', 1) - expect(version).toBeDefined() - expect(version?.version).toBe(1) - expect(version?.tag).toBe('v1') - }) - - it('should return null if version not found', async () => { - mockIndex.set('test-1', { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - }) - await manager.save('test-1') - - const version = await manager.getVersion('test-1', 999) - expect(version).toBeNull() - }) - }) - - describe('getVersionByTag()', () => { - it('should get version by tag', async () => { - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { status: 'alpha' } - }) - await manager.save('app-1', { tag: 'alpha' }) - - mockIndex.set('app-1', { - id: 'app-1', - type: 'thing', - name: 'App', - metadata: { status: 'beta' } - }) - await manager.save('app-1', { tag: 'beta' }) - - const betaVersion = await manager.getVersionByTag('app-1', 'beta') - expect(betaVersion).toBeDefined() - expect(betaVersion?.tag).toBe('beta') - }) - - it('should return null if tag not found', async () => { - mockIndex.set('test-1', { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - }) - await manager.save('test-1', { tag: 'v1' }) - - const version = await manager.getVersionByTag('test-1', 'nonexistent') - expect(version).toBeNull() - }) - }) - - describe('getVersionCount()', () => { - it('should count versions', async () => { - mockIndex.set('doc-1', { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { v: 1 } - }) - - expect(await manager.getVersionCount('doc-1')).toBe(0) - - await manager.save('doc-1') - expect(await manager.getVersionCount('doc-1')).toBe(1) - - mockIndex.set('doc-1', { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { v: 2 } - }) - await manager.save('doc-1') - expect(await manager.getVersionCount('doc-1')).toBe(2) - }) - }) - - describe('Branch Awareness', () => { - it('should track versions per branch', async () => { - mockIndex.set('branch-test', { - id: 'branch-test', - type: 'thing', - name: 'Test', - metadata: {} - }) - - // Save on main - mockBrain.currentBranch = 'main' - const mainV1 = await manager.save('branch-test', { tag: 'main-v1' }) - expect(mainV1.branch).toBe('main') - - // Save on feature - mockBrain.currentBranch = 'feature' - const featureV1 = await manager.save('branch-test', { tag: 'feature-v1' }) - expect(featureV1.branch).toBe('feature') - - // Versions should be independent - expect(mainV1.version).toBe(1) - expect(featureV1.version).toBe(1) // Each branch starts at 1 - }) - }) -}) diff --git a/tests/unit/versioning/VersionStorage.test.ts b/tests/unit/versioning/VersionStorage.test.ts deleted file mode 100644 index 21be5872..00000000 --- a/tests/unit/versioning/VersionStorage.test.ts +++ /dev/null @@ -1,567 +0,0 @@ -/** - * VersionStorage Unit Tests (v5.3.0) - * - * Tests content-addressable storage: - * - SHA-256 content hashing - * - Content deduplication - * - Storage adapter integration - * - Save/load/delete operations - */ - -import { describe, it, expect, beforeEach, vi } from 'vitest' -import { VersionStorage } from '../../../src/versioning/VersionStorage.js' -import type { NounMetadata } from '../../../src/coreTypes.js' -import type { EntityVersion } from '../../../src/versioning/VersionManager.js' - -describe('VersionStorage', () => { - let storage: VersionStorage - let mockBrain: any - let mockFiles: Map - - beforeEach(() => { - mockFiles = new Map() - - mockBrain = { - storageAdapter: { - // v6.3.0: VersionStorage now uses saveMetadata/getMetadata instead of writeFile/readFile - saveMetadata: vi.fn(async (key: string, data: any) => { - mockFiles.set(key, JSON.stringify(data)) - }), - getMetadata: vi.fn(async (key: string) => { - const data = mockFiles.get(key) - if (!data) return null - return JSON.parse(data) - }), - // Legacy methods for tests that still use them - exists: vi.fn(async (path: string) => mockFiles.has(path)), - writeFile: vi.fn(async (path: string, data: string) => { - mockFiles.set(path, data) - }), - readFile: vi.fn(async (path: string) => { - const data = mockFiles.get(path) - if (!data) throw new Error('File not found') - return data - }), - deleteFile: vi.fn(async (path: string) => { - mockFiles.delete(path) - }) - } - } - - storage = new VersionStorage(mockBrain) - }) - - describe('hashEntity()', () => { - it('should generate consistent SHA-256 hashes', () => { - const entity: NounMetadata = { - id: 'user-123', - type: 'user', - name: 'Alice', - metadata: { email: 'alice@example.com' } - } - - const hash1 = storage.hashEntity(entity) - const hash2 = storage.hashEntity(entity) - - expect(hash1).toBe(hash2) - expect(hash1).toHaveLength(64) // SHA-256 = 64 hex chars - }) - - it('should generate different hashes for different content', () => { - const entity1: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: {} - } - - const entity2: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Bob', - metadata: {} - } - - const hash1 = storage.hashEntity(entity1) - const hash2 = storage.hashEntity(entity2) - - expect(hash1).not.toBe(hash2) - }) - - it('should ignore property order (stable JSON)', () => { - const entity1: NounMetadata = { - id: 'user-1', - type: 'user', - name: 'Alice', - metadata: { a: 1, b: 2, c: 3 } - } - - const entity2: NounMetadata = { - type: 'user', - metadata: { c: 3, b: 2, a: 1 }, - name: 'Alice', - id: 'user-1' - } - - const hash1 = storage.hashEntity(entity1) - const hash2 = storage.hashEntity(entity2) - - expect(hash1).toBe(hash2) - }) - - it('should handle nested objects', () => { - const entity: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { - nested: { - deep: { - value: 'hello' - } - } - } - } - - const hash = storage.hashEntity(entity) - expect(hash).toHaveLength(64) - }) - - it('should handle arrays', () => { - const entity: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { - tags: ['a', 'b', 'c'] - } - } - - const hash = storage.hashEntity(entity) - expect(hash).toHaveLength(64) - }) - - it('should handle null and undefined', () => { - const entity1: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { value: null } - } - - const entity2: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { value: undefined } - } - - const hash1 = storage.hashEntity(entity1) - const hash2 = storage.hashEntity(entity2) - - expect(hash1).not.toBe(hash2) - }) - }) - - describe('saveVersion()', () => { - it('should save version content to storage', async () => { - const entity: NounMetadata = { - id: 'user-123', - type: 'user', - name: 'Alice', - metadata: { email: 'alice@example.com' } - } - - const version: EntityVersion = { - version: 1, - entityId: 'user-123', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - await storage.saveVersion(version, entity) - - // v6.3.0: Uses __system_version_ prefix for saveMetadata keys - const expectedKey = `__system_version_user-123_${version.contentHash}` - expect(mockFiles.has(expectedKey)).toBe(true) - - const savedData = mockFiles.get(expectedKey) - expect(savedData).toBeDefined() - expect(JSON.parse(savedData!)).toEqual(entity) - }) - - it('should deduplicate identical content', async () => { - const entity: NounMetadata = { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Same content' } - } - - const contentHash = storage.hashEntity(entity) - - const version1: EntityVersion = { - version: 1, - entityId: 'doc-1', - branch: 'main', - commitHash: 'commit-1', - timestamp: Date.now(), - contentHash - } - - const version2: EntityVersion = { - version: 2, - entityId: 'doc-1', - branch: 'main', - commitHash: 'commit-2', - timestamp: Date.now() + 1000, - contentHash // Same hash! - } - - await storage.saveVersion(version1, entity) - const writeCallCount1 = mockBrain.storageAdapter.writeFile.mock.calls.length - - await storage.saveVersion(version2, entity) - const writeCallCount2 = mockBrain.storageAdapter.writeFile.mock.calls.length - - // Should not write again (deduplication) - expect(writeCallCount2).toBe(writeCallCount1) - }) - - it('should store different content separately', async () => { - const entity1: NounMetadata = { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Version 1' } - } - - const entity2: NounMetadata = { - id: 'doc-1', - type: 'document', - name: 'Doc', - metadata: { content: 'Version 2' } - } - - const version1: EntityVersion = { - version: 1, - entityId: 'doc-1', - branch: 'main', - commitHash: 'commit-1', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity1) - } - - const version2: EntityVersion = { - version: 2, - entityId: 'doc-1', - branch: 'main', - commitHash: 'commit-2', - timestamp: Date.now() + 1000, - contentHash: storage.hashEntity(entity2) - } - - await storage.saveVersion(version1, entity1) - await storage.saveVersion(version2, entity2) - - // v6.3.0: Uses __system_version_ prefix for saveMetadata keys - const key1 = `__system_version_doc-1_${version1.contentHash}` - const key2 = `__system_version_doc-1_${version2.contentHash}` - - expect(mockFiles.has(key1)).toBe(true) - expect(mockFiles.has(key2)).toBe(true) - expect(key1).not.toBe(key2) - }) - }) - - describe('loadVersion()', () => { - it('should load version content from storage', async () => { - const entity: NounMetadata = { - id: 'user-123', - type: 'user', - name: 'Alice', - metadata: { email: 'alice@example.com' } - } - - const version: EntityVersion = { - version: 1, - entityId: 'user-123', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - // Save first - await storage.saveVersion(version, entity) - - // Load - const loaded = await storage.loadVersion(version) - - expect(loaded).toEqual(entity) - }) - - it('should return null if version not found', async () => { - const version: EntityVersion = { - version: 1, - entityId: 'nonexistent', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: 'fake-hash' - } - - const loaded = await storage.loadVersion(version) - expect(loaded).toBeNull() - }) - - it('should handle complex nested data', async () => { - const entity: NounMetadata = { - id: 'complex-1', - type: 'thing', - name: 'Complex', - metadata: { - nested: { - array: [1, 2, 3], - object: { key: 'value' } - }, - tags: ['a', 'b', 'c'] - } - } - - const version: EntityVersion = { - version: 1, - entityId: 'complex-1', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - await storage.saveVersion(version, entity) - const loaded = await storage.loadVersion(version) - - expect(loaded).toEqual(entity) - }) - }) - - describe('deleteVersion()', () => { - it('should handle version deletion (content-addressed, no-op)', async () => { - // v6.3.0: Version content is content-addressed and may be shared by multiple versions. - // The deleteVersion method is a no-op - it doesn't actually delete the content. - // Version INDEX is deleted via VersionIndex.removeVersion(). - // A GC process could clean up unreferenced content in the future. - - const entity: NounMetadata = { - id: 'user-123', - type: 'user', - name: 'Alice', - metadata: {} - } - - const version: EntityVersion = { - version: 1, - entityId: 'user-123', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - // Save - await storage.saveVersion(version, entity) - const key = `__system_version_user-123_${version.contentHash}` - expect(mockFiles.has(key)).toBe(true) - - // Delete is a no-op for content (content-addressed, may be shared) - await storage.deleteVersion(version) - // Content is NOT deleted to avoid breaking other versions with same content hash - expect(mockFiles.has(key)).toBe(true) // v6.3.0: Content preserved - }) - - it('should handle deleting non-existent version gracefully', async () => { - const version: EntityVersion = { - version: 1, - entityId: 'nonexistent', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: 'fake-hash' - } - - // Should not throw - await storage.deleteVersion(version) - }) - }) - - describe('Storage Adapter Integration', () => { - it('should work with memory storage adapter', async () => { - const entity: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: { value: 123 } - } - - const version: EntityVersion = { - version: 1, - entityId: 'test-1', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - await storage.saveVersion(version, entity) - const loaded = await storage.loadVersion(version) - - expect(loaded).toEqual(entity) - }) - - it('should use adapter.saveMetadata API', async () => { - // v6.3.0: VersionStorage now uses saveMetadata/getMetadata exclusively - const testStorage = new Map() - - mockBrain.storageAdapter = { - saveMetadata: vi.fn(async (key: string, data: any) => { - testStorage.set(key, JSON.stringify(data)) - }), - getMetadata: vi.fn(async (key: string) => { - const data = testStorage.get(key) - if (!data) return null - return JSON.parse(data) - }) - } - - storage = new VersionStorage(mockBrain) - - const entity: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - } - - const version: EntityVersion = { - version: 1, - entityId: 'test-1', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - await storage.saveVersion(version, entity) - expect(mockBrain.storageAdapter.saveMetadata).toHaveBeenCalled() - - const loaded = await storage.loadVersion(version) - expect(mockBrain.storageAdapter.getMetadata).toHaveBeenCalled() - expect(loaded).toEqual(entity) - }) - - it('should throw if storage adapter missing', async () => { - mockBrain.storageAdapter = null - storage = new VersionStorage(mockBrain) - - const entity: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - } - - const version: EntityVersion = { - version: 1, - entityId: 'test-1', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - await expect( - storage.saveVersion(version, entity) - ).rejects.toThrow('Storage adapter not available') - }) - - it('should throw if adapter does not support required operations', async () => { - mockBrain.storageAdapter = {} // No methods (no saveMetadata) - storage = new VersionStorage(mockBrain) - - const entity: NounMetadata = { - id: 'test-1', - type: 'thing', - name: 'Test', - metadata: {} - } - - const version: EntityVersion = { - version: 1, - entityId: 'test-1', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash: storage.hashEntity(entity) - } - - // v6.3.0: Error from calling undefined saveMetadata - await expect( - storage.saveVersion(version, entity) - ).rejects.toThrow() // TypeError: adapter.saveMetadata is not a function - }) - }) - - describe('Key Generation', () => { - it('should generate correct storage keys', async () => { - const entity: NounMetadata = { - id: 'user-123', - type: 'user', - name: 'Alice', - metadata: {} - } - - const contentHash = storage.hashEntity(entity) - const version: EntityVersion = { - version: 1, - entityId: 'user-123', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash - } - - await storage.saveVersion(version, entity) - - // v6.3.0: Uses __system_version_ prefix for saveMetadata keys - const expectedKey = `__system_version_user-123_${contentHash}` - expect(mockFiles.has(expectedKey)).toBe(true) - }) - - it('should handle entity IDs with special characters', async () => { - const entity: NounMetadata = { - id: 'user:123:profile', - type: 'user', - name: 'Alice', - metadata: {} - } - - const contentHash = storage.hashEntity(entity) - const version: EntityVersion = { - version: 1, - entityId: 'user:123:profile', - branch: 'main', - commitHash: 'commit-123', - timestamp: Date.now(), - contentHash - } - - await storage.saveVersion(version, entity) - - // v6.3.0: Uses __system_version_ prefix for saveMetadata keys - const expectedKey = `__system_version_user:123:profile_${contentHash}` - expect(mockFiles.has(expectedKey)).toBe(true) - }) - }) -})