From 364360d44751a01d36585e654e4bef98d3c457f8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 27 Jan 2026 15:38:21 -0800 Subject: [PATCH] fix: exclude __words__ keyword index from corruption detection and getStats() The __words__ keyword index stores 50-5000 entries per entity (one per word), which inflated avg entries/entity well above the corruption threshold of 100. This caused: 1. validateConsistency() to falsely detect corruption on every startup, triggering unnecessary clearAllIndexData() + rebuild() cycles 2. getStats() to log false "Metadata index may be corrupted" warnings and report inflated totalEntries/totalIds stats Both methods now skip __words__ when counting, so stats and health checks reflect metadata fields only (noun, type, createdAt, etc.). Keyword search is unaffected since the __words__ field index itself is not modified. --- docs/API_REFERENCE.md | 756 ++++++++-------- docs/BATCHING.md | 143 ++- docs/CREATING-AUGMENTATIONS.md | 430 ++++----- docs/DEVELOPER_LEARNING_PATH.md | 4 +- docs/FIND_SYSTEM.md | 835 +++++++++--------- docs/PERFORMANCE.md | 8 +- docs/README.md | 147 ++- docs/RELEASE-GUIDE.md | 2 +- docs/STAGE3-CANONICAL-TAXONOMY.md | 2 +- docs/api/COMPREHENSIVE_API_OVERVIEW.md | 234 ++--- .../architecture/data-storage-architecture.md | 32 +- docs/architecture/index-architecture.md | 32 +- .../initialization-and-rebuild.md | 8 +- docs/architecture/storage-architecture.md | 382 ++++---- docs/augmentations/COMPLETE-REFERENCE.md | 248 +++--- docs/augmentations/DEVELOPER-GUIDE.md | 530 +++++------ docs/deployment/CLOUD_DEPLOYMENT_GUIDE.md | 782 ++++++++-------- docs/deployment/aws-deployment.md | 480 +++++----- docs/deployment/gcp-deployment.md | 648 +++++++------- docs/features/instant-fork.md | 161 ++-- docs/guides/import-anything.md | 162 ++-- docs/guides/import-flow.md | 4 +- docs/guides/import-progress-examples.md | 278 +++--- docs/guides/import-progress-implementation.md | 664 +++++++------- docs/guides/import-quick-reference.md | 4 +- docs/guides/standard-import-progress.md | 2 +- docs/guides/streaming-imports.md | 4 +- docs/operations/capacity-planning.md | 397 +++++---- docs/operations/cost-optimization-aws-s3.md | 170 ++-- docs/operations/cost-optimization-azure.md | 306 ++++--- .../cost-optimization-cloudflare-r2.md | 162 ++-- docs/operations/cost-optimization-gcs.md | 142 ++- docs/transactions.md | 246 +++--- docs/vfs/QUICK_START.md | 358 ++++---- docs/vfs/README.md | 348 ++++---- docs/vfs/TROUBLESHOOTING.md | 84 +- docs/vfs/VFS_CORE.md | 4 +- docs/vfs/VFS_INITIALIZATION.md | 8 +- src/api/ConfigAPI.ts | 12 +- src/api/DataAPI.ts | 2 +- src/augmentations/brainyAugmentation.ts | 10 +- .../FormatHandlerRegistry.ts | 2 +- .../IntelligentImportAugmentation.ts | 2 +- .../intelligentImport/handlers/csvHandler.ts | 12 +- .../handlers/excelHandler.ts | 18 +- .../handlers/imageHandler.ts | 6 +- .../intelligentImport/handlers/pdfHandler.ts | 14 +- src/augmentations/intelligentImport/index.ts | 4 +- src/augmentations/intelligentImport/types.ts | 8 +- src/augmentations/typeMatching/brainyTypes.ts | 2 +- src/cli/commands/core.ts | 4 +- src/cli/commands/cow.ts | 4 +- src/cli/commands/neural.ts | 4 +- src/cli/commands/storage.ts | 2 +- src/cli/commands/utility.ts | 2 +- src/cli/commands/vfs.ts | 6 +- src/cli/index.ts | 10 +- src/cli/interactive.ts | 2 +- src/coreTypes.ts | 68 +- src/embeddings/candle-wasm/src/lib.rs | 4 +- src/embeddings/wasm/CandleEmbeddingEngine.ts | 10 +- src/embeddings/wasm/modelLoader.ts | 2 +- src/graph/graphAdjacencyIndex.ts | 37 +- src/hnsw/hnswIndex.ts | 61 +- src/hnsw/typeAwareHNSWIndex.ts | 26 +- src/import/FormatDetector.ts | 8 +- src/import/ImportCoordinator.ts | 110 +-- src/importers/SmartCSVImporter.ts | 14 +- src/importers/SmartDOCXImporter.ts | 6 +- src/importers/SmartExcelImporter.ts | 28 +- src/importers/SmartJSONImporter.ts | 6 +- src/importers/SmartMarkdownImporter.ts | 6 +- src/importers/SmartPDFImporter.ts | 16 +- src/importers/SmartYAMLImporter.ts | 6 +- src/importers/VFSStructureGenerator.ts | 26 +- src/index.ts | 6 +- src/integrations/index.ts | 2 +- src/interfaces/IIndex.ts | 6 +- src/neural/entityExtractor.ts | 20 +- src/neural/naturalLanguageProcessor.ts | 2 +- src/neural/signals/PatternSignal.ts | 4 +- src/setup.ts | 2 +- src/storage/adapters/azureBlobStorage.ts | 108 ++- src/storage/adapters/baseStorageAdapter.ts | 43 +- src/storage/adapters/fileSystemStorage.ts | 109 ++- src/storage/adapters/gcsStorage.ts | 135 ++- .../adapters/historicalStorageAdapter.ts | 10 +- src/storage/adapters/memoryStorage.ts | 49 +- src/storage/adapters/opfsStorage.ts | 43 +- src/storage/adapters/optimizedS3Search.ts | 4 +- src/storage/adapters/r2Storage.ts | 49 +- src/storage/adapters/s3CompatibleStorage.ts | 98 +- src/storage/backwardCompatibility.ts | 4 +- src/storage/baseStorage.ts | 339 ++++--- src/storage/cow/BlobStorage.ts | 14 +- src/storage/cow/CommitLog.ts | 2 +- src/storage/cow/CommitObject.ts | 2 +- src/storage/cow/RefManager.ts | 2 +- src/storage/cow/binaryDataCodec.ts | 4 +- src/storage/storageFactory.ts | 23 +- src/types/brainyInterface.ts | 6 +- src/types/graphTypes.ts | 4 +- src/types/typeMigration.ts | 8 +- src/utils/entityIdMapper.ts | 6 +- src/utils/fieldTypeInference.ts | 6 +- src/utils/import-progress-tracker.ts | 3 +- src/utils/memoryDetection.ts | 6 +- src/utils/metadataFilter.ts | 2 +- src/utils/metadataIndex.ts | 209 ++--- src/utils/metadataIndexChunking.ts | 6 +- src/utils/periodicCleanup.ts | 2 +- src/utils/rebuildCounts.ts | 14 +- src/utils/unifiedCache.ts | 24 +- src/versioning/VersionDiff.ts | 2 +- src/versioning/VersionIndex.ts | 2 +- src/versioning/VersionManager.ts | 10 +- src/versioning/VersionStorage.ts | 18 +- src/versioning/VersioningAPI.ts | 2 +- src/vfs/MimeTypeDetector.ts | 2 +- src/vfs/PathResolver.ts | 26 +- src/vfs/VirtualFileSystem.ts | 88 +- src/vfs/importers/DirectoryImporter.ts | 10 +- src/vfs/index.ts | 2 +- src/vfs/semantic/SemanticPathResolver.ts | 2 +- .../semantic/projections/AuthorProjection.ts | 2 +- src/vfs/semantic/projections/TagProjection.ts | 2 +- .../projections/TemporalProjection.ts | 2 +- src/vfs/types.ts | 6 +- 128 files changed, 5637 insertions(+), 5682 deletions(-) diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 2ca57833..4beb2e49 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -26,16 +26,16 @@ import { Brainy, NounType, VerbType } from '@soulcraft/brainy' // Initialize const brain = new Brainy({ - storage: { type: 'filesystem', path: './brainy-data' }, - model: { type: 'fast', precision: 'Q8' } + storage: { type: 'filesystem', path: './brainy-data' }, + model: { type: 'fast', precision: 'Q8' } }) await brain.init() // Add entities const id = await brain.add({ - data: 'John Smith is a software engineer', - type: NounType.Person, - metadata: { role: 'engineer' } + data: 'John Smith is a software engineer', + type: NounType.Person, + metadata: { role: 'engineer' } }) // Search @@ -60,9 +60,9 @@ Creates a new Brainy instance. **Example:** ```typescript const brain = new Brainy({ - storage: { type: 'filesystem', options: { path: './data' } }, - model: { type: 'accurate', precision: 'FP32' }, - cache: { maxSize: 5000, ttl: 600000 } + storage: { type: 'filesystem', options: { path: './data' } }, + model: { type: 'accurate', precision: 'FP32' }, + cache: { maxSize: 5000, ttl: 600000 } }) ``` @@ -90,8 +90,8 @@ Adds a new entity to the brain. - `id` - Custom ID (auto-generated if not provided) - `vector` - Pre-computed embedding vector - `service` - Service name for multi-tenancy -- `confidence` - Type classification confidence (0-1) ✨ *New in v4.3.0* -- `weight` - Entity importance/salience (0-1) ✨ *New in v4.3.0* +- `confidence` - Type classification confidence (0-1) ✨ +- `weight` - Entity importance/salience (0-1) ✨ - `writeOnly` - Skip validation for high-speed ingestion **Returns:** Entity ID @@ -99,15 +99,15 @@ Adds a new entity to the brain. **Example:** ```typescript const id = await brain.add({ - data: 'Important meeting notes from Q4 planning', - type: NounType.Document, - metadata: { - date: '2024-01-15', - author: 'John Smith', - tags: ['planning', 'Q4'] - }, - confidence: 0.95, // High confidence in Document classification - weight: 0.85 // High importance + data: 'Important meeting notes from Q4 planning', + type: NounType.Document, + metadata: { + date: '2024-01-15', + author: 'John Smith', + tags: ['planning', 'Q4'] + }, + confidence: 0.95, // High confidence in Document classification + weight: 0.85 // High importance }) ``` @@ -116,18 +116,18 @@ const id = await brain.add({ ### `async get(id: string, options?: GetOptions): Promise` Retrieves an entity by ID. -✨ **v5.11.1 Performance Optimization**: +✨ **Performance Optimization**: - **Default (metadata-only)**: 76-81% faster, 95% less bandwidth - perfect for VFS, existence checks, metadata access - **Opt-in vectors**: Use `{ includeVectors: true }` when computing similarity **Parameters:** - `id` - Entity ID - `options` (optional): - - `includeVectors?: boolean` - Include 384-dim vectors (default: false for 76-81% speedup) + - `includeVectors?: boolean` - Include 384-dim vectors (default: false for 76-81% speedup) **Returns:** Entity object or null if not found -**Entity Properties:** ✨ *Updated in v4.3.0, v5.11.1* +**Entity Properties:** - `id` - Unique identifier - `type` - NounType classification - `data` - Original content @@ -144,9 +144,9 @@ Retrieves an entity by ID. // Perfect for VFS, metadata access, existence checks const entity = await brain.get('uuid-1234') if (entity) { - console.log(entity.type) // NounType.Document - console.log(entity.metadata) // { date: '2024-01-15', ... } - console.log(entity.vector) // [] (empty - not loaded for performance) + console.log(entity.type) // NounType.Document + console.log(entity.metadata) // { date: '2024-01-15', ... } + console.log(entity.vector) // [] (empty - not loaded for performance) } ``` @@ -155,8 +155,8 @@ if (entity) { // Use when computing similarity on this specific entity const entity = await brain.get('uuid-1234', { includeVectors: true }) if (entity) { - console.log(entity.vector.length) // 384 (full embeddings loaded) - // Now can use for similarity calculations + console.log(entity.vector.length) // 384 (full embeddings loaded) + // Now can use for similarity calculations } ``` @@ -172,17 +172,17 @@ Updates an existing entity. - `metadata` - New or partial metadata - `merge` - Merge metadata (true) or replace (false), default: true - `vector` - New embedding vector -- `confidence` - Update type classification confidence ✨ *New in v4.3.0* -- `weight` - Update entity importance/salience ✨ *New in v4.3.0* +- `confidence` - Update type classification confidence ✨ +- `weight` - Update entity importance/salience ✨ **Example:** ```typescript await brain.update({ - id: 'uuid-1234', - metadata: { status: 'reviewed' }, - confidence: 0.98, // Increase confidence after review - weight: 0.90, // Boost importance - merge: true // Keeps existing metadata, adds status + id: 'uuid-1234', + metadata: { status: 'reviewed' }, + confidence: 0.98, // Increase confidence after review + weight: 0.90, // Boost importance + merge: true // Keeps existing metadata, adds status }) ``` @@ -224,12 +224,12 @@ Creates a relationship between two entities. **Example:** ```typescript const relationId = await brain.relate({ - from: 'person-123', - to: 'org-456', - type: VerbType.WorksWith, - weight: 0.95, - metadata: { since: '2020-01-01' }, - bidirectional: true + from: 'person-123', + to: 'org-456', + type: VerbType.WorksWith, + weight: 0.95, + metadata: { since: '2020-01-01' }, + bidirectional: true }) ``` @@ -263,9 +263,9 @@ Gets relationships for entities. ```typescript // Get all relationships from an entity const relations = await brain.getRelations({ - from: 'person-123', - type: [VerbType.WorksWith, VerbType.ReportsTo], - limit: 50 + from: 'person-123', + type: [VerbType.WorksWith, VerbType.ReportsTo], + limit: 50 }) ``` @@ -298,13 +298,13 @@ Adds multiple entities in batch. **Example:** ```typescript const result = await brain.addMany({ - items: [ - { data: 'Doc 1', type: NounType.Document }, - { data: 'Doc 2', type: NounType.Document }, - { data: 'Doc 3', type: NounType.Document } - ], - parallel: true, - onProgress: (done, total) => console.log(`${done}/${total}`) + items: [ + { data: 'Doc 1', type: NounType.Document }, + { data: 'Doc 2', type: NounType.Document }, + { data: 'Doc 3', type: NounType.Document } + ], + parallel: true, + onProgress: (done, total) => console.log(`${done}/${total}`) }) console.log(`Added: ${result.successful.length}`) @@ -325,12 +325,12 @@ Updates multiple entities in batch. **Example:** ```typescript const result = await brain.updateMany({ - updates: ids.map(id => ({ - id, - metadata: { processed: true }, - merge: true - })), - parallel: true + updates: ids.map(id => ({ + id, + metadata: { processed: true }, + merge: true + })), + parallel: true }) ``` @@ -350,14 +350,14 @@ Deletes multiple entities. ```typescript // Delete specific IDs await brain.deleteMany({ - ids: ['id1', 'id2', 'id3'] + ids: ['id1', 'id2', 'id3'] }) // Delete by type await brain.deleteMany({ - type: NounType.Document, - where: { status: 'draft' }, - limit: 100 + type: NounType.Document, + where: { status: 'draft' }, + limit: 100 }) ``` @@ -375,11 +375,11 @@ Creates multiple relationships in batch. **Example:** ```typescript const result = await brain.relateMany({ - relations: [ - { from: 'a', to: 'b', type: VerbType.RelatedTo }, - { from: 'b', to: 'c', type: VerbType.DependsOn }, - { from: 'c', to: 'a', type: VerbType.References } - ] + relations: [ + { from: 'a', to: 'b', type: VerbType.RelatedTo }, + { from: 'b', to: 'c', type: VerbType.DependsOn }, + { from: 'c', to: 'a', type: VerbType.References } + ] }) ``` @@ -406,7 +406,7 @@ Universal search with Triple Intelligence fusion. **Returns:** Array of Result objects with scores -**Result Properties:** ✨ *Enhanced in v4.3.0* +**Result Properties:** ✨ - `id` - Entity ID - `score` - Relevance score (0-1) - `type` - Entity type (flattened for convenience) *Enhanced* @@ -422,38 +422,38 @@ Universal search with Triple Intelligence fusion. // Natural language search const results = await brain.find('recent product launches') -// NEW in v4.3.0: Direct access to flattened fields -console.log(results[0].metadata) // Direct access (convenient!) -console.log(results[0].confidence) // Type confidence -console.log(results[0].weight) // Entity importance +// Direct access to flattened fields +console.log(results[0].metadata) // Direct access (convenient!) +console.log(results[0].confidence) // Type confidence +console.log(results[0].weight) // Entity importance // Backward compatible: Nested access still works -console.log(results[0].entity.metadata) // Also works +console.log(results[0].entity.metadata) // Also works // Structured search with fusion const results = await brain.find({ - query: 'machine learning', - type: [NounType.Document, NounType.Project], - where: { year: 2024 }, - connected: { - to: 'research-dept', - via: VerbType.CreatedBy - }, - fusion: { - strategy: 'adaptive', - weights: { vector: 0.5, graph: 0.3, field: 0.2 } - }, - limit: 20, - explain: true + query: 'machine learning', + type: [NounType.Document, NounType.Project], + where: { year: 2024 }, + connected: { + to: 'research-dept', + via: VerbType.CreatedBy + }, + fusion: { + strategy: 'adaptive', + weights: { vector: 0.5, graph: 0.3, field: 0.2 } + }, + limit: 20, + explain: true }) // Access results with clean, predictable patterns for (const result of results) { - console.log(`Score: ${result.score}`) - console.log(`Type: ${result.type}`) - console.log(`Confidence: ${result.confidence ?? 'N/A'}`) - console.log(`Weight: ${result.weight ?? 'N/A'}`) - console.log(`Metadata:`, result.metadata) + console.log(`Score: ${result.score}`) + console.log(`Type: ${result.type}`) + console.log(`Confidence: ${result.confidence ?? 'N/A'}`) + console.log(`Weight: ${result.weight ?? 'N/A'}`) + console.log(`Metadata:`, result.metadata) } ``` @@ -469,29 +469,29 @@ Finds similar entities using vector similarity. - `type` - Filter by type(s) - `where` - Metadata filters -**Returns:** Array of Result objects (same structure as `find()`) ✨ *Enhanced in v4.3.0* +**Returns:** Array of Result objects (same structure as `find()`) ✨ **Example:** ```typescript const similar = await brain.similar({ - to: 'doc-123', - limit: 5, - threshold: 0.8, - type: NounType.Document + to: 'doc-123', + limit: 5, + threshold: 0.8, + type: NounType.Document }) -// NEW in v4.3.0: Access flattened fields directly +// Access flattened fields directly for (const result of similar) { - console.log(`Similarity: ${result.score}`) - console.log(`Type: ${result.type}`) // Flattened - console.log(`Confidence: ${result.confidence}`) // Flattened - console.log(`Metadata:`, result.metadata) // Flattened + console.log(`Similarity: ${result.score}`) + console.log(`Type: ${result.type}`) // Flattened + console.log(`Confidence: ${result.confidence}`) // Flattened + console.log(`Metadata:`, result.metadata) // Flattened } ``` --- -### `async embed(data: any): Promise` ✨ *v7.1.0* +### `async embed(data: any): Promise` Generates an embedding vector from data. **Parameters:** @@ -507,7 +507,7 @@ console.log(vector.length) // 384 --- -### `async embedBatch(texts: string[]): Promise` ✨ *v7.1.0, Optimized v7.9.0* +### `async embedBatch(texts: string[]): Promise` Batch embed multiple texts using native WASM batch API (single forward pass). **Parameters:** @@ -515,22 +515,22 @@ Batch embed multiple texts using native WASM batch API (single forward pass). **Returns:** Array of 384-dimensional vectors -> **v7.9.0**: Uses the engine's native `embed_batch()` for a single model forward pass instead of N individual `embed()` calls. +> Uses the engine's native `embed_batch()` for a single model forward pass instead of N individual `embed()` calls. **Example:** ```typescript const embeddings = await brain.embedBatch([ - 'Machine learning is fascinating', - 'Deep neural networks', - 'Natural language processing' + 'Machine learning is fascinating', + 'Deep neural networks', + 'Natural language processing' ]) -console.log(embeddings.length) // 3 +console.log(embeddings.length) // 3 console.log(embeddings[0].length) // 384 ``` --- -### `async similarity(textA: string, textB: string): Promise` ✨ *v7.1.0* +### `async similarity(textA: string, textB: string): Promise` Calculate semantic similarity between two texts. **Parameters:** @@ -542,15 +542,15 @@ Calculate semantic similarity between two texts. **Example:** ```typescript const score = await brain.similarity( - 'The cat sat on the mat', - 'A feline was resting on the rug' + 'The cat sat on the mat', + 'A feline was resting on the rug' ) console.log(score) // ~0.85 (high semantic similarity) ``` --- -### `async neighbors(entityId: string, options?): Promise` ✨ *v7.1.0* +### `async neighbors(entityId: string, options?): Promise` Get graph neighbors of an entity. **Parameters:** @@ -569,20 +569,20 @@ const neighbors = await brain.neighbors(entityId) // Get outgoing connections only const outgoing = await brain.neighbors(entityId, { - direction: 'outgoing', - limit: 10 + direction: 'outgoing', + limit: 10 }) // Multi-hop traversal const extended = await brain.neighbors(entityId, { - depth: 2, - direction: 'both' + depth: 2, + direction: 'both' }) ``` --- -### `async findDuplicates(options?): Promise` ✨ *v7.1.0* +### `async findDuplicates(options?): Promise` Find semantic duplicates in the database. **Parameters:** @@ -598,23 +598,23 @@ Find semantic duplicates in the database. const duplicates = await brain.findDuplicates() for (const group of duplicates) { - console.log('Original:', group.entity.id) - for (const dup of group.duplicates) { - console.log(` Duplicate: ${dup.entity.id} (${dup.similarity.toFixed(2)})`) - } + console.log('Original:', group.entity.id) + for (const dup of group.duplicates) { + console.log(` Duplicate: ${dup.entity.id} (${dup.similarity.toFixed(2)})`) + } } // Find person duplicates with higher threshold const personDupes = await brain.findDuplicates({ - type: NounType.PERSON, - threshold: 0.9, - limit: 50 + type: NounType.PERSON, + threshold: 0.9, + limit: 50 }) ``` --- -### `async indexStats(): Promise` ✨ *v7.1.0* +### `async indexStats(): Promise` Get comprehensive index statistics. **Returns:** @@ -639,7 +639,7 @@ console.log(`Fields: ${stats.metadataFields.join(', ')}`) --- -### `async cluster(options?): Promise` ✨ *v7.1.0* +### `async cluster(options?): Promise` Cluster entities by semantic similarity. Groups entities into clusters based on their embedding similarity using @@ -660,23 +660,23 @@ a greedy algorithm with HNSW-based neighbor lookup. const clusters = await brain.cluster() for (const cluster of clusters) { - console.log(`${cluster.clusterId}: ${cluster.entities.length} entities`) + console.log(`${cluster.clusterId}: ${cluster.entities.length} entities`) } // Find document clusters with centroids const docClusters = await brain.cluster({ - type: NounType.Document, - threshold: 0.85, - minClusterSize: 3, - includeCentroid: true + type: NounType.Document, + threshold: 0.85, + minClusterSize: 3, + includeCentroid: true }) // Use centroids for cluster comparison for (const cluster of docClusters) { - console.log(`${cluster.clusterId}: ${cluster.entities.length} entities`) - if (cluster.centroid) { - console.log(` Centroid dimensions: ${cluster.centroid.length}`) - } + console.log(`${cluster.clusterId}: ${cluster.entities.length} entities`) + if (cluster.centroid) { + console.log(` Centroid dimensions: ${cluster.centroid.length}`) + } } ``` @@ -719,13 +719,13 @@ AI-powered suggestions based on current data. **Example:** ```typescript const suggestions = await brain.suggest({ - context: 'product development', - limit: 5 + context: 'product development', + limit: 5 }) // Use suggestions for (const query of suggestions.queries) { - console.log(`Try searching: ${query}`) + console.log(`Try searching: ${query}`) } ``` @@ -826,10 +826,10 @@ Persistent local storage using the filesystem. ```typescript const brain = new Brainy({ - storage: { - type: 'filesystem', - rootDirectory: './brainy-data' - } + storage: { + type: 'filesystem', + rootDirectory: './brainy-data' + } }) ``` @@ -838,15 +838,15 @@ Scalable cloud storage with S3. ```typescript const brain = new Brainy({ - storage: { - type: 's3', - s3Storage: { - bucketName: 'my-bucket', - region: 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY - } - } + storage: { + type: 's3', + s3Storage: { + bucketName: 'my-bucket', + region: 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + } + } }) ``` @@ -855,15 +855,15 @@ Scalable cloud storage with Cloudflare R2 (S3-compatible). ```typescript const brain = new Brainy({ - storage: { - type: 'r2', - r2Storage: { - bucketName: 'my-bucket', - accountId: process.env.CF_ACCOUNT_ID, - accessKeyId: process.env.R2_ACCESS_KEY_ID, - secretAccessKey: process.env.R2_SECRET_ACCESS_KEY - } - } + storage: { + type: 'r2', + r2Storage: { + bucketName: 'my-bucket', + accountId: process.env.CF_ACCOUNT_ID, + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY + } + } }) ``` @@ -881,42 +881,42 @@ const brain = new Brainy({ With Application Default Credentials (Cloud Run/GCE): ```typescript const brain = new Brainy({ - storage: { - type: 'gcs-native', // ⚠️ Must be 'gcs-native' for native SDK - gcsNativeStorage: { - bucketName: 'my-bucket' - // No credentials needed - ADC automatic! - } - } + storage: { + type: 'gcs-native', // ⚠️ Must be 'gcs-native' for native SDK + gcsNativeStorage: { + bucketName: 'my-bucket' + // No credentials needed - ADC automatic! + } + } }) ``` With Service Account Key File: ```typescript const brain = new Brainy({ - storage: { - type: 'gcs-native', - gcsNativeStorage: { - bucketName: 'my-bucket', - keyFilename: '/path/to/service-account.json' - } - } + storage: { + type: 'gcs-native', + gcsNativeStorage: { + bucketName: 'my-bucket', + keyFilename: '/path/to/service-account.json' + } + } }) ``` With Service Account Credentials Object: ```typescript const brain = new Brainy({ - storage: { - type: 'gcs-native', - gcsNativeStorage: { - bucketName: 'my-bucket', - credentials: { - client_email: 'service@project.iam.gserviceaccount.com', - private_key: process.env.GCS_PRIVATE_KEY - } - } - } + storage: { + type: 'gcs-native', + gcsNativeStorage: { + bucketName: 'my-bucket', + credentials: { + client_email: 'service@project.iam.gserviceaccount.com', + private_key: process.env.GCS_PRIVATE_KEY + } + } + } }) ``` @@ -925,12 +925,12 @@ const brain = new Brainy({ You can omit the `type` field and let Brainy auto-detect based on the config object: ```typescript const brain = new Brainy({ - storage: { - gcsNativeStorage: { - bucketName: 'my-bucket' - // type defaults to 'auto', will use native SDK - } - } + storage: { + gcsNativeStorage: { + bucketName: 'my-bucket' + // type defaults to 'auto', will use native SDK + } + } }) ``` @@ -939,16 +939,16 @@ GCS using HMAC keys for S3-compatible access. **Consider migrating to 'gcs-nativ ```typescript const brain = new Brainy({ - storage: { - type: 'gcs', // ⚠️ Must be 'gcs' for S3-compatible mode - gcsStorage: { // ⚠️ Must use 'gcsStorage' (not 'gcsNativeStorage') - bucketName: 'my-bucket', - region: 'us-central1', - accessKeyId: process.env.GCS_ACCESS_KEY_ID, - secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY, - endpoint: 'https://storage.googleapis.com' - } - } + storage: { + type: 'gcs', // ⚠️ Must be 'gcs' for S3-compatible mode + gcsStorage: { // ⚠️ Must use 'gcsStorage' (not 'gcsNativeStorage') + bucketName: 'my-bucket', + region: 'us-central1', + accessKeyId: process.env.GCS_ACCESS_KEY_ID, + secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY, + endpoint: 'https://storage.googleapis.com' + } + } }) ``` @@ -956,25 +956,25 @@ const brain = new Brainy({ ```typescript // ❌ WRONG - type/config mismatch (will fall back to memory storage) { - type: 'gcs', - gcsNativeStorage: { bucketName: 'my-bucket' } + type: 'gcs', + gcsNativeStorage: { bucketName: 'my-bucket' } } // ❌ WRONG - type/config mismatch (will fall back to memory storage) { - type: 'gcs-native', - gcsStorage: { bucketName: 'my-bucket', accessKeyId: '...', secretAccessKey: '...' } + type: 'gcs-native', + gcsStorage: { bucketName: 'my-bucket', accessKeyId: '...', secretAccessKey: '...' } } // ✅ CORRECT - type matches config object { - type: 'gcs-native', - gcsNativeStorage: { bucketName: 'my-bucket' } + type: 'gcs-native', + gcsNativeStorage: { bucketName: 'my-bucket' } } // ✅ CORRECT - auto-detection { - gcsNativeStorage: { bucketName: 'my-bucket' } + gcsNativeStorage: { bucketName: 'my-bucket' } } ``` @@ -995,51 +995,51 @@ If you're currently using `type: 'gcs'` with HMAC keys, migrating to `type: 'gcs **Before (S3-Compatible with HMAC):** ```typescript const brain = new Brainy({ - storage: { - type: 'gcs', // ⚠️ Old: S3-compatible mode - gcsStorage: { // ⚠️ Old: HMAC credentials - bucketName: 'my-bucket', - accessKeyId: process.env.GCS_ACCESS_KEY_ID, - secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY - } - } + storage: { + type: 'gcs', // ⚠️ Old: S3-compatible mode + gcsStorage: { // ⚠️ Old: HMAC credentials + bucketName: 'my-bucket', + accessKeyId: process.env.GCS_ACCESS_KEY_ID, + secretAccessKey: process.env.GCS_SECRET_ACCESS_KEY + } + } }) ``` **After (Native SDK with ADC):** ```typescript const brain = new Brainy({ - storage: { - type: 'gcs-native', // ✅ New: Native SDK mode - gcsNativeStorage: { // ✅ New: ADC authentication - bucketName: 'my-bucket' - // ADC handles authentication automatically - } - } + storage: { + type: 'gcs-native', // ✅ New: Native SDK mode + gcsNativeStorage: { // ✅ New: ADC authentication + bucketName: 'my-bucket' + // ADC handles authentication automatically + } + } }) ``` **⚠️ Important Migration Notes:** 1. **Change BOTH the type AND the config object:** - - `type: 'gcs'` → `type: 'gcs-native'` - - `gcsStorage` → `gcsNativeStorage` + - `type: 'gcs'` → `type: 'gcs-native'` + - `gcsStorage` → `gcsNativeStorage` 2. **Remove HMAC keys** - Not needed with ADC: - - Remove `accessKeyId` - - Remove `secretAccessKey` - - Remove `region` (optional with native SDK) + - Remove `accessKeyId` + - Remove `secretAccessKey` + - Remove `region` (optional with native SDK) 3. **Set up ADC in your environment:** - ```bash - # Cloud Run/GCE: Nothing needed, ADC is automatic + ```bash + # Cloud Run/GCE: Nothing needed, ADC is automatic - # Local development: - gcloud auth application-default login + # Local development: + gcloud auth application-default login - # Or set GOOGLE_APPLICATION_CREDENTIALS: - export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" - ``` + # Or set GOOGLE_APPLICATION_CREDENTIALS: + export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" + ``` **Data Migration:** ✅ **No data migration required!** Both adapters use the same path structure: @@ -1066,9 +1066,9 @@ Sets configuration value. **Example:** ```typescript await config.set({ - key: 'api.key', - value: 'secret-key-123', - encrypt: true + key: 'api.key', + value: 'secret-key-123', + encrypt: true }) ``` @@ -1085,9 +1085,9 @@ Gets configuration value. **Example:** ```typescript const apiKey = await config.get({ - key: 'api.key', - decrypt: true, - defaultValue: 'default-key' + key: 'api.key', + decrypt: true, + defaultValue: 'default-key' }) ``` @@ -1137,8 +1137,8 @@ Creates backup of all data. **Example:** ```typescript const backup = await data.backup({ - includeVectors: true, - compress: true + includeVectors: true, + compress: true }) // Save backup to file fs.writeFileSync('backup.json', JSON.stringify(backup)) @@ -1159,9 +1159,9 @@ Restores from backup. ```typescript const backup = JSON.parse(fs.readFileSync('backup.json')) await data.restore({ - backup, - merge: false, - overwrite: true + backup, + merge: false, + overwrite: true }) ``` @@ -1178,8 +1178,8 @@ Clears specified data types. **Example:** ```typescript await data.clear({ - entities: true, - relations: true + entities: true, + relations: true }) ``` @@ -1216,44 +1216,44 @@ const csvResult = await brain.import('customers.csv') // Import Excel workbook const excelResult = await brain.import('sales-data.xlsx', { - excelSheets: ['Q1', 'Q2'] // Import specific sheets + excelSheets: ['Q1', 'Q2'] // Import specific sheets }) // Import PDF with table extraction const pdfResult = await brain.import('report.pdf', { - pdfExtractTables: true + pdfExtractTables: true }) // Import data array const dataResult = await brain.import([ - { name: 'Alice', role: 'Engineer' }, - { name: 'Bob', role: 'Designer' } + { name: 'Alice', role: 'Engineer' }, + { name: 'Bob', role: 'Designer' } ], { - batchSize: 100, - relationships: true // Auto-extract relationships + batchSize: 100, + relationships: true // Auto-extract relationships }) // Import with custom CSV delimiter const tsvResult = await brain.import('data.tsv', { - format: 'csv', - csvDelimiter: '\t' + format: 'csv', + csvDelimiter: '\t' }) ``` **Returns:** ```typescript { - success: boolean - imported: number // Number of items successfully imported - failed: number // Number of items that failed - entityIds: string[] // IDs of created entities - metadata: { - format: string // Detected format - encoding?: string // Detected encoding (CSV) - delimiter?: string // Detected delimiter (CSV) - sheets?: string[] // Processed sheets (Excel) - pageCount?: number // Number of pages (PDF) - } + success: boolean + imported: number // Number of items successfully imported + failed: number // Number of items that failed + entityIds: string[] // IDs of created entities + metadata: { + format: string // Detected format + encoding?: string // Detected encoding (CSV) + delimiter?: string // Detected delimiter (CSV) + sheets?: string[] // Processed sheets (Excel) + pageCount?: number // Number of pages (PDF) + } } ``` @@ -1270,9 +1270,9 @@ Exports data in various formats. **Example:** ```typescript const exported = await data.export({ - format: 'csv', - where: { type: NounType.Document }, - includeVectors: false + format: 'csv', + where: { type: NounType.Document }, + includeVectors: false }) ``` @@ -1284,18 +1284,18 @@ Gets complete statistics about entities and relationships. All stats are **O(1) **Returns:** ```typescript { - entities: { - total: number // Total entity count - byType: Record // Entity count by type - } - relationships: { - totalRelationships: number // Total relationship/edge count - relationshipsByType: Record // Relationship count by type - uniqueSourceNodes: number // Number of unique source entities - uniqueTargetNodes: number // Number of unique target entities - totalNodes: number // Total unique entities in relationships - } - density: number // Relationships per entity ratio + entities: { + total: number // Total entity count + byType: Record // Entity count by type + } + relationships: { + totalRelationships: number // Total relationship/edge count + relationshipsByType: Record // Relationship count by type + uniqueSourceNodes: number // Number of unique source entities + uniqueTargetNodes: number // Number of unique target entities + totalNodes: number // Total unique entities in relationships + } + density: number // Relationships per entity ratio } ``` @@ -1306,7 +1306,7 @@ const stats = brain.getStats() // Total counts (O(1) operations) const totalNouns = stats.entities.total const totalVerbs = stats.relationships.totalRelationships -const totalRelations = stats.relationships.totalRelationships // alias +const totalRelations = stats.relationships.totalRelationships // alias // Counts by type (O(1) operations) const nounTypes = stats.entities.byType @@ -1346,16 +1346,16 @@ Query entities with pagination. **Example:** ```typescript const result = await brain.query.entities({ - type: NounType.Document, - where: { status: 'published' }, - limit: 50 + type: NounType.Document, + where: { status: 'published' }, + limit: 50 }) // Get next page if (result.nextCursor) { - const nextPage = await brain.query.entities({ - cursor: result.nextCursor - }) + const nextPage = await brain.query.entities({ + cursor: result.nextCursor + }) } ``` @@ -1385,7 +1385,7 @@ Calculates similarity between items. **Example:** ```typescript const similarity = await neural.similar('doc1', 'doc2', { - explain: true + explain: true }) ``` @@ -1405,8 +1405,8 @@ General purpose clustering. **Example:** ```typescript const clusters = await neural.clusters({ - algorithm: 'kmeans', - k: 5 + algorithm: 'kmeans', + k: 5 }) ``` @@ -1418,9 +1418,9 @@ Domain-specific clustering. **Example:** ```typescript const clusters = await neural.clustersByDomain({ - domain: 'technology', - field: 'category', - maxClusters: 10 + domain: 'technology', + field: 'category', + maxClusters: 10 }) ``` @@ -1439,9 +1439,9 @@ Detects anomalous entities. **Example:** ```typescript const outliers = await neural.outliers({ - method: 'isolation', - threshold: 3.0, - returnScores: true + method: 'isolation', + threshold: 3.0, + returnScores: true }) ``` @@ -1460,9 +1460,9 @@ Generates visualization data for entities. **Example:** ```typescript const vizData = await neural.visualize({ - layout: 'force', - dimensions: 3, - includeEdges: true + layout: 'force', + dimensions: 3, + includeEdges: true }) ``` @@ -1478,7 +1478,7 @@ Converts natural language to structured query. **Example:** ```typescript const structured = await nlp.processNaturalQuery( - "Find all documents about AI created last month" + "Find all documents about AI created last month" ) // Returns structured query with type, time filters, etc. ``` @@ -1499,18 +1499,18 @@ Extracts entities from text using neural matching to NounTypes. **Example:** ```typescript const entities = await nlp.extract( - "John Smith from Microsoft visited New York on Jan 15", - { - types: [NounType.Person, NounType.Organization, NounType.Location], - confidence: 0.7, - neuralMatching: true - } + "John Smith from Microsoft visited New York on Jan 15", + { + types: [NounType.Person, NounType.Organization, NounType.Location], + confidence: 0.7, + neuralMatching: true + } ) // Returns: // [ -// { text: "John Smith", type: NounType.Person, confidence: 0.92 }, -// { text: "Microsoft", type: NounType.Organization, confidence: 0.88 }, -// { text: "New York", type: NounType.Location, confidence: 0.85 } +// { text: "John Smith", type: NounType.Person, confidence: 0.92 }, +// { text: "Microsoft", type: NounType.Organization, confidence: 0.88 }, +// { text: "New York", type: NounType.Location, confidence: 0.85 } // ] ``` @@ -1531,17 +1531,17 @@ Analyzes text sentiment. **Example:** ```typescript const sentiment = await nlp.sentiment( - "The product quality is excellent but the price is too high", - { - granularity: 'aspect', - aspects: ['quality', 'price'] - } + "The product quality is excellent but the price is too high", + { + granularity: 'aspect', + aspects: ['quality', 'price'] + } ) // Returns: // overall: { score: 0.2, label: 'mixed' } // aspects: { -// quality: { score: 0.9, magnitude: 0.8 }, -// price: { score: -0.7, magnitude: 0.7 } +// quality: { score: 0.9, magnitude: 0.8 }, +// price: { score: -0.7, magnitude: 0.7 } // } ``` @@ -1559,16 +1559,16 @@ Sets data source. **Example:** ```typescript async function* dataGenerator() { - for (let i = 0; i < 100; i++) { - yield { id: i, data: `Item ${i}` } - } + for (let i = 0; i < 100; i++) { + yield { id: i, data: `Item ${i}` } + } } brain.stream() - .source(dataGenerator()) - .map(item => item.data) - .sink(console.log) - .run() + .source(dataGenerator()) + .map(item => item.data) + .sink(console.log) + .run() ``` --- @@ -1616,14 +1616,14 @@ Sinks data to Brainy. **Example:** ```typescript brain.stream() - .source(dataSource) - .map(transform) - .batch(100) - .toBrainy({ - type: NounType.Document, - metadata: { source: 'stream' } - }) - .run() + .source(dataSource) + .map(transform) + .batch(100) + .toBrainy({ + type: NounType.Document, + metadata: { source: 'stream' } + }) + .run() ``` --- @@ -1665,56 +1665,56 @@ Executes the pipeline. ### Core Interfaces -✨ *Updated in v4.3.0 - Added confidence/weight to Entity, flattened Result fields* +✨ *Updated in Added confidence/weight to Entity, flattened Result fields* ```typescript interface Entity { - id: string - vector: Vector - type: NounType - data?: any - metadata?: T - service?: string - createdAt: number - updatedAt?: number - confidence?: number // NEW: Type classification confidence (0-1) - weight?: number // NEW: Entity importance/salience (0-1) + id: string + vector: Vector + type: NounType + data?: any + metadata?: T + service?: string + createdAt: number + updatedAt?: number + confidence?: number // NEW: Type classification confidence (0-1) + weight?: number // NEW: Entity importance/salience (0-1) } interface Relation { - id: string - from: string - to: string - type: VerbType - weight?: number - confidence?: number // Relationship confidence - metadata?: T - evidence?: RelationEvidence // Why this relationship exists - service?: string - createdAt: number + id: string + from: string + to: string + type: VerbType + weight?: number + confidence?: number // Relationship confidence + metadata?: T + evidence?: RelationEvidence // Why this relationship exists + service?: string + createdAt: number } interface Result { - // Search metadata - id: string - score: number + // Search metadata + id: string + score: number - // NEW: Flattened entity fields for convenience - type?: NounType - metadata?: T - data?: any - confidence?: number - weight?: number + // NEW: Flattened entity fields for convenience + type?: NounType + metadata?: T + data?: any + confidence?: number + weight?: number - // Full entity (backward compatible) - entity: Entity + // Full entity (backward compatible) + entity: Entity - // Score explanation - explanation?: ScoreExplanation + // Score explanation + explanation?: ScoreExplanation } ``` -**Key Changes in v4.3.0:** +**Key Changes:** - ✅ `Entity` now exposes `confidence` and `weight` - ✅ `Result` flattens commonly-used entity fields to top level - ✅ Direct access: `result.metadata` instead of `result.entity.metadata` @@ -1767,7 +1767,7 @@ await brain.addMany({ items: documents }) // ❌ Bad - individual adds in loop for (const doc of documents) { - await brain.add(doc) // Slow! + await brain.add(doc) // Slow! } ``` @@ -1775,18 +1775,18 @@ for (const doc of documents) { ```typescript // For high-speed ingestion await brain.add({ - data: content, - type: NounType.Document, - writeOnly: true // Skip validation + data: content, + type: NounType.Document, + writeOnly: true // Skip validation }) ``` ### 5. Clean Up Resources ```typescript try { - // Your operations + // Your operations } finally { - await brain.close() // Always close + await brain.close() // Always close } ``` @@ -1819,22 +1819,22 @@ All methods validate parameters and throw descriptive errors: ```typescript try { - await brain.add({ data: '', type: NounType.Document }) + await brain.add({ data: '', type: NounType.Document }) } catch (error) { - // Error: "must provide either data or vector" + // Error: "must provide either data or vector" } try { - await brain.find({ limit: -1 }) + await brain.find({ limit: -1 }) } catch (error) { - // Error: "limit must be non-negative" + // Error: "limit must be non-negative" } try { - await brain.update({ id: 'xyz', metadata: null, merge: false }) + await brain.update({ id: 'xyz', metadata: null, merge: false }) } catch (error) { - // Error: "must specify at least one field to update" - // Note: Use metadata: {} to clear, not null + // Error: "must specify at least one field to update" + // Note: Use metadata: {} to clear, not null } ``` diff --git a/docs/BATCHING.md b/docs/BATCHING.md index acc11fe6..8701b508 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -1,23 +1,22 @@ -# Batch Operations API v5.12.0 - +# Batch Operations API > **Enterprise Production-Ready** | Zero N+1 Query Patterns | 90%+ Performance Improvement ## Overview -Brainy v5.12.0 introduces comprehensive batch operations at the storage layer, eliminating N+1 query patterns and dramatically improving performance for VFS operations, relationship queries, and entity retrieval on cloud storage. +Brainy introduces comprehensive batch operations at the storage layer, eliminating N+1 query patterns and dramatically improving performance for VFS operations, relationship queries, and entity retrieval on cloud storage. ### Problem Solved -**Before v5.12.0:** +**Before optimization:** - VFS `getTreeStructure()` on cloud storage: **12.7 seconds** for directory with 12 files - N+1 query pattern: 1 directory query + N individual file queries (22 sequential calls × 580ms latency) -**After v5.12.0:** +**After optimization:** - VFS `getTreeStructure()`: **<1 second** for 12 files (90%+ improvement) - 2-3 batched calls instead of 22 sequential calls - Native cloud storage batch APIs for maximum throughput -**IMPORTANT:** The v5.12.0 batch optimizations apply **ONLY to `getTreeStructure()`**, not to `readFile()` or individual `get()` operations. See v6.0.0 changes for comprehensive storage path optimizations. +**IMPORTANT:** The batch optimizations apply **ONLY to `getTreeStructure()`**, not to `readFile()` or individual `get()` operations. See changes for comprehensive storage path optimizations. --- @@ -56,7 +55,7 @@ results.size // → 3 (number of found entities) ### 2. `storage.getNounMetadataBatch(ids)` -Batch metadata retrieval with direct O(1) path construction (v6.0.0+). +Batch metadata retrieval with direct O(1) path construction. ```typescript const storage = brain.storage as BaseStorage @@ -65,8 +64,8 @@ const ids = ['id1', 'id2', 'id3'] const metadataMap: Map = await storage.getNounMetadataBatch(ids) for (const [id, metadata] of metadataMap) { - console.log(metadata.noun) // Type: 'document', 'person', etc. - console.log(metadata.data) // Entity data + console.log(metadata.noun) // Type: 'document', 'person', etc. + console.log(metadata.data) // Entity data } ``` @@ -92,22 +91,22 @@ const storage = brain.storage as BaseStorage // Get all relationships from multiple sources const results: Map = await storage.getVerbsBySourceBatch([ - 'person1', - 'person2' + 'person1', + 'person2' ]) // Filter by verb type const createsResults = await storage.getVerbsBySourceBatch( - ['person1', 'person2'], - 'creates' + ['person1', 'person2'], + 'creates' ) // Process results for (const [sourceId, verbs] of results) { - console.log(`${sourceId} has ${verbs.length} relationships`) - verbs.forEach(verb => { - console.log(` → ${verb.verb} → ${verb.targetId}`) - }) + console.log(`${sourceId} has ${verbs.length} relationships`) + verbs.forEach(verb => { + console.log(` → ${verb.verb} → ${verb.targetId}`) + }) } ``` @@ -130,8 +129,8 @@ COW-aware batch path resolution with branch inheritance. const storage = brain.storage as BaseStorage const paths = [ - 'entities/nouns/{shard}/id1/metadata.json', - 'entities/nouns/{shard}/id2/metadata.json' + 'entities/nouns/{shard}/id1/metadata.json', + 'entities/nouns/{shard}/id2/metadata.json' ] // Resolves to: branches/{branch}/entities/nouns/{shard}/{id}/metadata.json @@ -160,9 +159,9 @@ const results = await gcsStorage.readBatch(paths) // Configuration gcsStorage.getBatchConfig() // → { -// maxBatchSize: 1000, -// maxConcurrent: 100, -// operationsPerSecond: 1000 +// maxBatchSize: 1000, +// maxConcurrent: 100, +// operationsPerSecond: 1000 // } ``` @@ -185,9 +184,9 @@ const results = await s3Storage.readBatch(paths) // Configuration s3Storage.getBatchConfig() // → { -// maxBatchSize: 1000, -// maxConcurrent: 150, -// operationsPerSecond: 5000 +// maxBatchSize: 1000, +// maxConcurrent: 150, +// operationsPerSecond: 5000 // } ``` @@ -208,9 +207,9 @@ const results = await r2Storage.readBatch(paths) // Configuration r2Storage.getBatchConfig() // → { -// maxBatchSize: 1000, -// maxConcurrent: 150, -// operationsPerSecond: 6000 +// maxBatchSize: 1000, +// maxConcurrent: 150, +// operationsPerSecond: 6000 // } ``` @@ -231,9 +230,9 @@ const results = await azureStorage.readBatch(paths) // Configuration azureStorage.getBatchConfig() // → { -// maxBatchSize: 1000, -// maxConcurrent: 100, -// operationsPerSecond: 3000 +// maxBatchSize: 1000, +// maxConcurrent: 100, +// operationsPerSecond: 3000 // } ``` @@ -254,7 +253,7 @@ VFS operations automatically use batch APIs for maximum performance. // OLD: Sequential N+1 pattern (12.7 seconds for 12 files) const tree = await brain.vfs.getTreeStructure('/my-dir') -// NEW v5.12.0: Parallel breadth-first with batching (<1 second) +// NEW Parallel breadth-first with batching (<1 second) // ✅ PathResolver.getChildren() uses brain.batchGet() internally // ✅ Parallel traversal of directories at same tree level // ✅ 2-3 batched calls instead of 22 sequential calls @@ -264,16 +263,16 @@ const tree = await brain.vfs.getTreeStructure('/my-dir') ``` VFS.getTreeStructure() - ↓ PARALLEL (breadth-first traversal) - → PathResolver.getChildren() [all dirs at level processed in parallel] - ↓ BATCHED - → brain.batchGet(childIds) [1 call instead of N] - ↓ BATCHED - → storage.getNounMetadataBatch(ids) [1 call instead of N] - ↓ ADAPTER-SPECIFIC - → GCS: readBatch() with 100 concurrent downloads - → S3: readBatch() with 150 concurrent downloads - → Memory: Promise.all() parallel reads + ↓ PARALLEL (breadth-first traversal) + → PathResolver.getChildren() [all dirs at level processed in parallel] + ↓ BATCHED + → brain.batchGet(childIds) [1 call instead of N] + ↓ BATCHED + → storage.getNounMetadataBatch(ids) [1 call instead of N] + ↓ ADAPTER-SPECIFIC + → GCS: readBatch() with 100 concurrent downloads + → S3: readBatch() with 150 concurrent downloads + → Memory: Promise.all() parallel reads ``` **Performance Gains:** @@ -285,11 +284,11 @@ VFS.getTreeStructure() ## Advanced Features Compatibility -### ✅ ID-First Storage Architecture (v6.0.0+) +### ✅ ID-First Storage Architecture All batch operations use direct ID-first paths - no type lookup needed! -**NEW v6.0.0 Path Structure:** +**ID-First Path Structure:** ``` entities/nouns/{SHARD}/{ID}/metadata.json entities/verbs/{SHARD}/{ID}/metadata.json @@ -299,7 +298,7 @@ entities/verbs/{SHARD}/{ID}/metadata.json ```typescript // Every ID maps directly to exactly ONE path - 40x faster! const id = 'abc-123' -const shard = getShardIdFromUuid(id) // → 'ab' (first 2 hex chars) +const shard = getShardIdFromUuid(id) // → 'ab' (first 2 hex chars) const path = `entities/nouns/${shard}/${id}/metadata.json` // No type cache needed! @@ -343,7 +342,7 @@ const fork = await brain.fork('experiment') // Batch operations are isolated await brain.batchGet([id1, id2]) // → Reads from: branches/main/... -await fork.batchGet([id1, id2]) // → Reads from: branches/experiment/... +await fork.batchGet([id1, id2]) // → Reads from: branches/experiment/... ``` **Inheritance:** @@ -391,7 +390,7 @@ Historical queries use `HistoricalStorageAdapter` which wraps batch operations t ### VFS Operations (12 Files) -| Storage | Before v5.12.0 | After v5.12.0 | Improvement | +| Storage | Before optimization | After optimization | Improvement | |---------|---------------|---------------|-------------| | **GCS** | 12.7s | <1s | **92% faster** | | **S3** | 13.2s | <1s | **92% faster** | @@ -430,8 +429,8 @@ Batch operations gracefully handle missing or invalid entities: ```typescript const validId = 'abc-123-...' const invalidIds = [ - '11111111-1111-1111-1111-111111111111', - '22222222-2222-2222-2222-222222222222' + '11111111-1111-1111-1111-111111111111', + '22222222-2222-2222-2222-222222222222' ] const results = await brain.batchGet([validId, ...invalidIds]) @@ -471,8 +470,8 @@ results.size // → 1 (deduplicated automatically) ```typescript const entities = [] for (const id of ids) { - const entity = await brain.get(id) - if (entity) entities.push(entity) + const entity = await brain.get(id) + if (entity) entities.push(entity) } ``` @@ -492,8 +491,8 @@ const entities = Array.from(results.values()) ```typescript const allVerbs = [] for (const sourceId of sourceIds) { - const verbs = await brain.getRelations({ from: sourceId }) - allVerbs.push(...verbs) + const verbs = await brain.getRelations({ from: sourceId }) + allVerbs.push(...verbs) } ``` @@ -504,7 +503,7 @@ const results = await storage.getVerbsBySourceBatch(sourceIds) const allVerbs = [] for (const verbs of results.values()) { - allVerbs.push(...verbs) + allVerbs.push(...verbs) } ``` @@ -522,7 +521,7 @@ const results = await brain.batchGet(ids) // ❌ BAD: Individual gets in loop for (const id of ids) { - await brain.get(id) + await brain.get(id) } ``` @@ -556,13 +555,13 @@ const results = await brain.batchGet(ids) // Check results for (const id of ids) { - if (results.has(id)) { - // Entity exists - const entity = results.get(id) - } else { - // Entity missing (not an error) - console.log(`Entity ${id} not found`) - } + if (results.has(id)) { + // Entity exists + const entity = results.get(id) + } else { + // Entity missing (not an error) + console.log(`Entity ${id} not found`) + } } ``` @@ -597,15 +596,15 @@ npx vitest run tests/integration/storage-batch-operations.test.ts ``` User Code (brain.batchGet) - ↓ + ↓ High-Level API (src/brainy.ts) - ↓ + ↓ Storage Layer (src/storage/baseStorage.ts) - ↓ + ↓ COW Layer (readBatchWithInheritance) - ↓ + ↓ Adapter Layer (readBatchFromAdapter) - ↓ + ↓ Cloud Adapter (GCS/S3/Azure native batch APIs) ``` @@ -616,11 +615,11 @@ If an adapter doesn't implement `readBatch()`, the system automatically falls ba ```typescript // BaseStorage.readBatchFromAdapter() if (typeof selfWithBatch.readBatch === 'function') { - // Use native batch API - return await selfWithBatch.readBatch(resolvedPaths) + // Use native batch API + return await selfWithBatch.readBatch(resolvedPaths) } else { - // Automatic parallel fallback - return await Promise.all(resolvedPaths.map(path => this.read(path))) + // Automatic parallel fallback + return await Promise.all(resolvedPaths.map(path => this.read(path))) } ``` @@ -658,7 +657,7 @@ if (typeof selfWithBatch.readBatch === 'function') { - Zero N+1 query patterns **Compatibility:** -- ✅ ID-first storage (v6.0.0+) +- ✅ ID-first storage - ✅ Sharding (256 shards) - ✅ COW (branch isolation, inheritance) - ✅ fork() and checkout() diff --git a/docs/CREATING-AUGMENTATIONS.md b/docs/CREATING-AUGMENTATIONS.md index 1c7fd04a..2983472c 100644 --- a/docs/CREATING-AUGMENTATIONS.md +++ b/docs/CREATING-AUGMENTATIONS.md @@ -1,6 +1,6 @@ # Creating Augmentations for Brainy -> **Updated for v4.0.0** - Includes metadata structure changes and type system improvements +> **Updated** - Includes metadata structure changes and type system improvements ## The BrainyAugmentation Interface @@ -8,38 +8,38 @@ Every augmentation implements this simple yet powerful interface: ```typescript interface BrainyAugmentation { - // Identification - name: string // Unique name for your augmentation + // Identification + name: string // Unique name for your augmentation - // Execution control - timing: 'before' | 'after' | 'around' | 'replace' // When to execute - operations: string[] // Which operations to intercept - priority: number // Execution order (higher = first) + // Execution control + timing: 'before' | 'after' | 'around' | 'replace' // When to execute + operations: string[] // Which operations to intercept + priority: number // Execution order (higher = first) - // Lifecycle methods - initialize(context: AugmentationContext): Promise - execute(operation: string, params: any, next: () => Promise): Promise - shutdown?(): Promise // Optional cleanup + // Lifecycle methods + initialize(context: AugmentationContext): Promise + execute(operation: string, params: any, next: () => Promise): Promise + shutdown?(): Promise // Optional cleanup } ``` -## v4.0.0 Breaking Changes for Augmentation Developers +## Breaking Changes for Augmentation Developers ### 1. Metadata Structure Separation -v4.0.0 introduces strict metadata/vector separation for billion-scale performance: +Brainy introduces strict metadata/vector separation for billion-scale performance: ```typescript -// ✅ v4.0.0: Metadata has required type field +// ✅ Metadata has required type field interface NounMetadata { - noun: NounType // Required! Must be a valid noun type - [key: string]: any // Your custom metadata + noun: NounType // Required! Must be a valid noun type + [key: string]: any // Your custom metadata } interface VerbMetadata { - verb: VerbType // Required! Must be a valid verb type - sourceId: string - targetId: string - [key: string]: any + verb: VerbType // Required! Must be a valid verb type + sourceId: string + targetId: string + [key: string]: any } ``` @@ -61,7 +61,7 @@ The verb relationship field changed from `type` to `verb`: // ❌ v3.x verb.type === 'relatedTo' -// ✅ v4.0.0 +// ✅ Current verb.verb === 'relatedTo' ``` @@ -69,7 +69,7 @@ verb.verb === 'relatedTo' Storage augmentations are special - they provide the storage backend for Brainy. -### Important: v4.0.0 Storage Requirements +### Important: Storage Requirements Your storage adapter MUST: 1. **Wrap metadata** with required `noun`/`verb` fields @@ -81,67 +81,67 @@ import { StorageAugmentation } from 'brainy/augmentations' import { BaseStorageAdapter, HNSWNoun, HNSWNounWithMetadata, NounMetadata } from 'brainy' export class MyCustomStorage extends BaseStorageAdapter { - // Internal method: Returns pure structure - async _getNoun(id: string): Promise { - const data = await this.fetchFromDatabase(id) - return data ? { - id: data.id, - vector: data.vector, - nounType: data.type - } : null - } + // Internal method: Returns pure structure + async _getNoun(id: string): Promise { + const data = await this.fetchFromDatabase(id) + return data ? { + id: data.id, + vector: data.vector, + nounType: data.type + } : null + } - // Public method: Returns WithMetadata structure - async getNoun(id: string): Promise { - const noun = await this._getNoun(id) - if (!noun) return null + // Public method: Returns WithMetadata structure + async getNoun(id: string): Promise { + const noun = await this._getNoun(id) + if (!noun) return null - // Fetch metadata separately (v4.0.0 pattern) - const metadata = await this.getNounMetadata(id) + // Fetch metadata separately + const metadata = await this.getNounMetadata(id) - return { - ...noun, - metadata: metadata || { noun: noun.nounType || 'thing' } - } - } + return { + ...noun, + metadata: metadata || { noun: noun.nounType || 'thing' } + } + } - // CRITICAL: Always save with proper metadata structure - async saveNoun(noun: HNSWNoun, metadata?: NounMetadata): Promise { - // Validate metadata has required 'noun' field - if (!metadata?.noun) { - throw new Error('v4.0.0: NounMetadata requires "noun" field') - } + // CRITICAL: Always save with proper metadata structure + async saveNoun(noun: HNSWNoun, metadata?: NounMetadata): Promise { + // Validate metadata has required 'noun' field + if (!metadata?.noun) { + throw new Error('NounMetadata requires "noun" field') + } - await this.database.save({ - id: noun.id, - vector: noun.vector, - nounType: noun.nounType, - metadata: metadata // Stored separately in v4.0.0 - }) - } + await this.database.save({ + id: noun.id, + vector: noun.vector, + nounType: noun.nounType, + metadata: metadata // Stored separately + }) + } } export class MyStorageAugmentation extends StorageAugmentation { - private config: MyStorageConfig + private config: MyStorageConfig - constructor(config: MyStorageConfig) { - super() - this.name = 'my-custom-storage' - this.config = config - } + constructor(config: MyStorageConfig) { + super() + this.name = 'my-custom-storage' + this.config = config + } - // Called during storage resolution phase - async provideStorage(): Promise { - const storage = new MyCustomStorage(this.config) - this.storageAdapter = storage - return storage - } + // Called during storage resolution phase + async provideStorage(): Promise { + const storage = new MyCustomStorage(this.config) + this.storageAdapter = storage + return storage + } - // Called during augmentation initialization - protected async onInitialize(): Promise { - await this.storageAdapter!.init() - this.log(`Custom storage initialized`) - } + // Called during augmentation initialization + protected async onInitialize(): Promise { + await this.storageAdapter!.init() + this.log(`Custom storage initialized`) + } } ``` @@ -151,9 +151,9 @@ export class MyStorageAugmentation extends StorageAugmentation { // Register before brain.init() const brain = new Brainy() brain.augmentations.register(new MyStorageAugmentation({ - connectionString: 'redis://localhost:6379' + connectionString: 'redis://localhost:6379' })) -await brain.init() // Will use your storage! +await brain.init() // Will use your storage! ``` ## Creating a Feature Augmentation @@ -164,43 +164,43 @@ Here's a complete example of a caching augmentation: import { BaseAugmentation, BrainyAugmentation } from 'brainy/augmentations' export class CachingAugmentation extends BaseAugmentation { - private cache = new Map() - - constructor() { - super() - this.name = 'smart-cache' - this.timing = 'around' // Wrap operations - this.operations = ['search'] // Only cache searches - this.priority = 50 // Mid-priority - } - - async execute(operation: string, params: any, next: () => Promise): Promise { - if (operation === 'search') { - // Check cache - const cacheKey = JSON.stringify(params) - if (this.cache.has(cacheKey)) { - this.log('Cache hit!') - return this.cache.get(cacheKey) - } - - // Execute and cache - const result = await next() - this.cache.set(cacheKey, result) - return result - } - - // Pass through other operations - return next() - } - - protected async onInitialize(): Promise { - this.log('Cache initialized') - } - - async shutdown(): Promise { - this.cache.clear() - await super.shutdown() - } + private cache = new Map() + + constructor() { + super() + this.name = 'smart-cache' + this.timing = 'around' // Wrap operations + this.operations = ['search'] // Only cache searches + this.priority = 50 // Mid-priority + } + + async execute(operation: string, params: any, next: () => Promise): Promise { + if (operation === 'search') { + // Check cache + const cacheKey = JSON.stringify(params) + if (this.cache.has(cacheKey)) { + this.log('Cache hit!') + return this.cache.get(cacheKey) + } + + // Execute and cache + const result = await next() + this.cache.set(cacheKey, result) + return result + } + + // Pass through other operations + return next() + } + + protected async onInitialize(): Promise { + this.log('Cache initialized') + } + + async shutdown(): Promise { + this.cache.clear() + await super.shutdown() + } } ``` @@ -210,20 +210,20 @@ export class CachingAugmentation extends BaseAugmentation { ```typescript timing = 'before' async execute(op, params, next) { - // Validate/transform input - const validated = await validate(params) - return next(validated) // Pass modified params + // Validate/transform input + const validated = await validate(params) + return next(validated) // Pass modified params } ``` -### 2. `after` - Post-processing +### 2. `after` - Post-processing ```typescript timing = 'after' async execute(op, params, next) { - const result = await next() - // Log, analyze, or modify result - console.log(`Operation ${op} returned:`, result) - return result + const result = await next() + // Log, analyze, or modify result + console.log(`Operation ${op} returned:`, result) + return result } ``` @@ -231,15 +231,15 @@ async execute(op, params, next) { ```typescript timing = 'around' async execute(op, params, next) { - console.log('Starting', op) - try { - const result = await next() - console.log('Success', op) - return result - } catch (error) { - console.log('Failed', op, error) - throw error - } + console.log('Starting', op) + try { + const result = await next() + console.log('Success', op) + return result + } catch (error) { + console.log('Failed', op, error) + throw error + } } ``` @@ -247,8 +247,8 @@ async execute(op, params, next) { ```typescript timing = 'replace' async execute(op, params, next) { - // Don't call next() - replace entirely! - return myCustomImplementation(params) + // Don't call next() - replace entirely! + return myCustomImplementation(params) } ``` @@ -266,10 +266,10 @@ Common operations in Brainy: ```typescript interface AugmentationContext { - brain: Brainy // The brain instance - storage: StorageAdapter // Storage backend - config: BrainyConfig // Configuration - log: (message: string, level?: 'info' | 'warn' | 'error') => void + brain: Brainy // The brain instance + storage: StorageAdapter // Storage backend + config: BrainyConfig // Configuration + log: (message: string, level?: 'info' | 'warn' | 'error') => void } ``` @@ -278,50 +278,50 @@ interface AugmentationContext { ### 1. Redis Storage Augmentation ```typescript export class RedisStorageAugmentation extends StorageAugmentation { - async provideStorage(): Promise { - return new RedisAdapter({ - host: 'localhost', - port: 6379, - // Implement full StorageAdapter interface - }) - } + async provideStorage(): Promise { + return new RedisAdapter({ + host: 'localhost', + port: 6379, + // Implement full StorageAdapter interface + }) + } } ``` ### 2. Audit Trail Augmentation ```typescript export class AuditAugmentation extends BaseAugmentation { - timing = 'after' - operations = ['add', 'update', 'delete'] - - async execute(op, params, next) { - const result = await next() - - // Log to audit trail - await this.logAudit({ - operation: op, - params, - result, - timestamp: new Date(), - user: this.context.config.currentUser - }) - - return result - } + timing = 'after' + operations = ['add', 'update', 'delete'] + + async execute(op, params, next) { + const result = await next() + + // Log to audit trail + await this.logAudit({ + operation: op, + params, + result, + timestamp: new Date(), + user: this.context.config.currentUser + }) + + return result + } } ``` ### 3. Rate Limiting Augmentation ```typescript export class RateLimitAugmentation extends BaseAugmentation { - timing = 'before' - operations = ['search'] - private limiter = new RateLimiter({ rps: 100 }) - - async execute(op, params, next) { - await this.limiter.acquire() // Wait if rate limited - return next() - } + timing = 'before' + operations = ['search'] + private limiter = new RateLimiter({ rps: 100 }) + + async execute(op, params, next) { + await this.limiter.acquire() // Wait if rate limited + return next() + } } ``` @@ -332,12 +332,12 @@ Future capability for premium augmentations: ```typescript // package.json { - "name": "@brain-cloud/redis-storage", - "brainy": { - "type": "augmentation", - "category": "storage", - "premium": true - } + "name": "@brain-cloud/redis-storage", + "brainy": { + "type": "augmentation", + "category": "storage", + "premium": true + } } // Users can install via: @@ -356,43 +356,43 @@ Future capability for premium augmentations: 6. **Log appropriately** - Use context.log() for consistent output 7. **Document your augmentation** - Include examples -### v4.0.0 Specific Best Practices +### Specific Best Practices 8. **Always include `noun` field** when creating/modifying NounMetadata: - ```typescript - const metadata: NounMetadata = { - noun: 'thing', // REQUIRED! - yourField: 'value' - } - ``` + ```typescript + const metadata: NounMetadata = { + noun: 'thing', // REQUIRED! + yourField: 'value' + } + ``` 9. **Use `verb` property** not `type` when working with relationships: - ```typescript - // ✅ Correct - if (verb.verb === 'relatedTo') { ... } + ```typescript + // ✅ Correct + if (verb.verb === 'relatedTo') { ... } - // ❌ Wrong (v3.x pattern) - if (verb.type === 'relatedTo') { ... } - ``` + // ❌ Wrong (v3.x pattern) + if (verb.type === 'relatedTo') { ... } + ``` 10. **Access metadata correctly** from storage: - ```typescript - // ✅ Correct - metadata is already structured - const nounType = noun.metadata.noun + ```typescript + // ✅ Correct - metadata is already structured + const nounType = noun.metadata.noun - // ⚠️ Fallback pattern for robustness - const nounType = noun.metadata?.noun || 'thing' - ``` + // ⚠️ Fallback pattern for robustness + const nounType = noun.metadata?.noun || 'thing' + ``` 11. **Respect the two-file storage pattern** - Don't mix vector and metadata operations: - ```typescript - // ✅ Good - Separate concerns - await storage.saveNoun(noun) - await storage.saveMetadata(noun.id, metadata) + ```typescript + // ✅ Good - Separate concerns + await storage.saveNoun(noun) + await storage.saveMetadata(noun.id, metadata) - // ❌ Bad - Mixing concerns - await storage.saveNounWithEverything(combinedData) - ``` + // ❌ Bad - Mixing concerns + await storage.saveNounWithEverything(combinedData) + ``` ## Testing Your Augmentation @@ -401,23 +401,23 @@ import { Brainy } from 'brainy' import { MyAugmentation } from './my-augmentation' describe('MyAugmentation', () => { - let brain: Brainy - - beforeEach(async () => { - brain = new Brainy() - brain.augmentations.register(new MyAugmentation()) - await brain.init() - }) - - afterEach(async () => { - await brain.destroy() - }) - - it('should enhance searches', async () => { - // Test your augmentation's effect - const results = await brain.search('test') - expect(results).toHaveProperty('enhanced', true) - }) + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy() + brain.augmentations.register(new MyAugmentation()) + await brain.init() + }) + + afterEach(async () => { + await brain.destroy() + }) + + it('should enhance searches', async () => { + // Test your augmentation's effect + const results = await brain.search('test') + expect(results).toHaveProperty('enhanced', true) + }) }) ``` diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md index 3436d922..91914b15 100644 --- a/docs/DEVELOPER_LEARNING_PATH.md +++ b/docs/DEVELOPER_LEARNING_PATH.md @@ -613,7 +613,7 @@ vfsFiles.forEach(f => { }) // 3. VFS FILTERING IN KNOWLEDGE QUERIES -console.log('\n🔍 Understanding VFS filtering (v4.4.0)...\n') +console.log('\n🔍 Understanding VFS filtering...\n') // Create some knowledge entities const conceptId = await brain.add({ @@ -758,7 +758,7 @@ await brain.close() ### Key Concepts -#### 1. **VFS Filtering Architecture (v4.4.0)** +#### 1. **VFS Filtering Architecture** ```typescript // 🎯 DEFAULT BEHAVIOR: Clean Separation diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 56ecbb00..bb9745db 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -13,8 +13,7 @@ Brainy's `find()` method is the most advanced query system in any vector databas - **Data Structure**: Multi-layer graph with 16 connections per node - **Use Cases**: "Find similar documents", "Content like this" -### 2. Text Intelligence (Word Index) - v7.7.0 -- **Purpose**: Keyword/exact text matching +### 2. Text Intelligence (Word Index) - **Purpose**: Keyword/exact text matching - **Algorithm**: Inverted word index with FNV-1a hashing - **Performance**: O(log C) where C = chunks (~50 values each) - **Data Structure**: `__words__ → hash → Roaring Bitmap of entity IDs` @@ -28,7 +27,7 @@ Brainy's `find()` method is the most advanced query system in any vector databas - **Data Structure**: `Map>` + sorted value arrays - **Use Cases**: "Documents from 2023", "Status equals active" -### 4. Graph Intelligence (Adjacency Maps) +### 4. Graph Intelligence (Adjacency Maps) - **Purpose**: Relationship traversal and connection analysis - **Algorithm**: Pure O(1) neighbor lookups via Map operations - **Performance**: O(1) per hop, ~0.1ms typical @@ -54,29 +53,29 @@ await brain.find("documents by Smith published after 2020 with high citations") ```typescript // Direct query objects with full control await brain.find({ - // Vector search - query: "machine learning research", - - // Metadata filters - where: { - publishDate: { greaterThan: 2020 }, - citations: { between: [50, 1000] }, - status: "published" - }, - - // Type constraints - type: NounType.Document, - - // Graph traversal - connected: { - to: "mit-ai-lab", - via: VerbType.AffiliatedWith, - depth: 2 - }, - - // Control options - limit: 20, - explain: true // Get scoring breakdown + // Vector search + query: "machine learning research", + + // Metadata filters + where: { + publishDate: { greaterThan: 2020 }, + citations: { between: [50, 1000] }, + status: "published" + }, + + // Type constraints + type: NounType.Document, + + // Graph traversal + connected: { + to: "mit-ai-lab", + via: VerbType.AffiliatedWith, + depth: 2 + }, + + // Control options + limit: 20, + explain: true // Get scoring breakdown }) ``` @@ -84,15 +83,15 @@ await brain.find({ ```typescript // Find entities similar to a specific item await brain.find({ - near: { - id: "doc-123", - threshold: 0.8 // Minimum similarity - }, - type: NounType.Document + near: { + id: "doc-123", + threshold: 0.8 // Minimum similarity + }, + type: NounType.Document }) ``` -### 4. Hybrid Search (v7.7.0+) +### 4. Hybrid Search ```typescript // Zero-config hybrid: automatically combines text + semantic search await brain.find({ query: "David Smith" }) @@ -113,7 +112,7 @@ await brain.find({ query: "search term", hybridAlpha: 0.3 }) - Medium queries (3-4 words): alpha = 0.5 (balanced) - Long queries (5+ words): alpha = 0.7 (favor semantic matching) -### 5. Match Visibility (v7.8.0) +### 5. Match Visibility Search results include match details showing what matched: @@ -121,10 +120,10 @@ Search results include match details showing what matched: const results = await brain.find({ query: 'david the warrior' }) // Each result has: -results[0].textMatches // ["david", "warrior"] - exact words found -results[0].textScore // 0.25 - text match quality (0-1) -results[0].semanticScore // 0.87 - semantic similarity (0-1) -results[0].matchSource // 'both' | 'text' | 'semantic' +results[0].textMatches // ["david", "warrior"] - exact words found +results[0].textScore // 0.25 - text match quality (0-1) +results[0].semanticScore // 0.87 - semantic similarity (0-1) +results[0].matchSource // 'both' | 'text' | 'semantic' ``` Use this for: @@ -132,22 +131,22 @@ Use this for: - **Explaining** why a result was found (matchSource) - **Debugging** search behavior (separate scores) -### 6. Semantic Highlighting (v7.8.0) +### 6. Semantic Highlighting Highlight which concepts/words in text matched your query: ```typescript // Find semantically similar words + exact matches const highlights = await brain.highlight({ - query: "david the warrior", - text: "David Smith is a brave fighter who battles dragons" + query: "david the warrior", + text: "David Smith is a brave fighter who battles dragons" }) // Returns: // [ -// { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, -// { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, -// { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } +// { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, +// { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, +// { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } // ] ``` @@ -162,11 +161,11 @@ const highlights = await brain.highlight({ ```typescript // Highlight search results with different styles function highlightResult(text: string, highlights: Highlight[]) { - return highlights.map(h => ({ - text: h.text, - position: h.position, - style: h.matchType === 'text' ? 'strong' : 'emphasis' // Different UI styles - })) + return highlights.map(h => ({ + text: h.text, + position: h.position, + style: h.matchType === 'text' ? 'strong' : 'emphasis' // Different UI styles + })) } ``` @@ -182,7 +181,7 @@ function highlightResult(text: string, highlights: Highlight[]) { // Memory: ~40 bytes per unique field-value combination ``` -#### Sorted Index (Range Queries) +#### Sorted Index (Range Queries) ```typescript // Query: {publishDate: {greaterThan: 2020}} // Index: sortedIndices.get("publishDate") → [[2019, Set], [2020, Set], [2021, Set]] @@ -208,7 +207,7 @@ function highlightResult(text: string, highlights: Highlight[]) { // Query: {query: "machine learning"} // Process: // 1. Embed query text → 384-dimensional vector -// 2. Start at top layer (entry point) +// 2. Start at top layer (entry point) // 3. Greedy search for nearest neighbor at each layer // 4. Move down layers for progressively finer search // 5. Return k nearest neighbors with similarity scores @@ -221,10 +220,10 @@ function highlightResult(text: string, highlights: Highlight[]) { #### O(1) Neighbor Lookups ```typescript // Query: {connected: {to: "entity-123"}} -// Index lookup: -// - Outgoing: sourceIndex.get("entity-123") → Set -// - Incoming: targetIndex.get("entity-123") → Set -// - Both directions: union of both Sets +// Index lookup: +// - Outgoing: sourceIndex.get("entity-123") → Set +// - Incoming: targetIndex.get("entity-123") → Set +// - Both directions: union of both Sets // Performance: O(1) per hop, no matter the graph size // Memory: ~24 bytes per relationship (source + target + metadata) ``` @@ -234,19 +233,19 @@ function highlightResult(text: string, highlights: Highlight[]) { ### Phase 1: Query Parsing ```typescript if (typeof query === 'string') { - // Natural Language Processing - const nlpParams = await this.parseNaturalQuery(query) - - // Type-aware parsing: - // 1. Detect NounType using pre-embedded type vectors - // 2. Get type-specific fields from real data patterns - // 3. Semantic field matching with type affinity boosting - // 4. Validate field-type compatibility - // 5. Generate optimized query plan - - params = nlpParams + // Natural Language Processing + const nlpParams = await this.parseNaturalQuery(query) + + // Type-aware parsing: + // 1. Detect NounType using pre-embedded type vectors + // 2. Get type-specific fields from real data patterns + // 3. Semantic field matching with type affinity boosting + // 4. Validate field-type compatibility + // 5. Generate optimized query plan + + params = nlpParams } else { - params = query // Direct structured query + params = query // Direct structured query } ``` @@ -257,12 +256,12 @@ const searchPromises = [] // Vector search (if query text or vector provided) if (params.query || params.vector) { - searchPromises.push(this.executeVectorSearch(params)) + searchPromises.push(this.executeVectorSearch(params)) } -// Proximity search (if near parameter provided) +// Proximity search (if near parameter provided) if (params.near) { - searchPromises.push(this.executeProximitySearch(params)) + searchPromises.push(this.executeProximitySearch(params)) } // Wait for all searches to complete @@ -273,41 +272,41 @@ const searchResults = await Promise.all(searchPromises) ```typescript // Apply metadata filters using optimized indices if (params.where || params.type || params.service) { - const filter = { - ...params.where, - ...(params.type && { noun: params.type }), - ...(params.service && { service: params.service }) - } - - // Get optimal query plan based on field cardinalities - const queryPlan = await this.getOptimalQueryPlan(filter) - - // Execute filters in optimal order (low cardinality first) - const filteredIds = await this.metadataIndex.getIdsForFilter(filter) - - if (results.length > 0) { - // Intersect with vector search results - results = results.filter(r => filteredIds.includes(r.id)) - } else { - // Create results from metadata matches (metadata-only query) - results = await this.createResultsFromIds(filteredIds) - } + const filter = { + ...params.where, + ...(params.type && { noun: params.type }), + ...(params.service && { service: params.service }) + } + + // Get optimal query plan based on field cardinalities + const queryPlan = await this.getOptimalQueryPlan(filter) + + // Execute filters in optimal order (low cardinality first) + const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + + if (results.length > 0) { + // Intersect with vector search results + results = results.filter(r => filteredIds.includes(r.id)) + } else { + // Create results from metadata matches (metadata-only query) + results = await this.createResultsFromIds(filteredIds) + } } ``` -### Phase 4: Graph Traversal +### Phase 4: Graph Traversal ```typescript // Apply graph constraints using O(1) lookups if (params.connected) { - const connectedIds = await this.graphIndex.getConnectedIds(params.connected) - - if (results.length > 0) { - // Filter existing results to only connected entities - results = results.filter(r => connectedIds.includes(r.id)) - } else { - // Create results from connected entities - results = await this.createResultsFromIds(connectedIds) - } + const connectedIds = await this.graphIndex.getConnectedIds(params.connected) + + if (results.length > 0) { + // Filter existing results to only connected entities + results = results.filter(r => connectedIds.includes(r.id)) + } else { + // Create results from connected entities + results = await this.createResultsFromIds(connectedIds) + } } ``` @@ -315,12 +314,12 @@ if (params.connected) { ```typescript // Combine scores from multiple intelligence sources if (params.fusion && results.length > 0) { - results = this.applyFusionScoring(results, params.fusion) - - // Example fusion strategies: - // - Weighted: vectorScore × 0.6 + metadataScore × 0.2 + graphScore × 0.2 - // - Adaptive: adjust weights based on query characteristics - // - Progressive: prioritize based on result confidence + results = this.applyFusionScoring(results, params.fusion) + + // Example fusion strategies: + // - Weighted: vectorScore × 0.6 + metadataScore × 0.2 + graphScore × 0.2 + // - Adaptive: adjust weights based on query characteristics + // - Progressive: prioritize based on result confidence } // Sort by final score and apply pagination @@ -351,7 +350,7 @@ return results.slice(offset, offset + limit) ### 3. Field-Type Validation ```typescript // Prevents invalid queries and suggests alternatives: -// "people with publishDate > 2020" +// "people with publishDate > 2020" // → Warning: "Person entities rarely have publishDate field" // → Suggestion: "Did you mean createdAt, updatedAt, or birthDate?" // → Auto-correction: Use most likely alternative based on affinity data @@ -373,7 +372,7 @@ return results.slice(offset, offset + limit) Where: - n = number of entities in database -- t = number of types (169 total: 42 noun + 127 verb) +- t = number of types (169 total: 42 noun + 127 verb) - f = number of fields for detected entity type (typically 5-15) ### Scalability @@ -401,27 +400,27 @@ Where: // Phase 1: NLP Parsing // - "papers" → NounType.Document (0.94 confidence) // - "Stanford researchers" → entity search + NounType.Person -// - "recent" → publishDate: {greaterThan: 2023} +// - "recent" → publishDate: {greaterThan: 2023} // - "high citations" → citations: {greaterThan: 100} // - "connected to industry" → graph traversal via VerbType.AffiliatedWith // Phase 2: Generated Query { - type: NounType.Document, - where: { - publishDate: { greaterThan: 2023 }, - citations: { greaterThan: 100 } - }, - connected: { - to: ["stanford-researchers"], - via: VerbType.AffiliatedWith, - depth: 2 - } + type: NounType.Document, + where: { + publishDate: { greaterThan: 2023 }, + citations: { greaterThan: 100 } + }, + connected: { + to: ["stanford-researchers"], + via: VerbType.AffiliatedWith, + depth: 2 + } } // Phase 3: Execution Plan // 1. Metadata filter: publishDate > 2023 (O(log n) via sorted index) -// 2. Metadata filter: citations > 100 (O(log n) via sorted index) +// 2. Metadata filter: citations > 100 (O(log n) via sorted index) // 3. Graph traversal: connected to Stanford (O(1) per hop) // 4. Intersection: entities matching all constraints // 5. Sort by relevance score @@ -431,22 +430,22 @@ Where: ```typescript // Query: Direct structured query for maximum performance await brain.find({ - type: NounType.Document, // O(1) type filter - where: { - status: "published", // O(1) exact match - year: 2024, // O(1) exact match - citations: { greaterThan: 50 } // O(log n) range query - }, - connected: { - from: "author-123", // O(1) graph lookup - via: VerbType.Creates - }, - limit: 10 + type: NounType.Document, // O(1) type filter + where: { + status: "published", // O(1) exact match + year: 2024, // O(1) exact match + citations: { greaterThan: 50 } // O(log n) range query + }, + connected: { + from: "author-123", // O(1) graph lookup + via: VerbType.Creates + }, + limit: 10 }) // Total performance: ~1.2ms for 100K entities ``` -## Filter Syntax Reference (v5.8.0+) +## Filter Syntax Reference ### Where Clause: Complete Operator Guide @@ -457,25 +456,25 @@ Brainy provides a comprehensive set of operators for filtering entities by metad **Exact Match** (shorthand): ```typescript await brain.find({ - where: { - status: 'active', // Shorthand for { eq: 'active' } - year: 2024, // Exact match for numbers - verified: true // Boolean matching - } + where: { + status: 'active', // Shorthand for { eq: 'active' } + year: 2024, // Exact match for numbers + verified: true // Boolean matching + } }) ``` **Comparison Operators**: ```typescript await brain.find({ - where: { - age: { gt: 18 }, // Greater than - score: { gte: 80 }, // Greater than or equal - price: { lt: 100 }, // Less than - stock: { lte: 10 }, // Less than or equal - status: { eq: 'active' }, // Equals (explicit) - role: { ne: 'guest' } // Not equals - } + where: { + age: { gt: 18 }, // Greater than + score: { gte: 80 }, // Greater than or equal + price: { lt: 100 }, // Less than + stock: { lte: 10 }, // Less than or equal + status: { eq: 'active' }, // Equals (explicit) + role: { ne: 'guest' } // Not equals + } }) ``` @@ -486,11 +485,11 @@ await brain.find({ **Between** (inclusive): ```typescript await brain.find({ - where: { - publishDate: { between: [2020, 2024] }, // Year range - price: { between: [10.00, 99.99] }, // Price range - timestamp: { between: [startMs, endMs] } // Time range - } + where: { + publishDate: { between: [2020, 2024] }, // Year range + price: { between: [10.00, 99.99] }, // Price range + timestamp: { between: [startMs, endMs] } // Time range + } }) ``` @@ -501,11 +500,11 @@ await brain.find({ **In/Not In**: ```typescript await brain.find({ - where: { - category: { in: ['tech', 'science', 'research'] }, - status: { notIn: ['draft', 'deleted'] }, - priority: { in: [1, 2, 3] } - } + where: { + category: { in: ['tech', 'science', 'research'] }, + status: { notIn: ['draft', 'deleted'] }, + priority: { in: [1, 2, 3] } + } }) ``` @@ -516,11 +515,11 @@ await brain.find({ **Contains/Starts/Ends**: ```typescript await brain.find({ - where: { - title: { contains: 'machine learning' }, // Substring search - email: { startsWith: 'admin@' }, // Prefix match - filename: { endsWith: '.pdf' } // Suffix match - } + where: { + title: { contains: 'machine learning' }, // Substring search + email: { startsWith: 'admin@' }, // Prefix match + filename: { endsWith: '.pdf' } // Suffix match + } }) ``` @@ -540,11 +539,11 @@ query: 'artificial intelligence' **Exists/Missing**: ```typescript await brain.find({ - where: { - email: { exists: true }, // Has email field - deletedAt: { exists: false }, // No deletedAt field (not deleted) - profileImage: { exists: true } // Has profile image - } + where: { + email: { exists: true }, // Has email field + deletedAt: { exists: false }, // No deletedAt field (not deleted) + profileImage: { exists: true } // Has profile image + } }) ``` @@ -560,11 +559,11 @@ All conditions at the same level are implicitly AND: ```typescript await brain.find({ - where: { - status: 'published', // AND - year: { gte: 2020 }, // AND - citations: { gte: 50 } // AND - } + where: { + status: 'published', // AND + year: { gte: 2020 }, // AND + citations: { gte: 50 } // AND + } }) // Returns: entities matching ALL three conditions ``` @@ -572,13 +571,13 @@ await brain.find({ **Explicit AND with `allOf`**: ```typescript await brain.find({ - where: { - allOf: [ - { status: 'published' }, - { year: { gte: 2020 } }, - { citations: { gte: 50 } } - ] - } + where: { + allOf: [ + { status: 'published' }, + { year: { gte: 2020 } }, + { citations: { gte: 50 } } + ] + } }) ``` @@ -590,13 +589,13 @@ Match ANY condition: ```typescript await brain.find({ - where: { - anyOf: [ - { status: 'urgent' }, - { priority: { gte: 8 } }, - { assignee: 'admin' } - ] - } + where: { + anyOf: [ + { status: 'urgent' }, + { priority: { gte: 8 } }, + { assignee: 'admin' } + ] + } }) // Returns: entities matching ANY condition ``` @@ -604,13 +603,13 @@ await brain.find({ **Combined AND + OR**: ```typescript await brain.find({ - where: { - status: 'active', // Must be active - anyOf: [ // AND (urgent OR high priority) - { tags: { contains: 'urgent' } }, - { priority: { gte: 8 } } - ] - } + where: { + status: 'active', // Must be active + anyOf: [ // AND (urgent OR high priority) + { tags: { contains: 'urgent' } }, + { priority: { gte: 8 } } + ] + } }) ``` @@ -622,17 +621,17 @@ Complex boolean expressions: ```typescript await brain.find({ - where: { - allOf: [ - { status: 'published' }, - { - anyOf: [ - { featured: true }, - { citations: { gte: 100 } } - ] - } - ] - } + where: { + allOf: [ + { status: 'published' }, + { + anyOf: [ + { featured: true }, + { citations: { gte: 100 } } + ] + } + ] + } }) // Returns: published AND (featured OR highly cited) ``` @@ -677,8 +676,8 @@ Filter entities by NounType: ```typescript await brain.find({ - type: NounType.Document, - where: { year: { gte: 2020 } } + type: NounType.Document, + where: { year: { gte: 2020 } } }) ``` @@ -686,8 +685,8 @@ await brain.find({ ```typescript await brain.find({ - type: [NounType.Person, NounType.Organization], - where: { verified: true } + type: [NounType.Person, NounType.Organization], + where: { verified: true } }) ``` @@ -728,11 +727,11 @@ Traverse relationships using the GraphIndex: ```typescript await brain.find({ - connected: { - to: 'entity-id-123', // Connected to this entity - via: VerbType.WorksFor, // Through this relationship type - direction: 'out' // Direction: 'in', 'out', or 'both' - } + connected: { + to: 'entity-id-123', // Connected to this entity + via: VerbType.WorksFor, // Through this relationship type + direction: 'out' // Direction: 'in', 'out', or 'both' + } }) ``` @@ -742,11 +741,11 @@ await brain.find({ ```typescript await brain.find({ - connected: { - to: 'research-institution', - via: VerbType.AffiliatedWith, - depth: 2 // Up to 2 hops away - } + connected: { + to: 'research-institution', + via: VerbType.AffiliatedWith, + depth: 2 // Up to 2 hops away + } }) ``` @@ -756,35 +755,35 @@ await brain.find({ ```typescript await brain.find({ - type: NounType.Person, - where: { - verified: true, - reputation: { gte: 100 } - }, - connected: { - to: 'stanford-ai-lab', - via: VerbType.WorksAt, - direction: 'out' - }, - limit: 20 + type: NounType.Person, + where: { + verified: true, + reputation: { gte: 100 } + }, + connected: { + to: 'stanford-ai-lab', + via: VerbType.WorksAt, + direction: 'out' + }, + limit: 20 }) // Returns: Verified people with high reputation who work at Stanford AI Lab ``` -#### Pagination with Graph Queries (v5.8.0+) +#### Pagination with Graph Queries ```typescript // Page through high-degree nodes efficiently const neighbors = await brain.graphIndex.getNeighbors('hub-entity-id', { - direction: 'out', - limit: 50, - offset: 0 + direction: 'out', + limit: 50, + offset: 0 }) // Get verb IDs with pagination const verbIds = await brain.graphIndex.getVerbIdsBySource('source-id', { - limit: 100, - offset: 0 + limit: 100, + offset: 0 }) ``` @@ -792,36 +791,36 @@ const verbIds = await brain.graphIndex.getVerbIdsBySource('source-id', { **Note**: See `src/graph/graphAdjacencyIndex.ts` for low-level graph operations. -### Sorting Results (v4.5.4+) +### Sorting Results Sort query results by any field, including timestamps: ```typescript // Sort by timestamp (descending - newest first) await brain.find({ - type: NounType.Document, - orderBy: 'createdAt', - order: 'desc', - limit: 10 + type: NounType.Document, + orderBy: 'createdAt', + order: 'desc', + limit: 10 }) // Sort by custom field (ascending) await brain.find({ - where: { status: { eq: 'published' } }, - orderBy: 'priority', - order: 'asc' + where: { status: { eq: 'published' } }, + orderBy: 'priority', + order: 'asc' }) // Sort with filtering and pagination await brain.find({ - where: { - publishDate: { gte: startDate }, - citations: { gte: 50 } - }, - orderBy: 'citations', - order: 'desc', - limit: 20, - offset: 0 + where: { + publishDate: { gte: startDate }, + citations: { gte: 50 } + }, + orderBy: 'citations', + order: 'desc', + limit: 20, + offset: 0 }) ``` @@ -836,17 +835,17 @@ await brain.find({ ```typescript // Range query + sorting await brain.find({ - where: { - createdAt: { gte: Date.now() - 86400000 } // Last 24 hours - }, - orderBy: 'createdAt', - order: 'desc' // Newest first + where: { + createdAt: { gte: Date.now() - 86400000 } // Last 24 hours + }, + orderBy: 'createdAt', + order: 'desc' // Newest first }) // Works with updatedAt, accessed, modified await brain.find({ - orderBy: 'updatedAt', - order: 'desc' + orderBy: 'updatedAt', + order: 'desc' }) ``` @@ -854,22 +853,22 @@ await brain.find({ ```typescript // Sort search results by custom field instead of relevance await brain.find({ - query: "machine learning", - where: { publishDate: { gte: 2023 } }, - orderBy: 'citations', // Sort by citations, not relevance - order: 'desc' + query: "machine learning", + where: { publishDate: { gte: 2023 } }, + orderBy: 'citations', // Sort by citations, not relevance + order: 'desc' }) // Paginated sorted results async function getDocumentsByDate(page: number, pageSize: number = 20) { - return await brain.find({ - type: NounType.Document, - where: { status: { eq: 'published' } }, - orderBy: 'publishDate', - order: 'desc', - limit: pageSize, - offset: page * pageSize - }) + return await brain.find({ + type: NounType.Document, + where: { status: { eq: 'published' } }, + orderBy: 'publishDate', + order: 'desc', + limit: pageSize, + offset: page * pageSize + }) } ``` @@ -880,30 +879,30 @@ async function getDocumentsByDate(page: number, pageSize: number = 20) { **Offset-based pagination**: ```typescript async function getPaginatedResults(page: number, pageSize: number = 20) { - return await brain.find({ - type: NounType.Document, - where: { status: 'published' }, - orderBy: 'createdAt', - order: 'desc', - limit: pageSize, - offset: page * pageSize - }) + return await brain.find({ + type: NounType.Document, + where: { status: 'published' }, + orderBy: 'createdAt', + order: 'desc', + limit: pageSize, + offset: page * pageSize + }) } // Usage -const page1 = await getPaginatedResults(0) // First 20 -const page2 = await getPaginatedResults(1) // Next 20 +const page1 = await getPaginatedResults(0) // First 20 +const page2 = await getPaginatedResults(1) // Next 20 ``` -**Graph pagination** (v5.8.0+): +**Graph pagination**: ```typescript // Paginate through high-degree node relationships async function getNeighborPage(entityId: string, page: number, pageSize: number = 50) { - return await brain.graphIndex.getNeighbors(entityId, { - direction: 'out', - limit: pageSize, - offset: page * pageSize - }) + return await brain.graphIndex.getNeighbors(entityId, { + direction: 'out', + limit: pageSize, + offset: page * pageSize + }) } ``` @@ -916,23 +915,23 @@ async function getNeighborPage(entityId: string, page: number, pageSize: number // Last 24 hours const oneDayAgo = Date.now() - (24 * 60 * 60 * 1000) await brain.find({ - where: { - createdAt: { gte: oneDayAgo } - }, - orderBy: 'createdAt', - order: 'desc' + where: { + createdAt: { gte: oneDayAgo } + }, + orderBy: 'createdAt', + order: 'desc' }) // Last 7 days with additional filters const oneWeekAgo = Date.now() - (7 * 24 * 60 * 60 * 1000) await brain.find({ - type: NounType.Document, - where: { - createdAt: { gte: oneWeekAgo }, - status: 'published' - }, - orderBy: 'createdAt', - order: 'desc' + type: NounType.Document, + where: { + createdAt: { gte: oneWeekAgo }, + status: 'published' + }, + orderBy: 'createdAt', + order: 'desc' }) ``` @@ -940,21 +939,21 @@ await brain.find({ ```typescript // Specific year await brain.find({ - where: { - publishDate: { between: [ - new Date('2023-01-01').getTime(), - new Date('2023-12-31').getTime() - ]} - } + where: { + publishDate: { between: [ + new Date('2023-01-01').getTime(), + new Date('2023-12-31').getTime() + ]} + } }) // Quarter const Q1_2024_start = new Date('2024-01-01').getTime() const Q1_2024_end = new Date('2024-03-31').getTime() await brain.find({ - where: { - createdAt: { between: [Q1_2024_start, Q1_2024_end] } - } + where: { + createdAt: { between: [Q1_2024_start, Q1_2024_end] } + } }) ``` @@ -964,28 +963,28 @@ await brain.find({ ```typescript // Find: AI research papers from verified authors at top institutions const results = await brain.find({ - // Vector search (semantic) - query: 'artificial intelligence machine learning', + // Vector search (semantic) + query: 'artificial intelligence machine learning', - // Metadata filters - type: NounType.Document, - where: { - publishDate: { gte: 2020 }, - citations: { gte: 50 }, - peerReviewed: true - }, + // Metadata filters + type: NounType.Document, + where: { + publishDate: { gte: 2020 }, + citations: { gte: 50 }, + peerReviewed: true + }, - // Graph traversal - connected: { - to: topInstitutionIds, // Array of institution entity IDs - via: VerbType.AffiliatedWith, - depth: 2 // Authors affiliated with institutions (2 hops) - }, + // Graph traversal + connected: { + to: topInstitutionIds, // Array of institution entity IDs + via: VerbType.AffiliatedWith, + depth: 2 // Authors affiliated with institutions (2 hops) + }, - // Results - limit: 50, - orderBy: 'citations', - order: 'desc' + // Results + limit: 50, + orderBy: 'citations', + order: 'desc' }) ``` @@ -997,19 +996,19 @@ const results = await brain.find({ ```typescript // Standard query excludes deleted await brain.find({ - where: { - deletedAt: { exists: false } // Not soft-deleted - } + where: { + deletedAt: { exists: false } // Not soft-deleted + } }) // Or use compound filter await brain.find({ - where: { - allOf: [ - { status: 'active' }, - { deletedAt: { exists: false } } - ] - } + where: { + allOf: [ + { status: 'active' }, + { deletedAt: { exists: false } } + ] + } }) ``` @@ -1021,21 +1020,21 @@ await brain.find({ ```typescript // Find documents similar to a specific document await brain.find({ - near: { - id: 'doc-123', - threshold: 0.8 // Minimum 80% similarity - }, - type: NounType.Document, - limit: 10 + near: { + id: 'doc-123', + threshold: 0.8 // Minimum 80% similarity + }, + type: NounType.Document, + limit: 10 }) // With metadata constraints await brain.find({ - near: { id: 'paper-456', threshold: 0.75 }, - where: { - publishDate: { gte: 2020 }, - language: 'en' - } + near: { id: 'paper-456', threshold: 0.75 }, + where: { + publishDate: { gte: 2020 }, + language: 'en' + } }) ``` @@ -1047,8 +1046,8 @@ await brain.find({ ```typescript // Get total count (metadata-only query is fastest) const results = await brain.find({ - where: { status: 'published' }, - limit: 1 // We only need the count + where: { status: 'published' }, + limit: 1 // We only need the count }) // Note: Current API returns results, not counts // For production, consider caching counts or using metadata indices directly @@ -1059,10 +1058,10 @@ const results = await brain.find({ // Find all entities, then group by type in application const allEntities = await brain.find({ limit: 10000 }) const byType = allEntities.reduce((acc, entity) => { - const type = entity.noun || 'unknown' - if (!acc[type]) acc[type] = [] - acc[type].push(entity) - return acc + const type = entity.noun || 'unknown' + if (!acc[type]) acc[type] = [] + acc[type].push(entity) + return acc }, {}) ``` @@ -1071,14 +1070,14 @@ const byType = allEntities.reduce((acc, entity) => { **Any of multiple values**: ```typescript await brain.find({ - where: { - anyOf: [ - { priority: 'urgent' }, - { priority: 'high' }, - { assignee: 'admin' }, - { dueDate: { lte: Date.now() } } - ] - } + where: { + anyOf: [ + { priority: 'urgent' }, + { priority: 'high' }, + { assignee: 'admin' }, + { dueDate: { lte: Date.now() } } + ] + } }) // Returns: urgent OR high priority OR assigned to admin OR overdue ``` @@ -1087,23 +1086,23 @@ await brain.find({ ```typescript // Find: (Premium users OR trial users with activity) AND not banned await brain.find({ - type: NounType.Person, - where: { - allOf: [ - { - anyOf: [ - { subscription: 'premium' }, - { - allOf: [ - { subscription: 'trial' }, - { lastActive: { gte: Date.now() - 86400000 } } // 24h - ] - } - ] - }, - { banned: { ne: true } } - ] - } + type: NounType.Person, + where: { + allOf: [ + { + anyOf: [ + { subscription: 'premium' }, + { + allOf: [ + { subscription: 'trial' }, + { lastActive: { gte: Date.now() - 86400000 } } // 24h + ] + } + ] + }, + { banned: { ne: true } } + ] + } }) ``` @@ -1115,8 +1114,8 @@ await brain.find({ ```typescript // List all entities of a type const all = await brain.find({ - type: NounType.Document, - limit: 10 + type: NounType.Document, + limit: 10 }) console.log(`Found ${all.length} documents`) ``` @@ -1124,11 +1123,11 @@ console.log(`Found ${all.length} documents`) **Check 2: Test filters individually** ```typescript // Remove filters one by one to find the culprit -await brain.find({ where: { status: 'published' } }) // Works? -await brain.find({ where: { year: 2024 } }) // Works? +await brain.find({ where: { status: 'published' } }) // Works? +await brain.find({ where: { year: 2024 } }) // Works? await brain.find({ where: { - status: 'published', - year: 2024 // Combined - works? + status: 'published', + year: 2024 // Combined - works? }}) ``` @@ -1136,7 +1135,7 @@ await brain.find({ where: { ```typescript // Get a sample entity to see actual field names const sample = await brain.find({ type: NounType.Document, limit: 1 }) -console.log(Object.keys(sample[0].data)) // Actual fields +console.log(Object.keys(sample[0].data)) // Actual fields ``` **Common issues**: @@ -1150,9 +1149,9 @@ console.log(Object.keys(sample[0].data)) // Actual fields ```typescript // Use explain mode (if available) const results = await brain.find({ - query: 'machine learning', - where: { title: { contains: 'AI' } }, // ⚠️ O(n) substring search - explain: true + query: 'machine learning', + where: { title: { contains: 'AI' } }, // ⚠️ O(n) substring search + explain: true }) ``` @@ -1175,15 +1174,15 @@ where: { status: 'published' } ```typescript // ❌ Suboptimal: Slow filter first where: { - description: { contains: 'AI' }, // O(n) - runs first - year: 2024 // O(1) - runs second + description: { contains: 'AI' }, // O(n) - runs first + year: 2024 // O(1) - runs second } // ✅ Optimal: Fast filter first (automatic optimization) where: { - year: 2024, // O(1) - narrow results - status: 'published' // O(1) - further narrow - // Only then apply O(n) operations if needed + year: 2024, // O(1) - narrow results + status: 'published' // O(1) - further narrow + // Only then apply O(n) operations if needed } ``` @@ -1205,10 +1204,10 @@ import { NounType } from '@soulcraft/brainy' await brain.find({ type: NounType.Document }) // ❌ Error: Operator not recognized -where: { age: { greaterThan: 18 } } // Old API +where: { age: { greaterThan: 18 } } // Old API // ✅ Correct: Use canonical operators -where: { age: { gt: 18 } } // v5.0.0+ +where: { age: { gt: 18 } } ``` ### Graph Traversal Issues @@ -1217,25 +1216,25 @@ where: { age: { gt: 18 } } // v5.0.0+ ```typescript // Verify relationship exists const relations = await brain.getRelations({ - from: 'entity-a', - to: 'entity-b' + from: 'entity-a', + to: 'entity-b' }) console.log('Relationships:', relations) // Check direction await brain.find({ - connected: { - to: 'entity-id', - direction: 'in' // Try 'out' or 'both' - } + connected: { + to: 'entity-id', + direction: 'in' // Try 'out' or 'both' + } }) // Verify verb type await brain.find({ - connected: { - to: 'entity-id', - via: VerbType.WorksFor // Correct VerbType? - } + connected: { + to: 'entity-id', + via: VerbType.WorksFor // Correct VerbType? + } }) ``` @@ -1256,12 +1255,12 @@ await brain.update(entity.id, { data: entity.data }) ```typescript // ❌ Too strict: May return nothing await brain.find({ - near: { id: 'doc-123', threshold: 0.95 } + near: { id: 'doc-123', threshold: 0.95 } }) // ✅ Reasonable: 0.7-0.85 is typical await brain.find({ - near: { id: 'doc-123', threshold: 0.75 } + near: { id: 'doc-123', threshold: 0.75 } }) ``` @@ -1275,8 +1274,8 @@ console.log('Entity type:', entity.noun) // Verify type filter is working await brain.find({ - type: NounType.Document, - where: { id: 'unexpected-id' } // Should not return if wrong type + type: NounType.Document, + where: { id: 'unexpected-id' } // Should not return if wrong type }) ``` @@ -1291,7 +1290,7 @@ console.log(`Results: ${results.length}, Unique: ${uniqueIds.size}`) // Brainy should never return duplicates - report if found ``` -## VFS (Virtual File System) Visibility (v4.7.0+) +## VFS (Virtual File System) Visibility ### Default Behavior @@ -1312,8 +1311,8 @@ If you need to exclude VFS entities from specific queries, use the `excludeVFS` ```typescript // Exclude VFS files from results await brain.find({ - query: 'machine learning', - excludeVFS: true // Only return non-file entities + query: 'machine learning', + excludeVFS: true // Only return non-file entities }) ``` @@ -1322,14 +1321,14 @@ await brain.find({ ```typescript // Explicit filtering (same as excludeVFS: true) await brain.find({ - query: 'machine learning', - where: { vfsType: { exists: false } } + query: 'machine learning', + where: { vfsType: { exists: false } } }) // Or only search VFS files await brain.find({ - query: 'setup instructions', - where: { vfsType: 'file' } // Only files + query: 'setup instructions', + where: { vfsType: 'file' } // Only files }) ``` @@ -1342,24 +1341,24 @@ await brain.find({ ### Migration from v4.6.x -**BREAKING CHANGE** (v4.7.0): The `includeVFS` parameter has been removed: +**BREAKING CHANGE**: The `includeVFS` parameter has been removed: ```typescript // ❌ Old (v4.6.x and earlier) await brain.find({ - query: 'docs', - includeVFS: true // No longer needed! + query: 'docs', + includeVFS: true // No longer needed! }) -// ✅ New (v4.7.0+) +// ✅ New await brain.find({ - query: 'docs' // VFS included by default + query: 'docs' // VFS included by default }) // ✅ To exclude VFS (if needed) await brain.find({ - query: 'concepts', - excludeVFS: true + query: 'concepts', + excludeVFS: true }) ``` diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 15344b1e..ec63abc8 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -24,11 +24,11 @@ Where: - `f` = number of fields for entity type - `t` = number of types (42 nouns, 127 verbs) -### v5.11.1: brain.get() Metadata-Only Optimization +### brain.get() Metadata-Only Optimization ✨ **Massive Performance Improvement**: `brain.get()` is now **76-81% faster** by default! -| Operation | Before (v5.11.0) | After (v5.11.1) | Speedup | Use Case | +| Operation | Before | After | Speedup | Use Case | |-----------|------------------|-----------------|---------|----------| | **brain.get() (metadata-only)** | 43ms, 6KB | **10ms, 300 bytes** | **76-81%** | VFS, existence checks, metadata | | **brain.get({ includeVectors: true })** | 43ms, 6KB | 43ms, 6KB | 0% | Similarity calculations | @@ -308,7 +308,7 @@ const results = await Promise.all(searchPromises) - ✅ **Horizontally Scalable**: Stateless operations support clustering - ✅ **Zero Stubs**: Every line of code is production-ready -## Lazy Loading Performance (v5.7.7+) +## Lazy Loading Performance Brainy supports two initialization modes for optimal performance across different use cases: @@ -324,7 +324,7 @@ await brain.init() // Rebuilds indexes during init (~500ms-3s for 10K entities) - First query: Instant (indexes already loaded) - Use case: Traditional applications, long-running servers -### Mode 2: Lazy Loading (v5.7.7+) +### Mode 2: Lazy Loading ```javascript const brain = new Brainy({ disableAutoRebuild: true }) diff --git a/docs/README.md b/docs/README.md index bed4e26f..6b992f99 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ -# Brainy Documentation (v6.5.0) +# Brainy Documentation Welcome to the comprehensive documentation for Brainy, the multi-dimensional AI database with Triple Intelligence Engine. -## 🆕 What's New in v4.0.0 +## 🆕 What's New in **Production-Ready Cost Optimization:** - **Lifecycle Management**: Automatic tier transitions for S3, GCS, Azure (96% cost savings!) @@ -14,7 +14,7 @@ Welcome to the comprehensive documentation for Brainy, the multi-dimensional AI **Cost Impact Example (500TB dataset):** - Before: $138,000/year -- After v4.0.0: $5,940/year +- After $5,940/year - **Savings: $132,060/year (96%)** ## 📊 Implementation Status @@ -85,23 +85,23 @@ await brain.init() // Add entities (nouns) const articleId = await brain.add({ - data: "Revolutionary AI Breakthrough", - type: NounType.Document, - metadata: { category: "technology", rating: 4.8 } + data: "Revolutionary AI Breakthrough", + type: NounType.Document, + metadata: { category: "technology", rating: 4.8 } }) const authorId = await brain.add({ - data: "Dr. Sarah Chen", - type: NounType.Person, - metadata: { role: "researcher" } + data: "Dr. Sarah Chen", + type: NounType.Person, + metadata: { role: "researcher" } }) // Create relationships (verbs) await brain.relate({ - from: authorId, - to: articleId, - type: VerbType.CreatedBy, - metadata: { date: "2024-01-15", contribution: "primary" } + from: authorId, + to: articleId, + type: VerbType.CreatedBy, + metadata: { date: "2024-01-15", contribution: "primary" } }) // Query naturally @@ -117,7 +117,7 @@ const results = await brain.find("highly rated technology articles by researcher | [API Reference](./api/README.md) | **START HERE** - Complete API documentation with examples | | [VFS Quick Start](./vfs/QUICK_START.md) | Virtual filesystem in 30 seconds | -### 🆕 v4.0.0 Migration & Optimization +### 🆕 Migration & Optimization | Document | Description | |----------|-------------| @@ -135,15 +135,15 @@ const results = await brain.find("highly rated technology articles by researcher | [Noun-Verb Taxonomy](./architecture/noun-verb-taxonomy.md) | Revolutionary data model - entities and relationships | | [Triple Intelligence](./architecture/triple-intelligence.md) | Vector + Graph + Field unified query system | | [Zero Configuration](./architecture/zero-config.md) | Auto-adapts to any environment | -| [Storage Architecture](./architecture/storage-architecture.md) | **v4.0.0** - Storage adapters and optimization | -| [Data Storage Architecture](./architecture/data-storage-architecture.md) | **v4.0.0** - Metadata/vector separation, sharding | +| [Storage Architecture](./architecture/storage-architecture.md) | Storage adapters and optimization | +| [Data Storage Architecture](./architecture/data-storage-architecture.md) | Metadata/vector separation, sharding | | [Index Architecture](./architecture/index-architecture.md) | HNSW, Graph, and Metadata indexing | ### 💾 Storage & Deployment | Document | Description | |----------|-------------| -| [Cloud Deployment Guide](./deployment/CLOUD_DEPLOYMENT_GUIDE.md) | **v4.0.0** - Deploy on AWS, GCP, Azure, Cloudflare | +| [Cloud Deployment Guide](./deployment/CLOUD_DEPLOYMENT_GUIDE.md) | Deploy on AWS, GCP, Azure, Cloudflare | | [AWS Deployment](./deployment/aws-deployment.md) | AWS-specific deployment patterns | | [GCP Deployment](./deployment/gcp-deployment.md) | Google Cloud deployment | | [Kubernetes Deployment](./deployment/kubernetes-deployment.md) | K8s deployment configurations | @@ -257,75 +257,68 @@ const results = await brain.find("highly rated technology articles by researcher ``` docs/ -├── README.md (this file) # Complete documentation index -├── MIGRATION-V3-TO-V4.md # v4.0.0 migration guide +├── README.md (this file) # Complete documentation index +├── MIGRATION-V3-TO-V4.md # Migration guide │ -├── guides/ # User guides -│ ├── natural-language.md -│ ├── neural-api.md -│ ├── import-anything.md -│ ├── framework-integration.md -│ ├── nextjs-integration.md -│ ├── vue-integration.md -│ ├── distributed-system.md -│ ├── model-loading.md -│ └── enterprise-for-everyone.md +├── guides/ # User guides +│ ├── natural-language.md +│ ├── neural-api.md +│ ├── import-anything.md +│ ├── framework-integration.md +│ ├── nextjs-integration.md +│ ├── vue-integration.md +│ ├── distributed-system.md +│ ├── model-loading.md +│ └── enterprise-for-everyone.md │ -├── architecture/ # System architecture -│ ├── overview.md -│ ├── noun-verb-taxonomy.md -│ ├── triple-intelligence.md -│ ├── zero-config.md -│ ├── storage-architecture.md # v4.0.0 -│ ├── data-storage-architecture.md # v4.0.0 -│ ├── index-architecture.md -│ ├── distributed-storage.md -│ ├── augmentations.md -│ ├── augmentation-system-audit.md -│ ├── augmentations-actual.md -│ ├── finite-type-system.md -│ └── ... +├── architecture/ # System architecture +│ ├── overview.md +│ ├── noun-verb-taxonomy.md +│ ├── triple-intelligence.md +│ ├── zero-config.md +│ ├── storage-architecture.md│ ├── data-storage-architecture.md│ ├── index-architecture.md +│ ├── distributed-storage.md +│ ├── augmentations.md +│ ├── augmentation-system-audit.md +│ ├── augmentations-actual.md +│ ├── finite-type-system.md +│ └── ... │ -├── operations/ # Operations guides -│ ├── cost-optimization-aws-s3.md # v4.0.0 -│ ├── cost-optimization-gcs.md # v4.0.0 -│ ├── cost-optimization-azure.md # v4.0.0 -│ ├── cost-optimization-cloudflare-r2.md # v4.0.0 -│ └── capacity-planning.md +├── operations/ # Operations guides +│ ├── cost-optimization-aws-s3.md│ ├── cost-optimization-gcs.md│ ├── cost-optimization-azure.md│ ├── cost-optimization-cloudflare-r2.md│ └── capacity-planning.md │ -├── deployment/ # Deployment guides -│ ├── CLOUD_DEPLOYMENT_GUIDE.md # v4.0.0 -│ ├── aws-deployment.md -│ ├── gcp-deployment.md -│ └── kubernetes-deployment.md +├── deployment/ # Deployment guides +│ ├── CLOUD_DEPLOYMENT_GUIDE.md│ ├── aws-deployment.md +│ ├── gcp-deployment.md +│ └── kubernetes-deployment.md │ -├── vfs/ # Virtual Filesystem docs -│ ├── QUICK_START.md -│ ├── VFS_CORE.md -│ ├── SEMANTIC_VFS.md -│ ├── VFS_API_GUIDE.md -│ ├── NEURAL_EXTRACTION.md -│ ├── COMMON_PATTERNS.md -│ └── ... +├── vfs/ # Virtual Filesystem docs +│ ├── QUICK_START.md +│ ├── VFS_CORE.md +│ ├── SEMANTIC_VFS.md +│ ├── VFS_API_GUIDE.md +│ ├── NEURAL_EXTRACTION.md +│ ├── COMMON_PATTERNS.md +│ └── ... │ -├── api/ # API documentation -│ ├── README.md -│ └── COMPREHENSIVE_API_OVERVIEW.md +├── api/ # API documentation +│ ├── README.md +│ └── COMPREHENSIVE_API_OVERVIEW.md │ -├── augmentations/ # Augmentation docs -│ ├── COMPLETE-REFERENCE.md -│ ├── DEVELOPER-GUIDE.md -│ ├── CONFIGURATION.md -│ └── api-server.md +├── augmentations/ # Augmentation docs +│ ├── COMPLETE-REFERENCE.md +│ ├── DEVELOPER-GUIDE.md +│ ├── CONFIGURATION.md +│ └── api-server.md │ -├── features/ # Feature documentation -│ ├── complete-feature-list.md -│ └── v3-features.md +├── features/ # Feature documentation +│ ├── complete-feature-list.md +│ └── v3-features.md │ -└── internal/ # Internal docs - ├── AUDIT_REPORT.md - ├── CLEANUP_SUMMARY.md - └── HONEST_STATUS.md +└── internal/ # Internal docs + ├── AUDIT_REPORT.md + ├── CLEANUP_SUMMARY.md + └── HONEST_STATUS.md ``` ## Community diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md index bf8abb9f..ce7d32c5 100644 --- a/docs/RELEASE-GUIDE.md +++ b/docs/RELEASE-GUIDE.md @@ -48,7 +48,7 @@ git commit -m "chore: remove unused dependency" # DON'T use BREAKING CHANGE for internal changes git commit -m "feat: improve model delivery -BREAKING CHANGE: removed tar-stream dependency" # WRONG! This triggers v3.0.0 +BREAKING CHANGE: removed tar-stream dependency" # WRONG! This triggers ``` ## Release Workflow Checklist diff --git a/docs/STAGE3-CANONICAL-TAXONOMY.md b/docs/STAGE3-CANONICAL-TAXONOMY.md index b1fab35a..bfafb9e5 100644 --- a/docs/STAGE3-CANONICAL-TAXONOMY.md +++ b/docs/STAGE3-CANONICAL-TAXONOMY.md @@ -1,4 +1,4 @@ -# Brainy Stage 3: Canonical Taxonomy (v6.0.0) +# Brainy Stage 3: Canonical Taxonomy **Status:** FINAL - This is the definitive, timeless taxonomy **Total Types:** 169 (42 nouns + 127 verbs) diff --git a/docs/api/COMPREHENSIVE_API_OVERVIEW.md b/docs/api/COMPREHENSIVE_API_OVERVIEW.md index 3a729be5..fdcc6ebd 100644 --- a/docs/api/COMPREHENSIVE_API_OVERVIEW.md +++ b/docs/api/COMPREHENSIVE_API_OVERVIEW.md @@ -1,6 +1,6 @@ # Brainy Complete Public API Reference -> **Accurate API documentation for Brainy v6.5.0+** +> **Accurate API documentation for Brainy** ## Initialization @@ -13,14 +13,14 @@ await brain.init() // With configuration const brain = new Brainy({ - storage: { type: 'memory' }, // or { path: './my-data' } for filesystem - embeddingModel: 'Q8', // Q4, Q8, F16, F32 - silent: true // Suppress logs + storage: { type: 'memory' }, // or { path: './my-data' } for filesystem + embeddingModel: 'Q8', // Q4, Q8, F16, F32 + silent: true // Suppress logs }) await brain.init() ``` -## Readiness API (v7.3.0+) +## Readiness API For reliable initialization detection, especially in cloud environments with progressive initialization: @@ -28,10 +28,10 @@ For reliable initialization detection, especially in cloud environments with pro ```typescript const brain = new Brainy() -brain.init() // Fire and forget +brain.init() // Fire and forget // Elsewhere (e.g., API handler) -await brain.ready // Wait until init() completes +await brain.ready // Wait until init() completes const results = await brain.find({ query: 'test' }) ``` @@ -39,7 +39,7 @@ const results = await brain.find({ query: 'test' }) ```typescript if (brain.isInitialized) { - // Safe to use brain methods + // Safe to use brain methods } ``` @@ -49,7 +49,7 @@ if (brain.isInitialized) { // Returns true when ALL initialization is complete, including background tasks // Useful for cloud storage adapters with progressive initialization if (brain.isFullyInitialized()) { - console.log('All background tasks complete') + console.log('All background tasks complete') } ``` @@ -57,7 +57,7 @@ if (brain.isFullyInitialized()) { ```typescript const brain = new Brainy({ storage: { type: 'gcs', ... } }) -await brain.init() // Fast return in cloud (<200ms) +await brain.init() // Fast return in cloud (<200ms) // Optional: wait for all background tasks (bucket validation, count sync) await brain.awaitBackgroundInit() @@ -68,15 +68,15 @@ console.log('Fully initialized including background tasks') ```typescript app.get('/health', async (req, res) => { - try { - await brain.ready - res.json({ - status: 'ready', - fullyInitialized: brain.isFullyInitialized() - }) - } catch (error) { - res.status(503).json({ status: 'initializing', error: error.message }) - } + try { + await brain.ready + res.json({ + status: 'ready', + fullyInitialized: brain.isFullyInitialized() + }) + } catch (error) { + res.status(503).json({ status: 'initializing', error: error.message }) + } }) ``` @@ -87,16 +87,16 @@ app.get('/health', async (req, res) => { ```typescript // Add with data (auto-embedded) const id = await brain.add({ - data: 'Machine learning is a subset of AI', - type: NounType.Document, - metadata: { author: 'Alice', tags: ['ml', 'ai'] } + data: 'Machine learning is a subset of AI', + type: NounType.Document, + metadata: { author: 'Alice', tags: ['ml', 'ai'] } }) // Add with pre-computed vector const id = await brain.add({ - data: 'Content here', - vector: [0.1, 0.2, ...], // 384 dimensions - type: NounType.Concept + data: 'Content here', + vector: [0.1, 0.2, ...], // 384 dimensions + type: NounType.Concept }) ``` @@ -120,9 +120,9 @@ const entity = await brain.get('entity-id', { includeVectors: true }) ```typescript await brain.update({ - id: 'entity-id', - data: 'Updated content', // Re-embeds if changed - metadata: { reviewed: true } // Merges with existing + id: 'entity-id', + data: 'Updated content', // Re-embeds if changed + metadata: { reviewed: true } // Merges with existing }) ``` @@ -144,10 +144,10 @@ await brain.clear() ```typescript const relationId = await brain.relate({ - from: 'source-entity-id', - to: 'target-entity-id', - type: VerbType.RelatedTo, - metadata: { strength: 0.9 } + from: 'source-entity-id', + to: 'target-entity-id', + type: VerbType.RelatedTo, + metadata: { strength: 0.9 } }) ``` @@ -168,8 +168,8 @@ const relations = await brain.getRelations({ to: 'entity-id' }) // Filter by type const relations = await brain.getRelations({ - from: 'entity-id', - type: VerbType.Contains + from: 'entity-id', + type: VerbType.Contains }) ``` @@ -191,17 +191,17 @@ const results = await brain.find('machine learning algorithms') // With options const results = await brain.find({ - query: 'machine learning', - limit: 10, - threshold: 0.7, - type: NounType.Document, - where: { author: 'Alice' }, - excludeVFS: true // Exclude VFS files from results + query: 'machine learning', + limit: 10, + threshold: 0.7, + type: NounType.Document, + where: { author: 'Alice' }, + excludeVFS: true // Exclude VFS files from results }) // Natural language query (Triple Intelligence) const results = await brain.find( - 'Show me documents about AI written by Alice in 2024' + 'Show me documents about AI written by Alice in 2024' ) ``` @@ -218,15 +218,15 @@ const results = await brain.find( ```typescript // Find similar to entity ID const similar = await brain.similar({ - to: 'entity-id', - limit: 5 + to: 'entity-id', + limit: 5 }) // Find similar to vector const similar = await brain.similar({ - to: [0.1, 0.2, ...], // Vector - limit: 10, - type: NounType.Document + to: [0.1, 0.2, ...], // Vector + limit: 10, + type: NounType.Document }) ``` @@ -236,15 +236,15 @@ const similar = await brain.similar({ ```typescript const result = await brain.addMany({ - items: [ - { data: 'First item', type: NounType.Document }, - { data: 'Second item', type: NounType.Concept }, - { data: 'Third item', metadata: { priority: 'high' } } - ], - continueOnError: true, // Don't stop on failures - onProgress: (completed, total) => { - console.log(`${completed}/${total} complete`) - } + items: [ + { data: 'First item', type: NounType.Document }, + { data: 'Second item', type: NounType.Concept }, + { data: 'Third item', metadata: { priority: 'high' } } + ], + continueOnError: true, // Don't stop on failures + onProgress: (completed, total) => { + console.log(`${completed}/${total} complete`) + } }) console.log(`Added: ${result.successful.length}, Failed: ${result.failed.length}`) @@ -255,18 +255,18 @@ console.log(`Added: ${result.successful.length}, Failed: ${result.failed.length} ```typescript // Delete by IDs const result = await brain.deleteMany({ - ids: ['id1', 'id2', 'id3'], - continueOnError: true + ids: ['id1', 'id2', 'id3'], + continueOnError: true }) // Delete by type const result = await brain.deleteMany({ - type: NounType.TempData + type: NounType.TempData }) // Delete by metadata filter const result = await brain.deleteMany({ - where: { status: 'archived' } + where: { status: 'archived' } }) ``` @@ -274,12 +274,12 @@ const result = await brain.deleteMany({ ```typescript const ids = await brain.relateMany({ - items: [ - { from: 'a', to: 'b', type: VerbType.RelatedTo }, - { from: 'b', to: 'c', type: VerbType.Contains }, - { from: 'c', to: 'd', type: VerbType.References } - ], - continueOnError: true + items: [ + { from: 'a', to: 'b', type: VerbType.RelatedTo }, + { from: 'b', to: 'c', type: VerbType.Contains }, + { from: 'c', to: 'd', type: VerbType.References } + ], + continueOnError: true }) ``` @@ -287,11 +287,11 @@ const ids = await brain.relateMany({ ```typescript const result = await brain.updateMany({ - items: [ - { id: 'id1', metadata: { reviewed: true } }, - { id: 'id2', data: 'Updated content' } - ], - continueOnError: true + items: [ + { id: 'id1', metadata: { reviewed: true } }, + { id: 'id2', data: 'Updated content' } + ], + continueOnError: true }) ``` @@ -309,8 +309,8 @@ const detailed = await neural.similar('text1', 'text2', { detailed: true }) // Clustering const clusters = await neural.clusters() const clusters = await neural.clusters({ - algorithm: 'hierarchical', - maxClusters: 10 + algorithm: 'hierarchical', + maxClusters: 10 }) // K-nearest neighbors @@ -327,13 +327,13 @@ const domainClusters = await neural.clusterByDomain('category') // Temporal clustering const temporalClusters = await neural.clusterByTime('createdAt', [ - { start: new Date('2024-01-01'), end: new Date('2024-06-30'), label: 'H1' }, - { start: new Date('2024-07-01'), end: new Date('2024-12-31'), label: 'H2' } + { start: new Date('2024-01-01'), end: new Date('2024-06-30'), label: 'H1' }, + { start: new Date('2024-07-01'), end: new Date('2024-12-31'), label: 'H2' } ]) // Streaming clusters (for large datasets) for await (const batch of neural.clusterStream({ batchSize: 100 })) { - console.log(`Progress: ${batch.progress.percentage}%`) + console.log(`Progress: ${batch.progress.percentage}%`) } ``` @@ -355,11 +355,11 @@ await vfs.mkdir('/project/src', { recursive: true }) const files = await vfs.readdir('/project') await vfs.rmdir('/project', { recursive: true }) -// Bulk operations (v6.5.0+) +// Bulk operations const result = await vfs.bulkWrite([ - { type: 'mkdir', path: '/data' }, - { type: 'write', path: '/data/config.json', data: '{}' }, - { type: 'write', path: '/data/users.json', data: '[]' } + { type: 'mkdir', path: '/data' }, + { type: 'write', path: '/data/config.json', data: '{}' }, + { type: 'write', path: '/data/users.json', data: '[]' } ]) // Note: mkdir operations run first (sequentially), then other ops in parallel @@ -414,22 +414,22 @@ const snapshot = await brain.asOf('commit-id') ```typescript // Stream all entities for await (const entity of brain.streaming.entities()) { - console.log(entity.id) + console.log(entity.id) } // Stream with filters for await (const entity of brain.streaming.entities({ - type: NounType.Document, - where: { status: 'active' } + type: NounType.Document, + where: { status: 'active' } })) { - // Process each entity + // Process each entity } // Stream relationships for await (const relation of brain.streaming.relations({ - from: 'entity-id' + from: 'entity-id' })) { - console.log(relation) + console.log(relation) } ``` @@ -438,9 +438,9 @@ for await (const relation of brain.streaming.relations({ ```typescript // Paginated queries const page1 = await brain.pagination.find({ - query: 'machine learning', - page: 1, - pageSize: 20 + query: 'machine learning', + page: 1, + pageSize: 20 }) console.log(`Page ${page1.page} of ${page1.totalPages}`) @@ -448,9 +448,9 @@ console.log(`Total results: ${page1.total}`) // Get next page const page2 = await brain.pagination.find({ - query: 'machine learning', - page: 2, - pageSize: 20 + query: 'machine learning', + page: 2, + pageSize: 20 }) ``` @@ -517,20 +517,20 @@ VerbType.ModifiedBy ```typescript try { - await brain.get('nonexistent-id') + await brain.get('nonexistent-id') } catch (error) { - if (error.message.includes('not found')) { - // Handle missing entity - } + if (error.message.includes('not found')) { + // Handle missing entity + } } // VFS errors use POSIX codes try { - await vfs.readFile('/nonexistent') + await vfs.readFile('/nonexistent') } catch (error) { - if (error.code === 'ENOENT') { - // File not found - } + if (error.code === 'ENOENT') { + // File not found + } } ``` @@ -538,26 +538,26 @@ try { ```typescript const brain = new Brainy({ - // Storage - storage: { - type: 'memory' | 'filesystem', - path: './data', // For filesystem - forceMemoryStorage: false // Force memory even if path exists - }, + // Storage + storage: { + type: 'memory' | 'filesystem', + path: './data', // For filesystem + forceMemoryStorage: false // Force memory even if path exists + }, - // Embeddings - embeddingModel: 'Q8', // Q4, Q8, F16, F32 - dimensions: 384, // Auto-detected + // Embeddings + embeddingModel: 'Q8', // Q4, Q8, F16, F32 + dimensions: 384, // Auto-detected - // Performance - silent: false, // Suppress console output - verbose: false, // Extra logging + // Performance + silent: false, // Suppress console output + verbose: false, // Extra logging - // Augmentations (auto-enabled by default) - augmentations: { - cache: true, - display: true, - metrics: true - } + // Augmentations (auto-enabled by default) + augmentations: { + cache: true, + display: true, + metrics: true + } }) ``` diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 9c8d4967..53da37e5 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -1,4 +1,4 @@ -# Brainy Data Storage Architecture (v5.11.0) +# Brainy Data Storage Architecture **Complete file structure reference for all storage backends** @@ -13,7 +13,7 @@ This document explains how Brainy stores, indexes, and scales data across all st 3. [The 4 Indexes](#3-the-4-indexes) 4. [Sharding Strategy](#4-sharding-strategy) 5. [COW (Copy-on-Write) Architecture](#5-cow-copy-on-write-architecture) -6. [ID-First Storage Architecture (v6.0.0+)](#6-id-first-storage-architecture-v600) +6. [ID-First Storage Architecture](#6-id-first-storage-architecture-v600) 7. [VFS (Virtual File System)](#7-vfs-virtual-file-system) 8. [Storage Backend Mapping](#8-storage-backend-mapping) 9. [Performance Characteristics](#9-performance-characteristics) @@ -438,7 +438,7 @@ Unlike entities and relationships, system metadata consists of **index files** t Understanding how Brainy constructs storage paths is critical for debugging and optimization. -### Path Construction Steps (v6.0.0+) +### Path Construction Steps **For an entity (noun)**: ```typescript @@ -498,7 +498,7 @@ const fieldIndexPath = `_system/metadata_indexes/__metadata_field_index__${field const hnswNodePath = `_system/hnsw/nodes/${shard}/${entityId}.json` ``` -### Path Patterns Summary (v6.0.0+) +### Path Patterns Summary | Data Type | Path Pattern | Sharded? | Branched? | |-----------|--------------|----------|-----------| @@ -516,7 +516,7 @@ const hnswNodePath = `_system/hnsw/nodes/${shard}/${entityId}.json` | **HNSW node** | `_system/hnsw/nodes/{shard}/{uuid}.json` | ✅ Yes (UUID) | ❌ No | | **Field index** | `_system/metadata_indexes/__metadata_field_index__{field}.json` | ❌ No | ❌ No | -### Key Principles (v6.0.0+) +### Key Principles 1. **Shard Extraction**: Always use first 2 hex characters of UUID/SHA-256 2. **ID-First**: Shard + ID come BEFORE type (type is in metadata) @@ -549,7 +549,7 @@ Brainy uses four complementary index systems for different query patterns. - Memory (standard): ~200MB per 100K entities - Memory (lazy): ~15-33MB per 100K entities (5-10x less!) -**Automatic Lazy Mode** (v3.36.0+): Enables automatically when vectors don't fit in UnifiedCache +**Automatic Lazy Mode**: Enables automatically when vectors don't fit in UnifiedCache --- @@ -559,7 +559,7 @@ Brainy uses four complementary index systems for different query patterns. **Location**: MetadataIndexManager index on `noun` field **Data Structure**: RoaringBitmap32 per type value -**How It Works** (v6.0.0+): +**How It Works**: ```typescript // Find all Person entities const people = await brain.getNouns({ type: 'person' }) @@ -721,7 +721,7 @@ COW is Brainy's **git-like versioning system** that enables: - ✅ **Deduplication** (identical data stored only once) - ✅ **Version history** (full audit trail of all changes) -**Status**: ALWAYS ENABLED (v5.11.0+) - cannot be disabled +**Status**: ALWAYS ENABLED - cannot be disabled --- @@ -782,19 +782,19 @@ _cow/blobs/ab/abc123...sha256.bin (used by both entities) --- -## 6. ID-First Storage Architecture (v6.0.0+) +## 6. ID-First Storage Architecture ### 6.1 What is ID-First? **ID-first storage** organizes entities by **ID shard only** - no type directories! This eliminates 42-type sequential searches that caused 20-21 second delays on cloud storage. -**Old type-first structure** (v5.4.0-v5.12.0): +**Old type-first structure** (v5.12.0): ``` branches/main/entities/nouns/{TYPE}/metadata/00/001234...uuid.json # Problem: Requires knowing type OR searching 42 type directories! ``` -**NEW ID-first structure** (v6.0.0+): +**NEW ID-first structure**: ``` branches/main/entities/nouns/00/001234...uuid/metadata.json # Direct O(1) lookup - no type needed! @@ -809,7 +809,7 @@ branches/main/entities/nouns/00/001234...uuid/metadata.json // v5.x: Had to search 42 types if type unknown // Result: 21 seconds on GCS (42 types × 500ms) -// v6.0.0: Direct path from ID +// Direct path from ID const id = '001234...' const shard = id.substring(0, 2) // '00' const path = `branches/main/entities/nouns/${shard}/${id}/metadata.json` @@ -1165,7 +1165,7 @@ opfs://root/brainy/ ### 10.2 clear() Operation -**What clear() deletes** (v5.11.0+): +**What clear() deletes**: ✅ Deletes: - `branches/` → ALL entity data (all types, all shards, all branches, all forks) @@ -1184,7 +1184,7 @@ opfs://root/brainy/ **Example**: ```typescript -await brain.storage.clear() // ✅ Deletes ALL data correctly (v5.11.0+) +await brain.storage.clear() // ✅ Deletes ALL data correctly await brain.add({ data: 'Alice', type: 'person' }) // ✅ COW reinitializes automatically ``` @@ -1353,7 +1353,7 @@ const historicalData = await yesterday.getNouns({ type: 'Character' }) await brain.storage.clear() ``` -**What happens in storage** (v5.11.0+): +**What happens in storage**: ``` 1. Delete all entity data: → Remove: branches/ (entire directory) @@ -1452,7 +1452,7 @@ await brain.addBatch([ ## 11. Summary -**Complete Storage Structure (v6.0.0)**: +**Complete Storage Structure**: - **3 storage layers**: branches/ (data), _cow/ (versions), _system/ (indexes) - **2 files per entity**: metadata.json + vector.json (optimized I/O) - **4 indexes**: HNSW (semantic), Type Index (metadata-based), Graph (relationships), Metadata (fields) diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md index 9ae8c5cf..854e2627 100644 --- a/docs/architecture/index-architecture.md +++ b/docs/architecture/index-architecture.md @@ -25,7 +25,7 @@ Brainy has **3 main indexes** at the top level, each with multiple sub-indexes m - **FieldTypeInference** - DuckDB-inspired value-based field type detection - **Field Sparse Indexes** - Per-field sparse indexes with roaring bitmaps (dynamic count) - **Sorted Indexes** - Support orderBy queries (automatically maintained) -- **Word Index (`__words__`)** - Text search via FNV-1a word hashes (v7.7.0) +- **Word Index (`__words__`)** - Text search via FNV-1a word hashes **GraphAdjacencyIndex contains:** - **lsmTreeSource** - Source → Targets (outgoing edges) @@ -39,7 +39,7 @@ All indexes share a **UnifiedCache** for coordinated memory management, ensuring **Purpose**: Enable O(1) field-value lookups and O(log n) range queries on metadata fields using adaptive chunked sparse indexing. -### Internal Architecture (v3.42.0) +### Internal Architecture ```typescript class MetadataIndexManager { @@ -64,7 +64,7 @@ class MetadataIndexManager { ### Key Data Structures -#### Chunked Sparse Index (NEW in v3.42.0) +#### Chunked Sparse Index ```typescript // SparseIndex: Directory of chunks for a field // Example: field="status" @@ -87,7 +87,7 @@ interface ChunkDescriptor { class ChunkData { chunkId: number field: string - entries: Map // ~50 values per chunk (v3.43.0: roaring bitmaps!) + entries: Map // ~50 values per chunk (roaring bitmaps!) } ``` @@ -96,7 +96,7 @@ class ChunkData { - O(log n) range queries with zone maps - 630x file reduction (560k flat files → 89 chunk files) -#### Roaring Bitmap Optimization (NEW in v3.43.0) +#### Roaring Bitmap Optimization **Problem Solved**: JavaScript `Set` for storing entity IDs was inefficient: - Memory overhead: ~40 bytes per UUID string (36 chars + overhead) @@ -150,7 +150,7 @@ class ChunkData { **Multi-Field Intersection (THE BIG WIN!)**: ```typescript -// Before (v3.42.0): JavaScript array filtering +// Before: JavaScript array filtering async getIdsForFilter(filter: {status: 'active', role: 'admin'}): Promise { // 1. Fetch UUID arrays for each field const statusIds = await this.getIds('status', 'active') // ["uuid1", "uuid2", ...] @@ -160,7 +160,7 @@ async getIdsForFilter(filter: {status: 'active', role: 'admin'}): Promise roleIds.includes(id)) // O(n*m) array filtering } -// After (v3.43.0): Roaring bitmap intersection +// After: Roaring bitmap intersection async getIdsForMultipleFields(pairs: [{field, value}, ...]): Promise { // 1. Fetch roaring bitmaps (integers, not UUIDs) const bitmaps: RoaringBitmap32[] = [] @@ -229,7 +229,7 @@ interface ZoneMap { **Use case**: Enables NLP to understand "find characters named John" → knows 'name' is a character field -#### Word Index (`__words__`) - v7.7.0 +#### Word Index (`__words__`) - ```typescript // Special field for text/keyword search // Entity text content is tokenized and indexed as word hashes @@ -253,14 +253,14 @@ interface ZoneMap { - **Lowercase normalization**: Case-insensitive matching - **Automatic integration**: Words extracted via `extractIndexableFields()` -**Hybrid Search** (v7.7.0): Text results combined with vector results using Reciprocal Rank Fusion (RRF): +**Hybrid Search**: Text results combined with vector results using Reciprocal Rank Fusion (RRF): ```typescript // RRF formula: score(d) = sum(1 / (k + rank(d))) // where k = 60 (standard constant) // alpha = weight for semantic (0 = text only, 1 = semantic only) ``` -### Query Algorithm (v3.42.0) +### Query Algorithm **Exact Match Query**: ```typescript @@ -317,7 +317,7 @@ async getIdsForRange(field: string, min: any, max: any): Promise { - Adaptive chunking: ~50 values per chunk optimizes I/O - Immediate flushing: No need for dirty tracking or batch writes -### Temporal Bucketing (v3.41.0) +### Temporal Bucketing **Problem Solved**: High-cardinality timestamp fields created massive file pollution. - Example: 575 entities with unique timestamps → 358,407 index files (98.7% pollution!) @@ -396,7 +396,7 @@ const DEFAULT_EXCLUDE_FIELDS = [ ] ``` -**Note**: Timestamp fields like `modified`, `accessed`, `created` are NO LONGER excluded as of v3.41.0 - they are indexed with automatic bucketing. +**Note**: Timestamp fields like `modified`, `accessed`, `created` are NO LONGER excluded as of they are indexed with automatic bucketing. ## 2. HNSWIndex - Vector Similarity Search @@ -735,7 +735,7 @@ async stats(): Promise { } ``` -### 5. Index Rebuilding (v5.7.7: Lazy Loading Support) +### 5. Index Rebuilding (Lazy Loading Support) **Two modes of index loading:** @@ -762,7 +762,7 @@ async init(): Promise { } ``` -#### Mode 2: Lazy Loading on First Query (v5.7.7+) +#### Mode 2: Lazy Loading on First Query ```typescript // When disableAutoRebuild: true @@ -918,7 +918,7 @@ All indexes scale gracefully: - [Performance Guide](../PERFORMANCE.md) - Performance tuning - [Overview](./overview.md) - High-level architecture -## Summary: Index Hierarchy (v5.7.7) +## Summary: Index Hierarchy ### Level 1: Main Indexes (3) All have rebuild() methods and are covered by lazy loading: @@ -933,7 +933,7 @@ Automatically managed by parent rebuild(): - **4 LSM-trees** (lsmTreeSource, lsmTreeTarget, lsmTreeVerbsBySource, lsmTreeVerbsByTarget) - **In-memory graph structures** (sourceIndex, targetIndex, verbIndex) -### Lazy Loading (v5.7.7) +### Lazy Loading - **Mode 1**: Auto-rebuild on init() (default) - **Mode 2**: Lazy rebuild on first query (when `disableAutoRebuild: true`) - **Concurrency-safe**: Mutex prevents duplicate rebuilds diff --git a/docs/architecture/initialization-and-rebuild.md b/docs/architecture/initialization-and-rebuild.md index e7f74693..ab4e3281 100644 --- a/docs/architecture/initialization-and-rebuild.md +++ b/docs/architecture/initialization-and-rebuild.md @@ -15,7 +15,7 @@ This document explains how Brainy's four indexes (MetadataIndex, HNSWIndex, Grap | **GraphAdjacencyIndex** | Relationships via LSM-tree SSTables | LSM-tree auto-persistence | v3.44.0 | | **DeletedItemsIndex** | Set of deleted IDs | `storage.saveDeletedItems()` | v3.0.0 | -#### MetadataIndex Persistence Details (v4.2.1+) +#### MetadataIndex Persistence Details The MetadataIndex now persists two components: @@ -129,7 +129,7 @@ async init(): Promise { - 100-2000ms: One-time rebuild to create indices - Total: ~1-3 seconds (one time only) -#### Mode 2: Lazy Loading on First Query (v5.7.7+) +#### Mode 2: Lazy Loading on First Query When `disableAutoRebuild: true`, indexes remain empty after init() and rebuild on first query: @@ -350,7 +350,7 @@ public async rebuild(options?: { **Performance Impact**: 200-600x speedup (5 minutes → 500ms for 10K entities) -**Correct Pattern** (v3.45.0): +**Correct Pattern**: ```typescript // Load ALL nouns ONCE (not 31 times!) while (hasMore) { @@ -392,7 +392,7 @@ while (hasMore) { ```typescript // src/utils/metadataIndex.ts (lines 202-216) async init(): Promise { - // STEP 1: Load field registry to discover persisted indices (v4.2.1) + // STEP 1: Load field registry to discover persisted indices // This is THE KEY FIX - O(1) discovery of existing indices await this.loadFieldRegistry() diff --git a/docs/architecture/storage-architecture.md b/docs/architecture/storage-architecture.md index 5ccdba14..5c71d50c 100644 --- a/docs/architecture/storage-architecture.md +++ b/docs/architecture/storage-architecture.md @@ -1,40 +1,40 @@ -# Storage Architecture (v4.0.0) +# Storage Architecture -> **Updated for v4.0.0**: Metadata/vector separation, UUID-based sharding, lifecycle management +> **Updated**: Metadata/vector separation, UUID-based sharding, lifecycle management ## Storage Structure -### v4.0.0 Architecture: Metadata/Vector Separation +### Architecture: Metadata/Vector Separation -In v4.0.0, entities and relationships are split into **2 separate files** for optimal performance at billion-entity scale: +entities and relationships are split into **2 separate files** for optimal performance at billion-entity scale: ``` brainy-data/ -├── _system/ # System metadata (not sharded) -│ ├── statistics.json # Performance metrics -│ ├── __metadata_field_index__*.json # Field indexes -│ └── __metadata_sorted_index__*.json # Sorted indexes +├── _system/ # System metadata (not sharded) +│ ├── statistics.json # Performance metrics +│ ├── __metadata_field_index__*.json # Field indexes +│ └── __metadata_sorted_index__*.json # Sorted indexes │ ├── entities/ -│ ├── nouns/ -│ │ ├── vectors/ # HNSW graph data (sharded by UUID) -│ │ │ ├── 00/ # Shard 00 (first 2 hex digits) -│ │ │ │ ├── 00123456-....json # Vector + HNSW connections -│ │ │ │ └── 00abcdef-....json -│ │ │ ├── 01/ ... ff/ # 256 shards total -│ │ │ -│ │ └── metadata/ # Business data (sharded by UUID) -│ │ ├── 00/ -│ │ │ ├── 00123456-....json # Entity metadata only -│ │ │ └── 00abcdef-....json -│ │ ├── 01/ ... ff/ -│ │ -│ └── verbs/ -│ ├── vectors/ # Relationship vectors (sharded) -│ │ ├── 00/ ... ff/ -│ │ -│ └── metadata/ # Relationship data (sharded) -│ ├── 00/ ... ff/ +│ ├── nouns/ +│ │ ├── vectors/ # HNSW graph data (sharded by UUID) +│ │ │ ├── 00/ # Shard 00 (first 2 hex digits) +│ │ │ │ ├── 00123456-....json # Vector + HNSW connections +│ │ │ │ └── 00abcdef-....json +│ │ │ ├── 01/ ... ff/ # 256 shards total +│ │ │ +│ │ └── metadata/ # Business data (sharded by UUID) +│ │ ├── 00/ +│ │ │ ├── 00123456-....json # Entity metadata only +│ │ │ └── 00abcdef-....json +│ │ ├── 01/ ... ff/ +│ │ +│ └── verbs/ +│ ├── vectors/ # Relationship vectors (sharded) +│ │ ├── 00/ ... ff/ +│ │ +│ └── metadata/ # Relationship data (sharded) +│ ├── 00/ ... ff/ ``` ### Why Split Metadata and Vectors? @@ -50,9 +50,9 @@ brainy-data/ **How it works:** ```typescript const uuid = "3fa85f64-5717-4562-b3fc-2c963f66afa6" -const shard = uuid.substring(0, 2) // "3f" +const shard = uuid.substring(0, 2) // "3f" -// Vector path: entities/nouns/vectors/3f/3fa85f64-....json +// Vector path: entities/nouns/vectors/3f/3fa85f64-....json // Metadata path: entities/nouns/metadata/3f/3fa85f64-....json ``` @@ -64,103 +64,103 @@ const shard = uuid.substring(0, 2) // "3f" ## Storage Adapters -Brainy provides multiple storage adapters with identical APIs and v4.0.0 production features: +Brainy provides multiple storage adapters with identical APIs and production features: ### FileSystem Storage (Node.js) ```typescript const brain = new Brainy({ - storage: { - type: 'filesystem', - path: './data', - compression: true // v4.0.0: Gzip compression (60-80% space savings) - } + storage: { + type: 'filesystem', + path: './data', + compression: true // Gzip compression (60-80% space savings) + } }) ``` - **Use case**: Server applications, CLI tools - **Performance**: Direct file I/O with optional compression - **Persistence**: Permanent on disk -- **v4.0.0 Features**: - - **Gzip Compression**: 60-80% storage savings with minimal CPU overhead - - **Batch Delete**: Efficient bulk deletion with retries - - **UUID Sharding**: Automatic 256-shard distribution +- **Features**: + - **Gzip Compression**: 60-80% storage savings with minimal CPU overhead + - **Batch Delete**: Efficient bulk deletion with retries + - **UUID Sharding**: Automatic 256-shard distribution ### S3 Compatible Storage (AWS, MinIO, R2) ```typescript const brain = new Brainy({ - storage: { - type: 's3', - bucket: 'my-brainy-data', - region: 'us-east-1', - credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY - } - } + storage: { + type: 's3', + bucket: 'my-brainy-data', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + } + } }) ``` - **Use case**: Distributed applications, cloud deployments - **Performance**: Network dependent, with intelligent caching - **Persistence**: Cloud storage durability (99.999999999%) -- **v4.0.0 Features**: - - **Lifecycle Policies**: Automatic tier transitions (Standard → IA → Glacier → Deep Archive) - - **Intelligent-Tiering**: Automatic optimization based on access patterns (up to 95% savings) - - **Batch Delete**: Efficient bulk deletion (1000 objects per request) - - **Cost Impact**: $138k/year → $5.9k/year at 500TB (96% savings!) +- **Features**: + - **Lifecycle Policies**: Automatic tier transitions (Standard → IA → Glacier → Deep Archive) + - **Intelligent-Tiering**: Automatic optimization based on access patterns (up to 95% savings) + - **Batch Delete**: Efficient bulk deletion (1000 objects per request) + - **Cost Impact**: $138k/year → $5.9k/year at 500TB (96% savings!) ### Google Cloud Storage (GCS) ```typescript const brain = new Brainy({ - storage: { - type: 'gcs', - bucketName: 'my-brainy-data', - keyFilename: './service-account.json' // Or use ADC - } + storage: { + type: 'gcs', + bucketName: 'my-brainy-data', + keyFilename: './service-account.json' // Or use ADC + } }) ``` - **Use case**: Google Cloud deployments - **Performance**: Global CDN with edge caching - **Persistence**: 99.999999999% durability -- **v4.0.0 Features**: - - **Lifecycle Policies**: Automatic tier transitions (Standard → Nearline → Coldline → Archive) - - **Autoclass**: Intelligent automatic tier optimization - - **Batch Delete**: Efficient bulk operations - - **Cost Impact**: $138k/year → $8.3k/year at 500TB (94% savings!) +- **Features**: + - **Lifecycle Policies**: Automatic tier transitions (Standard → Nearline → Coldline → Archive) + - **Autoclass**: Intelligent automatic tier optimization + - **Batch Delete**: Efficient bulk operations + - **Cost Impact**: $138k/year → $8.3k/year at 500TB (94% savings!) ### Azure Blob Storage ```typescript const brain = new Brainy({ - storage: { - type: 'azure', - connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, - containerName: 'brainy-data' - } + storage: { + type: 'azure', + connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, + containerName: 'brainy-data' + } }) ``` - **Use case**: Azure cloud deployments - **Performance**: Global replication with CDN - **Persistence**: LRS, ZRS, GRS, RA-GRS options -- **v4.0.0 Features**: - - **Blob Tier Management**: Hot/Cool/Archive tiers (99% cost savings) - - **Lifecycle Policies**: Automatic tier transitions and deletions - - **Batch Delete**: BlobBatchClient for efficient bulk operations - - **Batch Tier Changes**: Move thousands of blobs efficiently - - **Archive Rehydration**: Smart rehydration with priority options +- **Features**: + - **Blob Tier Management**: Hot/Cool/Archive tiers (99% cost savings) + - **Lifecycle Policies**: Automatic tier transitions and deletions + - **Batch Delete**: BlobBatchClient for efficient bulk operations + - **Batch Tier Changes**: Move thousands of blobs efficiently + - **Archive Rehydration**: Smart rehydration with priority options ### Origin Private File System (Browser) ```typescript const brain = new Brainy({ - storage: { - type: 'opfs' - } + storage: { + type: 'opfs' + } }) ``` - **Use case**: Browser applications, PWAs - **Performance**: Near-native file system speed - **Persistence**: Permanent in browser (with quota limits) -- **v4.0.0 Features**: - - **Quota Monitoring**: Real-time quota tracking and warnings - - **Batch Delete**: Efficient bulk deletion - - **Storage Status**: Detailed usage/available reporting +- **Features**: + - **Quota Monitoring**: Real-time quota tracking and warnings + - **Batch Delete**: Efficient bulk deletion + - **Storage Status**: Detailed usage/available reporting ## Metadata Indexing System @@ -170,12 +170,12 @@ Tracks all unique values for each field: ```json // __metadata_field_index__field_category.json { - "values": { - "technology": 45, - "science": 32, - "business": 28 - }, - "lastUpdated": 1699564234567 + "values": { + "technology": 45, + "science": 32, + "business": 28 + }, + "lastUpdated": 1699564234567 } ``` @@ -185,11 +185,11 @@ Maps field+value combinations to entity IDs: ```json // __metadata_index__category_technology_chunk0.json { - "field": "category", - "value": "technology", - "ids": ["uuid1", "uuid2", "uuid3", ...], - "chunk": 0, - "total": 45 + "field": "category", + "value": "technology", + "ids": ["uuid1", "uuid2", "uuid3", ...], + "chunk": 0, + "total": 45 } ``` @@ -207,14 +207,14 @@ High-performance deduplication system for streaming data: ```json // __entity_registry__.json { - "mappings": { - "did:plc:alice123": "550e8400-e29b-41d4-a716-446655440000", - "handle:alice.bsky.social": "550e8400-e29b-41d4-a716-446655440000" - }, - "stats": { - "totalMappings": 10000, - "lastSync": 1699564234567 - } + "mappings": { + "did:plc:alice123": "550e8400-e29b-41d4-a716-446655440000", + "handle:alice.bsky.social": "550e8400-e29b-41d4-a716-446655440000" + }, + "stats": { + "totalMappings": 10000, + "lastSync": 1699564234567 + } } ``` @@ -229,14 +229,14 @@ Ensures durability and enables recovery: ```json { - "timestamp": 1699564234567, - "operation": "add", - "data": { - "id": "550e8400-e29b-41d4-a716-446655440000", - "content": "...", - "metadata": {} - }, - "checksum": "sha256:..." + "timestamp": 1699564234567, + "operation": "add", + "data": { + "id": "550e8400-e29b-41d4-a716-446655440000", + "content": "...", + "metadata": {} + }, + "checksum": "sha256:..." } ``` @@ -244,7 +244,7 @@ Ensures durability and enables recovery: 2. Replay operations from last checkpoint 3. Verify checksums for integrity -## Storage Optimization (v4.0.0) +## Storage Optimization ### 1. Lifecycle Policies (Cloud Storage) @@ -253,48 +253,48 @@ Ensures durability and enables recovery: ```typescript // S3: Set lifecycle policy for automatic archival await storage.setLifecyclePolicy({ - rules: [{ - id: 'archive-old-data', - prefix: 'entities/', - status: 'Enabled', - transitions: [ - { days: 30, storageClass: 'STANDARD_IA' }, // Move to IA after 30 days - { days: 90, storageClass: 'GLACIER' }, // Archive after 90 days - { days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep archive after 1 year - ] - }] + rules: [{ + id: 'archive-old-data', + prefix: 'entities/', + status: 'Enabled', + transitions: [ + { days: 30, storageClass: 'STANDARD_IA' }, // Move to IA after 30 days + { days: 90, storageClass: 'GLACIER' }, // Archive after 90 days + { days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep archive after 1 year + ] + }] }) // GCS: Set lifecycle policy await storage.setLifecyclePolicy({ - rules: [{ - condition: { age: 30 }, - action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } - }, { - condition: { age: 90 }, - action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } - }, { - condition: { age: 365 }, - action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } - }] + rules: [{ + condition: { age: 30 }, + action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } + }, { + condition: { age: 90 }, + action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } + }, { + condition: { age: 365 }, + action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } + }] }) // Azure: Set lifecycle policy await storage.setLifecyclePolicy({ - rules: [{ - name: 'archiveOldData', - enabled: true, - type: 'Lifecycle', - definition: { - filters: { blobTypes: ['blockBlob'] }, - actions: { - baseBlob: { - tierToCool: { daysAfterModificationGreaterThan: 30 }, - tierToArchive: { daysAfterModificationGreaterThan: 90 } - } - } - } - }] + rules: [{ + name: 'archiveOldData', + enabled: true, + type: 'Lifecycle', + definition: { + filters: { blobTypes: ['blockBlob'] }, + actions: { + baseBlob: { + tierToCool: { daysAfterModificationGreaterThan: 30 }, + tierToArchive: { daysAfterModificationGreaterThan: 90 } + } + } + } + }] }) ``` @@ -327,7 +327,7 @@ await storage.enableIntelligentTiering('entities/', 'auto-optimize') ```typescript // Enable GCS Autoclass await storage.enableAutoclass({ - terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier + terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier }) // Benefits: @@ -342,11 +342,11 @@ await storage.enableAutoclass({ ```typescript // Enable gzip compression for local storage const brain = new Brainy({ - storage: { - type: 'filesystem', - path: './data', - compression: true // 60-80% space savings - } + storage: { + type: 'filesystem', + path: './data', + compression: true // 60-80% space savings + } }) // Performance impact: @@ -359,11 +359,11 @@ const brain = new Brainy({ ### 5. Batch Operations ```typescript -// v4.0.0: Efficient batch delete +// Efficient batch delete await storage.batchDelete([ - 'entities/nouns/vectors/00/00123456-....json', - 'entities/nouns/metadata/00/00123456-....json', - // ... up to 1000 objects + 'entities/nouns/vectors/00/00123456-....json', + 'entities/nouns/metadata/00/00123456-....json', + // ... up to 1000 objects ]) // Benefits: @@ -375,9 +375,9 @@ await storage.batchDelete([ // Batch writes for performance await brain.addBatch([ - { content: "item1", metadata: {} }, - { content: "item2", metadata: {} }, - { content: "item3", metadata: {} } + { content: "item1", metadata: {} }, + { content: "item2", metadata: {} }, + { content: "item3", metadata: {} } ]) // Single transaction, optimized I/O ``` @@ -390,14 +390,14 @@ const status = await storage.getStorageStatus() console.log(status) // { -// type: 'opfs', -// available: true, -// details: { -// usage: 45829120, // 43.7 MB used -// quota: 536870912, // 512 MB available -// usagePercent: 8.5, -// quotaExceeded: false -// } +// type: 'opfs', +// available: true, +// details: { +// usage: 45829120, // 43.7 MB used +// quota: 536870912, // 512 MB available +// usagePercent: 8.5, +// quotaExceeded: false +// } // } // Proactive quota management: @@ -410,14 +410,14 @@ console.log(status) ```typescript // Change blob tier for cost optimization -await storage.changeBlobTier(blobPath, 'Cool') // Hot → Cool (50% savings) -await storage.changeBlobTier(blobPath, 'Archive') // Cool → Archive (99% savings) +await storage.changeBlobTier(blobPath, 'Cool') // Hot → Cool (50% savings) +await storage.changeBlobTier(blobPath, 'Archive') // Cool → Archive (99% savings) // Batch tier changes (efficient) await storage.batchChangeTier([blob1, blob2, blob3], 'Cool') // Rehydrate from Archive when needed -await storage.rehydrateBlob(blobPath, 'Standard') // Standard or High priority +await storage.rehydrateBlob(blobPath, 'Standard') // Standard or High priority ``` ### 8. Caching Strategy @@ -425,15 +425,15 @@ await storage.rehydrateBlob(blobPath, 'Standard') // Standard or High priority ```typescript // Configure caching per storage type const brain = new Brainy({ - storage: { - type: 'filesystem', - cache: { - enabled: true, - maxSize: 1000, // Maximum cached items - ttl: 300000, // 5 minutes - strategy: 'lru' // Least recently used - } - } + storage: { + type: 'filesystem', + cache: { + enabled: true, + maxSize: 1000, // Maximum cached items + ttl: 300000, // 5 minutes + strategy: 'lru' // Least recently used + } + } }) ``` @@ -443,8 +443,8 @@ const brain = new Brainy({ ```typescript // Automatic locking for write operations await brain.storage.withLock('resource-id', async () => { - // Exclusive access to resource - await brain.storage.saveNoun(id, data) + // Exclusive access to resource + await brain.storage.saveNoun(id, data) }) ``` @@ -459,9 +459,9 @@ await brain.storage.withLock('resource-id', async () => { ```typescript // Export entire database const backup = await brain.export({ - format: 'json', - includeVectors: true, - includeIndexes: false + format: 'json', + includeVectors: true, + includeIndexes: false }) ``` @@ -469,8 +469,8 @@ const backup = await brain.export({ ```typescript // Import from backup await brain.import(backup, { - mode: 'merge', // or 'replace' - validateSchema: true + mode: 'merge', // or 'replace' + validateSchema: true }) ``` @@ -514,15 +514,15 @@ await newBrain.import(data) const stats = await brain.storage.getStatistics() console.log(stats) // { -// totalSize: 1048576, -// entityCount: 1000, -// indexSize: 204800, -// walSize: 10240, -// cacheHitRate: 0.85 +// totalSize: 1048576, +// entityCount: 1000, +// indexSize: 204800, +// walSize: 10240, +// cacheHitRate: 0.85 // } ``` -## Best Practices (v4.0.0) +## Best Practices ### Choose the Right Adapter 1. **Development**: FileSystem with compression (local persistence, small storage footprint) @@ -537,7 +537,7 @@ console.log(stats) 4. **Archival**: Cloud storage with lifecycle policies (96% cost savings!) 5. **Large-scale**: Metadata/vector separation + UUID sharding + lifecycle policies -### v4.0.0 Cost Optimization +### Cost Optimization 1. **Enable lifecycle policies** for cloud storage (automated cost reduction) 2. **Use Intelligent-Tiering (S3)** or Autoclass (GCS) for automatic optimization 3. **Enable compression** for FileSystem storage (60-80% space savings) @@ -547,7 +547,7 @@ console.log(stats) **Example Cost Savings (500TB dataset):** - Without lifecycle policies: **$138,000/year** -- With v4.0.0 lifecycle policies: **$5,940/year** +- With lifecycle policies: **$5,940/year** - **Savings: $132,060/year (96%)** ### Monitor and Maintain diff --git a/docs/augmentations/COMPLETE-REFERENCE.md b/docs/augmentations/COMPLETE-REFERENCE.md index 73ef6173..3b427214 100644 --- a/docs/augmentations/COMPLETE-REFERENCE.md +++ b/docs/augmentations/COMPLETE-REFERENCE.md @@ -1,8 +1,8 @@ -# 🔌 Brainy v4.0.0 Augmentations Complete Reference +# 🔌 Brainy Augmentations Complete Reference > **All augmentations that power Brainy's extensibility - with locations, usage, and examples** > -> **⚠️ v4.0.0 Update**: Updated for metadata structure changes and billion-scale optimizations +> **⚠️ Update**: Updated for metadata structure changes and billion-scale optimizations ## Quick Start @@ -10,37 +10,37 @@ import { Brainy } from '@soulcraft/brainy' const brain = new Brainy({ - // Augmentations auto-configure based on environment - storage: 'auto', // Storage augmentation - cache: true, // Cache augmentation - index: true // Index augmentation + // Augmentations auto-configure based on environment + storage: 'auto', // Storage augmentation + cache: true, // Cache augmentation + index: true // Index augmentation }) -await brain.init() // Augmentations initialize automatically +await brain.init() // Augmentations initialize automatically ``` -## v4.0.0 Augmentation Architecture +## Augmentation Architecture ### Key Improvements for Billion-Scale Performance 1. **Metadata/Vector Separation**: Augmentations now work with separated metadata and vectors - - Metadata stored separately from vector data - - 99.2% memory reduction for type tracking - - Two-file storage pattern for optimal I/O + - Metadata stored separately from vector data + - 99.2% memory reduction for type tracking + - Two-file storage pattern for optimal I/O 2. **Type System Enforcement**: All metadata requires type fields - - `NounMetadata` requires `noun: NounType` - - `VerbMetadata` requires `verb: VerbType` - - Type inference system available as public API + - `NounMetadata` requires `noun: NounType` + - `VerbMetadata` requires `verb: VerbType` + - Type inference system available as public API 3. **Storage Adapter Pattern**: Internal vs public method distinction - - `_methods`: Return pure structures (HNSWNoun, HNSWVerb) - - Public methods: Return WithMetadata types - - MetadataEnforcer Proxy ensures proper access + - `_methods`: Return pure structures (HNSWNoun, HNSWVerb) + - Public methods: Return WithMetadata types + - MetadataEnforcer Proxy ensures proper access ### What This Means for Augmentation Users -**✅ If you use built-in augmentations**: No changes needed! They're all updated for v4.0.0. +**✅ If you use built-in augmentations**: No changes needed! They're all updated. **⚠️ If you created custom storage augmentations**: Update your storage adapter to: - Wrap metadata with required `noun`/`verb` fields @@ -69,81 +69,81 @@ Augmentations are modular extensions that add functionality to Brainy without cl ## Storage Augmentations (8 total) ### MemoryStorageAugmentation -**Location**: `src/augmentations/storageAugmentations.ts` -**Auto-enabled**: When `storage: 'memory'` or in test environments +**Location**: `src/augmentations/storageAugmentations.ts` +**Auto-enabled**: When `storage: 'memory'` or in test environments **Purpose**: In-memory storage for testing and temporary data ```typescript const brain = new Brainy({ storage: 'memory' }) ``` -### FileSystemStorageAugmentation -**Location**: `src/augmentations/storageAugmentations.ts` -**Auto-enabled**: When `storage: 'filesystem'` or Node.js detected +### FileSystemStorageAugmentation +**Location**: `src/augmentations/storageAugmentations.ts` +**Auto-enabled**: When `storage: 'filesystem'` or Node.js detected **Purpose**: Persistent file-based storage for Node.js applications ```typescript -const brain = new Brainy({ - storage: { type: 'filesystem', path: './data' } +const brain = new Brainy({ + storage: { type: 'filesystem', path: './data' } }) ``` ### OPFSStorageAugmentation -**Location**: `src/augmentations/storageAugmentations.ts` -**Auto-enabled**: When `storage: 'opfs'` or browser with OPFS support +**Location**: `src/augmentations/storageAugmentations.ts` +**Auto-enabled**: When `storage: 'opfs'` or browser with OPFS support **Purpose**: Browser-based persistent storage using Origin Private File System ```typescript const brain = new Brainy({ storage: 'opfs' }) ``` ### S3StorageAugmentation -**Location**: `src/augmentations/storageAugmentations.ts` -**Manual**: Requires AWS credentials +**Location**: `src/augmentations/storageAugmentations.ts` +**Manual**: Requires AWS credentials **Purpose**: AWS S3-compatible cloud storage ```typescript -const brain = new Brainy({ - storage: { - type: 's3', - bucket: 'my-bucket', - region: 'us-east-1', - credentials: { accessKeyId, secretAccessKey } - } +const brain = new Brainy({ + storage: { + type: 's3', + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { accessKeyId, secretAccessKey } + } }) ``` ### R2StorageAugmentation -**Location**: `src/augmentations/storageAugmentations.ts` -**Manual**: Requires Cloudflare credentials +**Location**: `src/augmentations/storageAugmentations.ts` +**Manual**: Requires Cloudflare credentials **Purpose**: Cloudflare R2 storage (S3-compatible) ```typescript -const brain = new Brainy({ - storage: { - type: 'r2', - accountId: 'xxx', - bucket: 'my-bucket', - credentials: { accessKeyId, secretAccessKey } - } +const brain = new Brainy({ + storage: { + type: 'r2', + accountId: 'xxx', + bucket: 'my-bucket', + credentials: { accessKeyId, secretAccessKey } + } }) ``` ### GCSStorageAugmentation -**Location**: `src/augmentations/storageAugmentations.ts` -**Manual**: Requires Google Cloud credentials +**Location**: `src/augmentations/storageAugmentations.ts` +**Manual**: Requires Google Cloud credentials **Purpose**: Google Cloud Storage ```typescript -const brain = new Brainy({ - storage: { - type: 'gcs', - bucket: 'my-bucket', - projectId: 'my-project' - } +const brain = new Brainy({ + storage: { + type: 'gcs', + bucket: 'my-bucket', + projectId: 'my-project' + } }) ``` ### StorageAugmentation (base) -**Location**: `src/augmentations/storageAugmentation.ts` +**Location**: `src/augmentations/storageAugmentation.ts` **Purpose**: Base class for custom storage implementations ### DynamicStorageAugmentation -**Location**: `src/augmentations/storageAugmentation.ts` +**Location**: `src/augmentations/storageAugmentation.ts` **Purpose**: Runtime storage adapter switching --- @@ -151,17 +151,17 @@ const brain = new Brainy({ ## Performance Augmentations (7 total) ### CacheAugmentation -**Location**: `src/augmentations/cacheAugmentation.ts` -**Auto-enabled**: When `cache: true` (default) +**Location**: `src/augmentations/cacheAugmentation.ts` +**Auto-enabled**: When `cache: true` (default) **Purpose**: LRU cache for search results and frequent queries ```typescript -brain.clearCache() // Exposed via API -brain.getCacheStats() // Cache hit/miss statistics +brain.clearCache() // Exposed via API +brain.getCacheStats() // Cache hit/miss statistics ``` ### IndexAugmentation -**Location**: `src/augmentations/indexAugmentation.ts` -**Auto-enabled**: When `index: true` (default) +**Location**: `src/augmentations/indexAugmentation.ts` +**Auto-enabled**: When `index: true` (default) **Purpose**: Metadata indexing for O(1) field lookups ```typescript brain.rebuildMetadataIndex() // Exposed via API @@ -170,41 +170,41 @@ brain.find({ where: { category: 'tech' } }) ``` ### MetricsAugmentation -**Location**: `src/augmentations/metricsAugmentation.ts` -**Auto-enabled**: Always active +**Location**: `src/augmentations/metricsAugmentation.ts` +**Auto-enabled**: Always active **Purpose**: Performance metrics and statistics collection ```typescript -brain.getStats() // Comprehensive metrics +brain.getStats() // Comprehensive metrics ``` ### MonitoringAugmentation -**Location**: `src/augmentations/monitoringAugmentation.ts` -**Manual**: Register for detailed monitoring +**Location**: `src/augmentations/monitoringAugmentation.ts` +**Manual**: Register for detailed monitoring **Purpose**: Real-time performance monitoring and alerts ### BatchProcessingAugmentation -**Location**: `src/augmentations/batchProcessingAugmentation.ts` -**Auto-enabled**: For batch operations +**Location**: `src/augmentations/batchProcessingAugmentation.ts` +**Auto-enabled**: For batch operations **Purpose**: Optimizes bulk add/update/delete operations ```typescript -brain.addNouns([...]) // Automatically batched +brain.addNouns([...]) // Automatically batched ``` ### RequestDeduplicatorAugmentation -**Location**: `src/augmentations/requestDeduplicatorAugmentation.ts` -**Auto-enabled**: Always active +**Location**: `src/augmentations/requestDeduplicatorAugmentation.ts` +**Auto-enabled**: Always active **Purpose**: Prevents duplicate concurrent operations ### ConnectionPoolAugmentation -**Location**: `src/augmentations/connectionPoolAugmentation.ts` -**Auto-enabled**: For network storage +**Location**: `src/augmentations/connectionPoolAugmentation.ts` +**Auto-enabled**: For network storage **Purpose**: Connection pooling for cloud storage adapters --- ## Data Integrity Augmentations (3 total) -**Auto-enabled**: When `wal: true` +**Auto-enabled**: When `wal: true` **Purpose**: Write-ahead logging for crash recovery ```typescript const brain = new Brainy({ wal: true }) @@ -212,8 +212,8 @@ const brain = new Brainy({ wal: true }) ``` ### EntityRegistryAugmentation -**Location**: `src/augmentations/entityRegistryAugmentation.ts` -**Auto-enabled**: For streaming operations +**Location**: `src/augmentations/entityRegistryAugmentation.ts` +**Auto-enabled**: For streaming operations **Purpose**: High-speed deduplication for real-time data ```typescript // Prevents duplicate entities in streaming scenarios @@ -221,8 +221,8 @@ brain.add(data) // Automatically deduplicated ``` ### AutoRegisterEntitiesAugmentation -**Location**: `src/augmentations/entityRegistryAugmentation.ts` -**Manual**: For automatic entity discovery +**Location**: `src/augmentations/entityRegistryAugmentation.ts` +**Manual**: For automatic entity discovery **Purpose**: Auto-discovers and registers entities from data --- @@ -230,20 +230,20 @@ brain.add(data) // Automatically deduplicated ## Intelligence Augmentations (2 total) ### NeuralImportAugmentation -**Location**: `src/augmentations/neuralImport.ts` -**Manual**: Via `brain.neuralImport()` +**Location**: `src/augmentations/neuralImport.ts` +**Manual**: Via `brain.neuralImport()` **Purpose**: AI-powered smart data import ```typescript const result = await brain.neuralImport(data, { - confidenceThreshold: 0.7, - autoApply: true + confidenceThreshold: 0.7, + autoApply: true }) // Automatically detects entities and relationships ``` ### IntelligentVerbScoringAugmentation -**Location**: `src/augmentations/intelligentVerbScoringAugmentation.ts` -**Auto-enabled**: When verbs are used +**Location**: `src/augmentations/intelligentVerbScoringAugmentation.ts` +**Auto-enabled**: When verbs are used **Purpose**: ML-based relationship strength scoring ```typescript brain.verbScoring.train(feedback) @@ -255,8 +255,8 @@ brain.verbScoring.getScore(verbId) ## Communication Augmentations (4 total) ### APIServerAugmentation -**Location**: `src/augmentations/apiServerAugmentation.ts` -**Manual**: For server deployments +**Location**: `src/augmentations/apiServerAugmentation.ts` +**Manual**: For server deployments **Purpose**: REST/WebSocket/MCP API server ```typescript const augmentation = new APIServerAugmentation() @@ -265,8 +265,8 @@ await brain.registerAugmentation(augmentation) ``` ### WebSocketConduitAugmentation -**Location**: `src/augmentations/conduitAugmentations.ts` -**Manual**: For Brainy-to-Brainy sync +**Location**: `src/augmentations/conduitAugmentations.ts` +**Manual**: For Brainy-to-Brainy sync **Purpose**: Real-time sync between Brainy instances ```typescript const conduit = new WebSocketConduitAugmentation() @@ -274,13 +274,13 @@ await conduit.establishConnection('ws://other-brain') ``` ### ServerSearchConduitAugmentation -**Location**: `src/augmentations/serverSearchAugmentations.ts` -**Manual**: For client-server search +**Location**: `src/augmentations/serverSearchAugmentations.ts` +**Manual**: For client-server search **Purpose**: Search remote Brainy instance, cache locally ### ServerSearchActivationAugmentation -**Location**: `src/augmentations/serverSearchAugmentations.ts` -**Manual**: Works with ServerSearchConduit +**Location**: `src/augmentations/serverSearchAugmentations.ts` +**Manual**: Works with ServerSearchConduit **Purpose**: Triggers and manages server search operations --- @@ -288,18 +288,18 @@ await conduit.establishConnection('ws://other-brain') ## External Integration (2 total) ### SynapseAugmentation (base) -**Location**: `src/augmentations/synapseAugmentation.ts` +**Location**: `src/augmentations/synapseAugmentation.ts` **Purpose**: Base class for external platform integrations ```typescript // Example: NotionSynapse, SlackSynapse, etc. class NotionSynapse extends SynapseAugmentation { - async fetchData() { /* Notion API calls */ } - async pushData() { /* Sync to Notion */ } + async fetchData() { /* Notion API calls */ } + async pushData() { /* Sync to Notion */ } } ``` ### ExampleFileSystemSynapse -**Location**: `src/augmentations/synapseAugmentation.ts` +**Location**: `src/augmentations/synapseAugmentation.ts` **Purpose**: Example implementation for file system sync --- @@ -309,11 +309,11 @@ class NotionSynapse extends SynapseAugmentation { ### Auto-Configuration ```typescript const brain = new Brainy({ - // These auto-register augmentations: - storage: 'auto', // Storage augmentation - cache: true, // Cache augmentation - index: true, // Index augmentation - metrics: true // Metrics augmentation + // These auto-register augmentations: + storage: 'auto', // Storage augmentation + cache: true, // Cache augmentation + index: true, // Index augmentation + metrics: true // Metrics augmentation }) ``` @@ -333,29 +333,29 @@ await brain.init() import { BaseAugmentation } from '@soulcraft/brainy' class MyAugmentation extends BaseAugmentation { - readonly name = 'my-augmentation' - readonly timing = 'after' // before | after | both - readonly operations = ['addNoun', 'search'] // Which ops to hook - readonly priority = 10 // Execution order (lower = earlier) - - protected async onInit(): Promise { - // Initialize your augmentation - } - - async execute( - operation: string, - params: any, - context?: AugmentationContext - ): Promise { - // Your augmentation logic - if (operation === 'addNoun') { - console.log('Noun added:', params) - } - } - - protected async onShutdown(): Promise { - // Cleanup - } + readonly name = 'my-augmentation' + readonly timing = 'after' // before | after | both + readonly operations = ['addNoun', 'search'] // Which ops to hook + readonly priority = 10 // Execution order (lower = earlier) + + protected async onInit(): Promise { + // Initialize your augmentation + } + + async execute( + operation: string, + params: any, + context?: AugmentationContext + ): Promise { + // Your augmentation logic + if (operation === 'addNoun') { + console.log('Noun added:', params) + } + } + + protected async onShutdown(): Promise { + // Cleanup + } } ``` diff --git a/docs/augmentations/DEVELOPER-GUIDE.md b/docs/augmentations/DEVELOPER-GUIDE.md index 568967a0..50467313 100644 --- a/docs/augmentations/DEVELOPER-GUIDE.md +++ b/docs/augmentations/DEVELOPER-GUIDE.md @@ -1,10 +1,10 @@ # 🛠️ Brainy Augmentation Developer Guide -> **How to create, test, and use augmentations in Brainy v4.0.0** +> **How to create, test, and use augmentations in Brainy** > -> **⚠️ v4.0.0 Update**: This guide has been updated with breaking changes for metadata structure and type system improvements. +> **⚠️ Update**: This guide has been updated with breaking changes for metadata structure and type system improvements. -## v4.0.0 Migration Guide +## Migration Guide ### What Changed? @@ -25,22 +25,22 @@ ```typescript // ❌ v3.x const verb = { - type: 'relatedTo', - sourceId: 'a', - targetId: 'b' + type: 'relatedTo', + sourceId: 'a', + targetId: 'b' } if (verb.type === 'relatedTo') { ... } -// ✅ v4.0.0 +// ✅ Current const verb = { - verb: 'relatedTo', - sourceId: 'a', - targetId: 'b' + verb: 'relatedTo', + sourceId: 'a', + targetId: 'b' } const metadata: VerbMetadata = { - verb: 'relatedTo', - sourceId: 'a', - targetId: 'b' + verb: 'relatedTo', + sourceId: 'a', + targetId: 'b' } if (verb.verb === 'relatedTo') { ... } ``` @@ -51,40 +51,40 @@ if (verb.verb === 'relatedTo') { ... } import { BaseAugmentation, BrainyAugmentation, AugmentationContext } from '@soulcraft/brainy' export class MyFirstAugmentation extends BaseAugmentation { - readonly name = 'my-first-augmentation' - readonly timing = 'after' as const // When to run: before | after | both - readonly operations = ['add'] as const // Which operations to hook - readonly priority = 10 // Execution order (lower = first) + readonly name = 'my-first-augmentation' + readonly timing = 'after' as const // When to run: before | after | both + readonly operations = ['add'] as const // Which operations to hook + readonly priority = 10 // Execution order (lower = first) - protected async onInit(): Promise { - // Initialize your augmentation - console.log('MyFirstAugmentation initialized!') - } + protected async onInit(): Promise { + // Initialize your augmentation + console.log('MyFirstAugmentation initialized!') + } - async execute( - operation: string, - params: any, - context?: AugmentationContext - ): Promise { - // Your augmentation logic - if (operation === 'add') { - console.log('Noun added:', params.noun) + async execute( + operation: string, + params: any, + context?: AugmentationContext + ): Promise { + // Your augmentation logic + if (operation === 'add') { + console.log('Noun added:', params.noun) - // v4.0.0: Access metadata correctly - if (params.noun?.metadata) { - console.log('Noun type:', params.noun.metadata.noun) // Required field - } + // Access metadata correctly + if (params.noun?.metadata) { + console.log('Noun type:', params.noun.metadata.noun) // Required field + } - // You can access the brain instance - const stats = await context?.brain.getStats() - console.log('Total nouns:', stats.totalNouns) - } - } + // You can access the brain instance + const stats = await context?.brain.getStats() + console.log('Total nouns:', stats.totalNouns) + } + } - protected async onShutdown(): Promise { - // Cleanup - console.log('MyFirstAugmentation shutting down') - } + protected async onShutdown(): Promise { + // Cleanup + console.log('MyFirstAugmentation shutting down') + } } ``` @@ -113,23 +113,23 @@ await brain.add('Hello World') ### 1. Registration Phase ```typescript const aug = new MyAugmentation() -brain.augmentations.register(aug) // Before brain.init()! +brain.augmentations.register(aug) // Before brain.init()! ``` ### 2. Initialization Phase ```typescript -await brain.init() // Calls aug.initialize() internally +await brain.init() // Calls aug.initialize() internally // Your onInit() method runs here ``` ### 3. Execution Phase ```typescript -await brain.add('data') // Your execute() method runs +await brain.add('data') // Your execute() method runs ``` ### 4. Shutdown Phase ```typescript -await brain.shutdown() // Your onShutdown() method runs +await brain.shutdown() // Your onShutdown() method runs ``` --- @@ -139,52 +139,52 @@ await brain.shutdown() // Your onShutdown() method runs ### `before` - Modify Input ```typescript class ValidationAugmentation extends BaseAugmentation { - readonly timing = 'before' as const - - async execute(operation: string, params: any): Promise { - if (operation === 'add') { - // Validate and/or modify params - if (!params.content) { - throw new Error('Content required') - } - // Return modified params - return { ...params, validated: true } - } - } + readonly timing = 'before' as const + + async execute(operation: string, params: any): Promise { + if (operation === 'add') { + // Validate and/or modify params + if (!params.content) { + throw new Error('Content required') + } + // Return modified params + return { ...params, validated: true } + } + } } ``` ### `after` - React to Results ```typescript class LoggingAugmentation extends BaseAugmentation { - readonly timing = 'after' as const - - async execute(operation: string, params: any): Promise { - if (operation === 'search') { - console.log(`Search for "${params.query}" returned ${params.result.length} results`) - } - // Don't return anything - just observe - } + readonly timing = 'after' as const + + async execute(operation: string, params: any): Promise { + if (operation === 'search') { + console.log(`Search for "${params.query}" returned ${params.result.length} results`) + } + // Don't return anything - just observe + } } ``` ### `both` - Before AND After ```typescript class TimingAugmentation extends BaseAugmentation { - readonly timing = 'both' as const - private startTime?: number - - async execute(operation: string, params: any, context?: AugmentationContext): Promise { - if (!this.startTime) { - // Before execution - this.startTime = Date.now() - } else { - // After execution - const duration = Date.now() - this.startTime - console.log(`${operation} took ${duration}ms`) - this.startTime = undefined - } - } + readonly timing = 'both' as const + private startTime?: number + + async execute(operation: string, params: any, context?: AugmentationContext): Promise { + if (!this.startTime) { + // Before execution + this.startTime = Date.now() + } else { + // After execution + const duration = Date.now() - this.startTime + console.log(`${operation} took ${duration}ms`) + this.startTime = undefined + } + } } ``` @@ -195,28 +195,28 @@ class TimingAugmentation extends BaseAugmentation { ### Core Operations You Can Hook ```typescript readonly operations = [ - 'add', // Adding data - 'update', // Updating data - 'delete', // Deleting data - 'get', // Retrieving data - 'search', // Searching - 'find', // Triple Intelligence queries - 'relate', // Adding relationships - 'unrelate', // Removing relationships - 'clear', // Clearing data - 'all' // Hook ALL operations + 'add', // Adding data + 'update', // Updating data + 'delete', // Deleting data + 'get', // Retrieving data + 'search', // Searching + 'find', // Triple Intelligence queries + 'relate', // Adding relationships + 'unrelate', // Removing relationships + 'clear', // Clearing data + 'all' // Hook ALL operations ] as const ``` ### Example: Multi-Operation Hook ```typescript class AuditAugmentation extends BaseAugmentation { - readonly operations = ['add', 'update', 'delete'] as const - - async execute(operation: string, params: any): Promise { - // Log all data modifications - await this.logToAuditTrail(operation, params) - } + readonly operations = ['add', 'update', 'delete'] as const + + async execute(operation: string, params: any): Promise { + // Log all data modifications + await this.logToAuditTrail(operation, params) + } } ``` @@ -226,26 +226,26 @@ class AuditAugmentation extends BaseAugmentation { ```typescript class ContextAwareAugmentation extends BaseAugmentation { - async execute( - operation: string, - params: any, - context?: AugmentationContext - ): Promise { - // Access the brain instance - const brain = context?.brain - if (!brain) return - - // Use any brain method - const stats = await brain.getStats() - const size = await brain.size() - const results = await brain.search('query') - - // Access other augmentations - const cache = brain.augmentations.get('cache') - if (cache) { - await cache.clear() - } - } + async execute( + operation: string, + params: any, + context?: AugmentationContext + ): Promise { + // Access the brain instance + const brain = context?.brain + if (!brain) return + + // Use any brain method + const stats = await brain.getStats() + const size = await brain.size() + const results = await brain.search('query') + + // Access other augmentations + const cache = brain.augmentations.get('cache') + if (cache) { + await cache.clear() + } + } } ``` @@ -256,92 +256,92 @@ class ContextAwareAugmentation extends BaseAugmentation { ### 1. Backup Augmentation ```typescript class BackupAugmentation extends BaseAugmentation { - readonly name = 'backup' - readonly timing = 'after' as const - readonly operations = ['add', 'update', 'delete'] as const - readonly priority = 5 - - private changes = 0 - private readonly backupThreshold = 100 - - async execute(operation: string, params: any, context?: AugmentationContext): Promise { - this.changes++ - - if (this.changes >= this.backupThreshold) { - await this.performBackup(context?.brain) - this.changes = 0 - } - } - - private async performBackup(brain?: any): Promise { - if (!brain) return - // Create instant COW snapshot - const snapshotName = `backup-${Date.now()}` - await brain.fork(snapshotName) - console.log(`Automatic snapshot created: ${snapshotName}`) - } + readonly name = 'backup' + readonly timing = 'after' as const + readonly operations = ['add', 'update', 'delete'] as const + readonly priority = 5 + + private changes = 0 + private readonly backupThreshold = 100 + + async execute(operation: string, params: any, context?: AugmentationContext): Promise { + this.changes++ + + if (this.changes >= this.backupThreshold) { + await this.performBackup(context?.brain) + this.changes = 0 + } + } + + private async performBackup(brain?: any): Promise { + if (!brain) return + // Create instant COW snapshot + const snapshotName = `backup-${Date.now()}` + await brain.fork(snapshotName) + console.log(`Automatic snapshot created: ${snapshotName}`) + } } ``` ### 2. Rate Limiting Augmentation ```typescript class RateLimitAugmentation extends BaseAugmentation { - readonly name = 'rate-limit' - readonly timing = 'before' as const - readonly operations = ['search', 'find'] as const - readonly priority = 100 // High priority - run first - - private requests = new Map() - private readonly limit = 100 // 100 requests - private readonly window = 60000 // per minute - - async execute(operation: string, params: any): Promise { - const now = Date.now() - const key = params.userId || 'anonymous' - - // Get request timestamps - const timestamps = this.requests.get(key) || [] - - // Remove old timestamps - const recent = timestamps.filter(t => now - t < this.window) - - // Check limit - if (recent.length >= this.limit) { - throw new Error('Rate limit exceeded') - } - - // Add current request - recent.push(now) - this.requests.set(key, recent) - } + readonly name = 'rate-limit' + readonly timing = 'before' as const + readonly operations = ['search', 'find'] as const + readonly priority = 100 // High priority - run first + + private requests = new Map() + private readonly limit = 100 // 100 requests + private readonly window = 60000 // per minute + + async execute(operation: string, params: any): Promise { + const now = Date.now() + const key = params.userId || 'anonymous' + + // Get request timestamps + const timestamps = this.requests.get(key) || [] + + // Remove old timestamps + const recent = timestamps.filter(t => now - t < this.window) + + // Check limit + if (recent.length >= this.limit) { + throw new Error('Rate limit exceeded') + } + + // Add current request + recent.push(now) + this.requests.set(key, recent) + } } ``` ### 3. Encryption Augmentation ```typescript class EncryptionAugmentation extends BaseAugmentation { - readonly name = 'encryption' - readonly timing = 'both' as const - readonly operations = ['add', 'get'] as const - readonly priority = 90 // Run early - - async execute(operation: string, params: any): Promise { - if (operation === 'add') { - // Encrypt before storing - if (params.metadata?.sensitive) { - params.content = await this.encrypt(params.content) - params.encrypted = true - } - return params - } + readonly name = 'encryption' + readonly timing = 'both' as const + readonly operations = ['add', 'get'] as const + readonly priority = 90 // Run early - if (operation === 'get' && params.result?.encrypted) { - // Decrypt after retrieval - params.result.content = await this.decrypt(params.result.content) - delete params.result.encrypted - return params.result - } - } + async execute(operation: string, params: any): Promise { + if (operation === 'add') { + // Encrypt before storing + if (params.metadata?.sensitive) { + params.content = await this.encrypt(params.content) + params.encrypted = true + } + return params + } + + if (operation === 'get' && params.result?.encrypted) { + // Decrypt after retrieval + params.result.content = await this.decrypt(params.result.content) + delete params.result.encrypted + return params.result + } + } } ``` @@ -355,26 +355,26 @@ import { Brainy } from '@soulcraft/brainy' import { MyAugmentation } from './my-augmentation' describe('MyAugmentation', () => { - it('should hook into addNoun', async () => { - const brain = new Brainy({ storage: 'memory' }) - const aug = new MyAugmentation() - - // Spy on the execute method - const executeSpy = vi.spyOn(aug, 'execute') - - brain.augmentations.register(aug) - await brain.init() - - // Trigger the augmentation - await brain.add('test data') + it('should hook into addNoun', async () => { + const brain = new Brainy({ storage: 'memory' }) + const aug = new MyAugmentation() - // Verify it was called - expect(executeSpy).toHaveBeenCalledWith( - 'add', - expect.objectContaining({ content: 'test data' }), - expect.any(Object) - ) - }) + // Spy on the execute method + const executeSpy = vi.spyOn(aug, 'execute') + + brain.augmentations.register(aug) + await brain.init() + + // Trigger the augmentation + await brain.add('test data') + + // Verify it was called + expect(executeSpy).toHaveBeenCalledWith( + 'add', + expect.objectContaining({ content: 'test data' }), + expect.any(Object) + ) + }) }) ``` @@ -391,61 +391,61 @@ describe('MyAugmentation', () => { ```typescript // Priority guidelines 100: Critical (auth, rate limiting) -50: Important (validation, transformation) -10: Normal (logging, metrics) -1: Optional (debugging, tracing) +50: Important (validation, transformation) +10: Normal (logging, metrics) +1: Optional (debugging, tracing) ``` ### 3. Handle Errors Gracefully ```typescript async execute(operation: string, params: any): Promise { - try { - await this.riskyOperation() - } catch (error) { - // Log but don't break the main operation - console.error(`Augmentation error in ${this.name}:`, error) - // Optionally report to monitoring - this.reportError(error) - } + try { + await this.riskyOperation() + } catch (error) { + // Log but don't break the main operation + console.error(`Augmentation error in ${this.name}:`, error) + // Optionally report to monitoring + this.reportError(error) + } } ``` ### 4. Be Performance Conscious ```typescript class CachedAugmentation extends BaseAugmentation { - private cache = new Map() - - async execute(operation: string, params: any): Promise { - const key = this.getCacheKey(params) - - // Check cache first - if (this.cache.has(key)) { - return this.cache.get(key) - } - - // Expensive operation - const result = await this.expensiveOperation(params) - this.cache.set(key, result) - - return result - } + private cache = new Map() + + async execute(operation: string, params: any): Promise { + const key = this.getCacheKey(params) + + // Check cache first + if (this.cache.has(key)) { + return this.cache.get(key) + } + + // Expensive operation + const result = await this.expensiveOperation(params) + this.cache.set(key, result) + + return result + } } ``` ### 5. Clean Up Resources ```typescript protected async onShutdown(): Promise { - // Close connections - await this.connection?.close() - - // Clear intervals - clearInterval(this.interval) - - // Flush buffers - await this.flush() - - // Clear caches - this.cache.clear() + // Close connections + await this.connection?.close() + + // Clear intervals + clearInterval(this.interval) + + // Flush buffers + await this.flush() + + // Clear caches + this.cache.clear() } ``` @@ -457,10 +457,10 @@ protected async onShutdown(): Promise { ``` my-augmentation/ ├── src/ -│ └── index.ts # Your augmentation -├── dist/ # Built output +│ └── index.ts # Your augmentation +├── dist/ # Built output ├── tests/ -│ └── augmentation.test.ts +│ └── augmentation.test.ts ├── package.json ├── tsconfig.json └── README.md @@ -469,21 +469,21 @@ my-augmentation/ ### package.json ```json { - "name": "@mycompany/brainy-custom-augmentation", - "version": "1.0.0", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "keywords": ["brainy-augmentation"], - "peerDependencies": { - "@soulcraft/brainy": ">=2.0.0" - }, - "brainy": { - "type": "augmentation", - "class": "CustomAugmentation", - "timing": "after", - "operations": ["add"], - "priority": 10 - } + "name": "@mycompany/brainy-custom-augmentation", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "keywords": ["brainy-augmentation"], + "peerDependencies": { + "@soulcraft/brainy": ">=2.0.0" + }, + "brainy": { + "type": "augmentation", + "class": "CustomAugmentation", + "timing": "after", + "operations": ["add"], + "priority": 10 + } } ``` @@ -492,7 +492,7 @@ my-augmentation/ # Coming in 2.1+ npm run build npm test -brainy publish # Publishes to brain-cloud registry +brainy publish # Publishes to brain-cloud registry ``` --- diff --git a/docs/deployment/CLOUD_DEPLOYMENT_GUIDE.md b/docs/deployment/CLOUD_DEPLOYMENT_GUIDE.md index 751e1d6a..a1ce2c82 100644 --- a/docs/deployment/CLOUD_DEPLOYMENT_GUIDE.md +++ b/docs/deployment/CLOUD_DEPLOYMENT_GUIDE.md @@ -1,8 +1,8 @@ -# Brainy v4.0.0 Cloud Deployment Guide +# Brainy Cloud Deployment Guide -This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy v4.0.0 codebase. +This guide provides production-ready deployment configurations for Brainy using S3CompatibleStorage (preferred) or FileSystemStorage across major cloud platforms. All examples are verified against the actual Brainy codebase. -## 🆕 v4.0.0 Production Features +## 🆕 Production Features **Cost Optimization at Scale:** - **Lifecycle Policies**: Automatic tier transitions for 96% cost savings @@ -12,7 +12,7 @@ This guide provides production-ready deployment configurations for Brainy using **Example Impact (500TB dataset):** - Before: $138,000/year -- With v4.0.0 lifecycle policies: $5,940/year +- With lifecycle policies: $5,940/year - **Savings: $132,060/year (96%)** See the [Cost Optimization](#cost-optimization-v40) section below for implementation details. @@ -43,53 +43,53 @@ let brain let handler exports.handler = async (event) => { - if (!brain) { - const storage = new S3CompatibleStorage({ - endpoint: 's3.amazonaws.com', - region: process.env.AWS_REGION, - bucket: process.env.BRAINY_BUCKET, - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, - prefix: 'brainy-data/' - }) - - brain = new Brainy({ - storage, - augmentations: [{ - name: 'api-server', - config: { - enabled: true, - auth: { - required: true, - apiKeys: [process.env.API_KEY] - } - } - }] - }) - - await brain.init() - - // Get the universal handler from the augmentation - const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') - handler = apiAugmentation.createUniversalHandler() - } - - // Convert Lambda event to Request - const url = `https://${event.requestContext.domainName}${event.rawPath}` - const request = new Request(url, { - method: event.requestContext.http.method, - headers: event.headers, - body: event.body - }) - - // Use the universal handler - const response = await handler(request) - - return { - statusCode: response.status, - headers: Object.fromEntries(response.headers), - body: await response.text() - } + if (!brain) { + const storage = new S3CompatibleStorage({ + endpoint: 's3.amazonaws.com', + region: process.env.AWS_REGION, + bucket: process.env.BRAINY_BUCKET, + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + prefix: 'brainy-data/' + }) + + brain = new Brainy({ + storage, + augmentations: [{ + name: 'api-server', + config: { + enabled: true, + auth: { + required: true, + apiKeys: [process.env.API_KEY] + } + } + }] + }) + + await brain.init() + + // Get the universal handler from the augmentation + const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') + handler = apiAugmentation.createUniversalHandler() + } + + // Convert Lambda event to Request + const url = `https://${event.requestContext.domainName}${event.rawPath}` + const request = new Request(url, { + method: event.requestContext.http.method, + headers: event.headers, + body: event.body + }) + + // Use the universal handler + const response = await handler(request) + + return { + statusCode: response.status, + headers: Object.fromEntries(response.headers), + body: await response.text() + } } ``` @@ -104,36 +104,36 @@ let brain let handler exports.brainyAPI = async (req, res) => { - if (!brain) { - const storage = new S3CompatibleStorage({ - endpoint: 'storage.googleapis.com', - bucket: process.env.GCS_BUCKET, - accessKeyId: process.env.GCS_ACCESS_KEY, - secretAccessKey: process.env.GCS_SECRET_KEY, - prefix: 'brainy/', - forcePathStyle: false, - region: 'US' - }) - - brain = new Brainy({ storage }) - await brain.init() - - const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') - handler = apiAugmentation.createUniversalHandler() - } - - // Convert Express req/res to Request/Response - const request = new Request(`https://${req.hostname}${req.originalUrl}`, { - method: req.method, - headers: req.headers, - body: JSON.stringify(req.body) - }) - - const response = await handler(request) - - res.status(response.status) - res.set(Object.fromEntries(response.headers)) - res.send(await response.text()) + if (!brain) { + const storage = new S3CompatibleStorage({ + endpoint: 'storage.googleapis.com', + bucket: process.env.GCS_BUCKET, + accessKeyId: process.env.GCS_ACCESS_KEY, + secretAccessKey: process.env.GCS_SECRET_KEY, + prefix: 'brainy/', + forcePathStyle: false, + region: 'US' + }) + + brain = new Brainy({ storage }) + await brain.init() + + const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') + handler = apiAugmentation.createUniversalHandler() + } + + // Convert Express req/res to Request/Response + const request = new Request(`https://${req.hostname}${req.originalUrl}`, { + method: req.method, + headers: req.headers, + body: JSON.stringify(req.body) + }) + + const response = await handler(request) + + res.status(response.status) + res.set(Object.fromEntries(response.headers)) + res.send(await response.text()) } ``` @@ -156,39 +156,39 @@ let brain let handler async function initBrainy() { - // Google Cloud Storage is S3-compatible - const storage = new S3CompatibleStorage({ - endpoint: 'storage.googleapis.com', - bucket: process.env.GCS_BUCKET || 'brainy-data', - accessKeyId: process.env.GCS_ACCESS_KEY, - secretAccessKey: process.env.GCS_SECRET_KEY, - prefix: 'brainy/', - forcePathStyle: false, - region: process.env.GCS_REGION || 'US' - }) - - brain = new Brainy({ - storage, - augmentations: [{ - name: 'api-server', - config: { - enabled: true, - port: PORT, - cors: { - origin: process.env.CORS_ORIGIN || '*' - }, - auth: { - required: process.env.AUTH_REQUIRED === 'true', - apiKeys: process.env.API_KEY ? [process.env.API_KEY] : [] - } - } - }] - }) - - await brain.init() - - const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') - handler = apiAugmentation.createUniversalHandler() + // Google Cloud Storage is S3-compatible + const storage = new S3CompatibleStorage({ + endpoint: 'storage.googleapis.com', + bucket: process.env.GCS_BUCKET || 'brainy-data', + accessKeyId: process.env.GCS_ACCESS_KEY, + secretAccessKey: process.env.GCS_SECRET_KEY, + prefix: 'brainy/', + forcePathStyle: false, + region: process.env.GCS_REGION || 'US' + }) + + brain = new Brainy({ + storage, + augmentations: [{ + name: 'api-server', + config: { + enabled: true, + port: PORT, + cors: { + origin: process.env.CORS_ORIGIN || '*' + }, + auth: { + required: process.env.AUTH_REQUIRED === 'true', + apiKeys: process.env.API_KEY ? [process.env.API_KEY] : [] + } + } + }] + }) + + await brain.init() + + const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') + handler = apiAugmentation.createUniversalHandler() } // Initialize on startup @@ -196,26 +196,26 @@ await initBrainy() // Health check endpoint app.get('/health', (req, res) => { - res.json({ status: 'healthy', service: 'brainy-api' }) + res.json({ status: 'healthy', service: 'brainy-api' }) }) // Universal handler for all API routes app.use('*', async (req, res) => { - const request = new Request(`https://${req.hostname}${req.originalUrl}`, { - method: req.method, - headers: req.headers, - body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined - }) - - const response = await handler(request) - - res.status(response.status) - res.set(Object.fromEntries(response.headers)) - res.send(await response.text()) + const request = new Request(`https://${req.hostname}${req.originalUrl}`, { + method: req.method, + headers: req.headers, + body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined + }) + + const response = await handler(request) + + res.status(response.status) + res.set(Object.fromEntries(response.headers)) + res.send(await response.text()) }) app.listen(PORT, () => { - console.log(`Brainy API running on port ${PORT}`) + console.log(`Brainy API running on port ${PORT}`) }) ``` @@ -237,43 +237,43 @@ CMD ["node", "server.js"] ```yaml # cloudbuild.yaml steps: - # Build the container image - - name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA', '.'] - - # Push to Container Registry - - name: 'gcr.io/cloud-builders/docker' - args: ['push', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'] - - # Deploy to Cloud Run - - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' - entrypoint: gcloud - args: - - 'run' - - 'deploy' - - 'brainy-api' - - '--image' - - 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA' - - '--region' - - 'us-central1' - - '--platform' - - 'managed' - - '--allow-unauthenticated' - - '--set-env-vars' - - 'GCS_BUCKET=brainy-storage,GCS_REGION=US' - - '--set-secrets' - - 'GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest' - - '--memory' - - '2Gi' - - '--cpu' - - '2' - - '--max-instances' - - '100' - - '--min-instances' - - '0' + # Build the container image + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA', '.'] + + # Push to Container Registry + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA'] + + # Deploy to Cloud Run + - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + entrypoint: gcloud + args: + - 'run' + - 'deploy' + - 'brainy-api' + - '--image' + - 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA' + - '--region' + - 'us-central1' + - '--platform' + - 'managed' + - '--allow-unauthenticated' + - '--set-env-vars' + - 'GCS_BUCKET=brainy-storage,GCS_REGION=US' + - '--set-secrets' + - 'GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest' + - '--memory' + - '2Gi' + - '--cpu' + - '2' + - '--max-instances' + - '100' + - '--min-instances' + - '0' images: - - 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA' + - 'gcr.io/$PROJECT_ID/brainy-api:$COMMIT_SHA' ``` **Deploy with gcloud CLI:** @@ -283,12 +283,12 @@ gcloud builds submit --config cloudbuild.yaml # Or deploy directly gcloud run deploy brainy-api \ - --source . \ - --platform managed \ - --region us-central1 \ - --allow-unauthenticated \ - --set-env-vars GCS_BUCKET=brainy-storage \ - --set-secrets "GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest" + --source . \ + --platform managed \ + --region us-central1 \ + --allow-unauthenticated \ + --set-env-vars GCS_BUCKET=brainy-storage \ + --set-secrets "GCS_ACCESS_KEY=gcs-access-key:latest,GCS_SECRET_KEY=gcs-secret-key:latest" ``` **Create GCS Bucket with S3-compatible access:** @@ -312,37 +312,37 @@ echo -n "YOUR_SECRET_KEY" | gcloud secrets create gcs-secret-key --data-file=- ```javascript // index.js module.exports = async function (context, req) { - const { Brainy } = require('@soulcraft/brainy') - const { S3CompatibleStorage } = require('@soulcraft/brainy/storage') - - const storage = new S3CompatibleStorage({ - endpoint: `${process.env.AZURE_STORAGE_ACCOUNT}.blob.core.windows.net`, - bucket: 'brainy-data', - accessKeyId: process.env.AZURE_STORAGE_ACCOUNT, - secretAccessKey: process.env.AZURE_STORAGE_KEY, - prefix: 'entities/', - forcePathStyle: false - }) - - const brain = new Brainy({ storage }) - await brain.init() - - const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') - const handler = apiAugmentation.createUniversalHandler() - - const request = new Request(`https://${context.req.headers.host}${context.req.url}`, { - method: context.req.method, - headers: context.req.headers, - body: JSON.stringify(context.req.body) - }) - - const response = await handler(request) - - context.res = { - status: response.status, - headers: Object.fromEntries(response.headers), - body: await response.text() - } + const { Brainy } = require('@soulcraft/brainy') + const { S3CompatibleStorage } = require('@soulcraft/brainy/storage') + + const storage = new S3CompatibleStorage({ + endpoint: `${process.env.AZURE_STORAGE_ACCOUNT}.blob.core.windows.net`, + bucket: 'brainy-data', + accessKeyId: process.env.AZURE_STORAGE_ACCOUNT, + secretAccessKey: process.env.AZURE_STORAGE_KEY, + prefix: 'entities/', + forcePathStyle: false + }) + + const brain = new Brainy({ storage }) + await brain.init() + + const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') + const handler = apiAugmentation.createUniversalHandler() + + const request = new Request(`https://${context.req.headers.host}${context.req.url}`, { + method: context.req.method, + headers: context.req.headers, + body: JSON.stringify(context.req.body) + }) + + const response = await handler(request) + + context.res = { + status: response.status, + headers: Object.fromEntries(response.headers), + body: await response.text() + } } ``` @@ -356,37 +356,37 @@ import { R2Storage } from '@soulcraft/brainy/storage' // Alias for S3CompatibleS let handler export default { - async fetch(request, env, ctx) { - if (!handler) { - const storage = new R2Storage({ - endpoint: `${env.ACCOUNT_ID}.r2.cloudflarestorage.com`, - bucket: 'brainy-data', - accessKeyId: env.R2_ACCESS_KEY_ID, - secretAccessKey: env.R2_SECRET_ACCESS_KEY, - region: 'auto', - forcePathStyle: true - }) - - const brain = new Brainy({ - storage, - augmentations: [{ - name: 'api-server', - config: { - enabled: true, - cors: { origin: '*' } - } - }] - }) - - await brain.init() - - const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') - handler = apiAugmentation.createUniversalHandler() - } - - // The handler works directly with Request/Response! - return handler(request) - } + async fetch(request, env, ctx) { + if (!handler) { + const storage = new R2Storage({ + endpoint: `${env.ACCOUNT_ID}.r2.cloudflarestorage.com`, + bucket: 'brainy-data', + accessKeyId: env.R2_ACCESS_KEY_ID, + secretAccessKey: env.R2_SECRET_ACCESS_KEY, + region: 'auto', + forcePathStyle: true + }) + + const brain = new Brainy({ + storage, + augmentations: [{ + name: 'api-server', + config: { + enabled: true, + cors: { origin: '*' } + } + }] + }) + + await brain.init() + + const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') + handler = apiAugmentation.createUniversalHandler() + } + + // The handler works directly with Request/Response! + return handler(request) + } } ``` @@ -418,52 +418,52 @@ import { S3CompatibleStorage } from '@soulcraft/brainy/storage' let handler export const config = { - runtime: 'edge', + runtime: 'edge', } export default async (request) => { - if (!handler) { - const storage = new S3CompatibleStorage({ - endpoint: 's3.amazonaws.com', - region: 'us-east-1', - bucket: process.env.S3_BUCKET, - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY - }) - - const brain = new Brainy({ - storage, - augmentations: [{ - name: 'api-server', - config: { enabled: true } - }] - }) - - await brain.init() - - const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') - handler = apiAugmentation.createUniversalHandler() - } - - return handler(request) + if (!handler) { + const storage = new S3CompatibleStorage({ + endpoint: 's3.amazonaws.com', + region: 'us-east-1', + bucket: process.env.S3_BUCKET, + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + }) + + const brain = new Brainy({ + storage, + augmentations: [{ + name: 'api-server', + config: { enabled: true } + }] + }) + + await brain.init() + + const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') + handler = apiAugmentation.createUniversalHandler() + } + + return handler(request) } ``` ```json // vercel.json { - "functions": { - "api/brainy.js": { - "maxDuration": 30, - "memory": 1024 - } - }, - "rewrites": [ - { - "source": "/api/:path*", - "destination": "/api/brainy" - } - ] + "functions": { + "api/brainy.js": { + "maxDuration": 30, + "memory": 1024 + } + }, + "rewrites": [ + { + "source": "/api/:path*", + "destination": "/api/brainy" + } + ] } ``` @@ -482,51 +482,51 @@ let brain let handler async function init() { - const storage = new S3CompatibleStorage({ - endpoint: process.env.S3_ENDPOINT || 's3.amazonaws.com', - region: process.env.S3_REGION || 'us-east-1', - bucket: process.env.S3_BUCKET, - accessKeyId: process.env.S3_ACCESS_KEY, - secretAccessKey: process.env.S3_SECRET_KEY, - prefix: 'brainy/' - }) - - brain = new Brainy({ - storage, - augmentations: [{ - name: 'api-server', - config: { - enabled: true, - port: PORT - } - }] - }) - - await brain.init() - - const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') - handler = apiAugmentation.createUniversalHandler() + const storage = new S3CompatibleStorage({ + endpoint: process.env.S3_ENDPOINT || 's3.amazonaws.com', + region: process.env.S3_REGION || 'us-east-1', + bucket: process.env.S3_BUCKET, + accessKeyId: process.env.S3_ACCESS_KEY, + secretAccessKey: process.env.S3_SECRET_KEY, + prefix: 'brainy/' + }) + + brain = new Brainy({ + storage, + augmentations: [{ + name: 'api-server', + config: { + enabled: true, + port: PORT + } + }] + }) + + await brain.init() + + const apiAugmentation = brain.augmentationRegistry.getAugmentation('api-server') + handler = apiAugmentation.createUniversalHandler() } await init() // Universal handler for all routes app.use('*', async (req, res) => { - const request = new Request(`http://localhost${req.originalUrl}`, { - method: req.method, - headers: req.headers, - body: JSON.stringify(req.body) - }) - - const response = await handler(request) - - res.status(response.status) - res.set(Object.fromEntries(response.headers)) - res.send(await response.text()) + const request = new Request(`http://localhost${req.originalUrl}`, { + method: req.method, + headers: req.headers, + body: JSON.stringify(req.body) + }) + + const response = await handler(request) + + res.status(response.status) + res.set(Object.fromEntries(response.headers)) + res.send(await response.text()) }) app.listen(PORT, () => { - console.log(`Brainy API running on port ${PORT}`) + console.log(`Brainy API running on port ${PORT}`) }) ``` @@ -552,26 +552,26 @@ restartPolicyMaxRetries = 3 ```yaml # render.yaml services: - - type: web - name: brainy-api - runtime: node - buildCommand: npm install - startCommand: node server.js - envVars: - - key: S3_BUCKET - value: brainy-data - - key: S3_ENDPOINT - value: s3.amazonaws.com - - key: S3_REGION - value: us-east-1 - - key: S3_ACCESS_KEY - sync: false - - key: S3_SECRET_KEY - sync: false - - key: API_KEY - generateValue: true - healthCheckPath: /health - autoDeploy: true + - type: web + name: brainy-api + runtime: node + buildCommand: npm install + startCommand: node server.js + envVars: + - key: S3_BUCKET + value: brainy-data + - key: S3_ENDPOINT + value: s3.amazonaws.com + - key: S3_REGION + value: us-east-1 + - key: S3_ACCESS_KEY + sync: false + - key: S3_SECRET_KEY + sync: false + - key: API_KEY + generateValue: true + healthCheckPath: /health + autoDeploy: true ``` ### Deno Deploy @@ -582,19 +582,19 @@ import { Brainy } from "npm:@soulcraft/brainy" import { S3CompatibleStorage } from "npm:@soulcraft/brainy/storage" const storage = new S3CompatibleStorage({ - endpoint: Deno.env.get("S3_ENDPOINT") || "s3.amazonaws.com", - bucket: Deno.env.get("S3_BUCKET")!, - accessKeyId: Deno.env.get("S3_ACCESS_KEY")!, - secretAccessKey: Deno.env.get("S3_SECRET_KEY")!, - region: "auto" + endpoint: Deno.env.get("S3_ENDPOINT") || "s3.amazonaws.com", + bucket: Deno.env.get("S3_BUCKET")!, + accessKeyId: Deno.env.get("S3_ACCESS_KEY")!, + secretAccessKey: Deno.env.get("S3_SECRET_KEY")!, + region: "auto" }) const brain = new Brainy({ - storage, - augmentations: [{ - name: 'api-server', - config: { enabled: true } - }] + storage, + augmentations: [{ + name: 'api-server', + config: { enabled: true } + }] }) await brain.init() @@ -653,29 +653,29 @@ RATE_LIMIT_MAX=100 ```javascript // REST API Client const response = await fetch('https://your-api.com/api/brainy/add', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer YOUR_API_KEY' - }, - body: JSON.stringify({ - data: 'Your content here', - metadata: { type: 'document' } - }) + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer YOUR_API_KEY' + }, + body: JSON.stringify({ + data: 'Your content here', + metadata: { type: 'document' } + }) }) const { id } = await response.json() // Search const searchResponse = await fetch('https://your-api.com/api/brainy/find', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer YOUR_API_KEY' - }, - body: JSON.stringify({ - query: 'neural networks' - }) + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer YOUR_API_KEY' + }, + body: JSON.stringify({ + query: 'neural networks' + }) }) const results = await searchResponse.json() @@ -683,14 +683,14 @@ const results = await searchResponse.json() // WebSocket Client const ws = new WebSocket('wss://your-api.com/ws') ws.onopen = () => { - ws.send(JSON.stringify({ - type: 'subscribe', - pattern: 'technology' - })) + ws.send(JSON.stringify({ + type: 'subscribe', + pattern: 'technology' + })) } ws.onmessage = (event) => { - const update = JSON.parse(event.data) - console.log('Real-time update:', update) + const update = JSON.parse(event.data) + console.log('Real-time update:', update) } ``` @@ -700,13 +700,13 @@ S3CompatibleStorage constructor parameters (verified from source): ```javascript { - endpoint: string, // Required (e.g., 's3.amazonaws.com') - bucket: string, // Required - accessKeyId: string, // Required - secretAccessKey: string, // Required - region?: string, // Optional (default: 'us-east-1') - prefix?: string, // Optional (e.g., 'brainy/') - forcePathStyle?: boolean, // Optional (needed for some S3-compatible services) + endpoint: string, // Required (e.g., 's3.amazonaws.com') + bucket: string, // Required + accessKeyId: string, // Required + secretAccessKey: string, // Required + region?: string, // Optional (default: 'us-east-1') + prefix?: string, // Optional (e.g., 'brainy/') + forcePathStyle?: boolean, // Optional (needed for some S3-compatible services) } ``` @@ -727,7 +727,7 @@ S3CompatibleStorage constructor parameters (verified from source): 4. **Implement rate limiting** to prevent abuse 5. **Configure CORS** appropriately for your use case -## Cost Optimization (v4.0.0) +## Cost Optimization ### Enable Lifecycle Policies @@ -738,16 +738,16 @@ const storage = brain.storage // Set lifecycle policy for automatic archival await storage.setLifecyclePolicy({ - rules: [{ - id: 'archive-old-data', - prefix: 'entities/', - status: 'Enabled', - transitions: [ - { days: 30, storageClass: 'STANDARD_IA' }, // $0.0125/GB after 30 days - { days: 90, storageClass: 'GLACIER' }, // $0.004/GB after 90 days - { days: 365, storageClass: 'DEEP_ARCHIVE' } // $0.00099/GB after 1 year - ] - }] + rules: [{ + id: 'archive-old-data', + prefix: 'entities/', + status: 'Enabled', + transitions: [ + { days: 30, storageClass: 'STANDARD_IA' }, // $0.0125/GB after 30 days + { days: 90, storageClass: 'GLACIER' }, // $0.004/GB after 90 days + { days: 365, storageClass: 'DEEP_ARCHIVE' } // $0.00099/GB after 1 year + ] + }] }) // Or enable Intelligent-Tiering for hands-off optimization @@ -765,14 +765,14 @@ await storage.enableIntelligentTiering('entities/', 'auto-optimize') **Efficient bulk deletions:** ```javascript -// v4.0.0: Batch delete (1000 objects per request) +// Batch delete (1000 objects per request) const idsToDelete = [/* array of entity IDs */] const paths = idsToDelete.flatMap(id => [ - `entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`, - `entities/nouns/metadata/${id.substring(0, 2)}/${id}.json` + `entities/nouns/vectors/${id.substring(0, 2)}/${id}.json`, + `entities/nouns/metadata/${id.substring(0, 2)}/${id}.json` ]) -await storage.batchDelete(paths) // Much faster than individual deletes +await storage.batchDelete(paths) // Much faster than individual deletes ``` ### Enable Compression (FileSystem) @@ -780,8 +780,8 @@ await storage.batchDelete(paths) // Much faster than individual deletes **For local/server deployments:** ```javascript const storage = new FileSystemStorage({ - path: './data', - compression: true // 60-80% space savings + path: './data', + compression: true // 60-80% space savings }) const brain = new Brainy({ storage }) @@ -793,8 +793,8 @@ const brain = new Brainy({ storage }) 2. **Use S3CompatibleStorage** for cloud deployments (better scalability) 3. **Enable the cache augmentation** for frequently accessed data 4. **Configure appropriate memory limits** for your runtime -5. **v4.0.0**: Enable lifecycle policies to reduce storage costs by 96% -6. **v4.0.0**: Use batch operations for cleanup tasks +5. Enable lifecycle policies to reduce storage costs by 96% +6. Use batch operations for cleanup tasks ## Troubleshooting @@ -810,15 +810,15 @@ const brain = new Brainy({ storage }) Enable debug logging by setting: ```javascript const brain = new Brainy({ - storage, - debug: true, - augmentations: [{ - name: 'api-server', - config: { - enabled: true, - verbose: true - } - }] + storage, + debug: true, + augmentations: [{ + name: 'api-server', + config: { + enabled: true, + verbose: true + } + }] }) ``` diff --git a/docs/deployment/aws-deployment.md b/docs/deployment/aws-deployment.md index b2afb4dc..e80cb8a5 100644 --- a/docs/deployment/aws-deployment.md +++ b/docs/deployment/aws-deployment.md @@ -18,24 +18,24 @@ const { Brainy } = require('@soulcraft/brainy') let brain exports.handler = async (event) => { - // Brainy auto-detects Lambda environment and configures accordingly - if (!brain) { - brain = new Brainy() // Zero config - auto-adapts to Lambda - await brain.init() - } - - const { method, ...params } = JSON.parse(event.body) - - switch(method) { - case 'add': - const id = await brain.add(params) - return { statusCode: 200, body: JSON.stringify({ id }) } - case 'find': - const results = await brain.find(params) - return { statusCode: 200, body: JSON.stringify({ results }) } - default: - return { statusCode: 400, body: 'Unknown method' } - } + // Brainy auto-detects Lambda environment and configures accordingly + if (!brain) { + brain = new Brainy() // Zero config - auto-adapts to Lambda + await brain.init() + } + + const { method, ...params } = JSON.parse(event.body) + + switch(method) { + case 'add': + const id = await brain.add(params) + return { statusCode: 200, body: JSON.stringify({ id }) } + case 'find': + const results = await brain.find(params) + return { statusCode: 200, body: JSON.stringify({ results }) } + default: + return { statusCode: 400, body: 'Unknown method' } + } } EOF @@ -56,26 +56,26 @@ docker push $ECR_URI/brainy:latest # Deploy with minimal ECS task definition cat > task-definition.json << 'EOF' { - "family": "brainy", - "networkMode": "awsvpc", - "requiresCompatibilities": ["FARGATE"], - "cpu": "256", - "memory": "512", - "containerDefinitions": [{ - "name": "brainy", - "image": "$ECR_URI/brainy:latest", - "environment": [ - {"name": "NODE_ENV", "value": "production"} - ], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/brainy", - "awslogs-region": "us-east-1", - "awslogs-stream-prefix": "ecs" - } - } - }] + "family": "brainy", + "networkMode": "awsvpc", + "requiresCompatibilities": ["FARGATE"], + "cpu": "256", + "memory": "512", + "containerDefinitions": [{ + "name": "brainy", + "image": "$ECR_URI/brainy:latest", + "environment": [ + {"name": "NODE_ENV", "value": "production"} + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/brainy", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "ecs" + } + } + }] } EOF @@ -138,14 +138,14 @@ const brain = new Brainy() ```javascript const brain = new Brainy({ - storage: { - type: 's3', - options: { - bucket: process.env.S3_BUCKET || 'auto', // 'auto' creates bucket - region: process.env.AWS_REGION || 'auto', // 'auto' detects region - // IAM role provides credentials automatically - } - } + storage: { + type: 's3', + options: { + bucket: process.env.S3_BUCKET || 'auto', // 'auto' creates bucket + region: process.env.AWS_REGION || 'auto', // 'auto' detects region + // IAM role provides credentials automatically + } + } }) ``` @@ -156,23 +156,23 @@ const brain = new Brainy({ ```yaml # Auto-scaling policy Resources: - AutoScalingTarget: - Type: AWS::ApplicationAutoScaling::ScalableTarget - Properties: - ServiceNamespace: ecs - ResourceId: service/default/brainy - ScalableDimension: ecs:service:DesiredCount - MinCapacity: 2 - MaxCapacity: 100 - - AutoScalingPolicy: - Type: AWS::ApplicationAutoScaling::ScalingPolicy - Properties: - PolicyType: TargetTrackingScaling - TargetTrackingScalingPolicyConfiguration: - TargetValue: 70.0 - PredefinedMetricSpecification: - PredefinedMetricType: ECSServiceAverageCPUUtilization + AutoScalingTarget: + Type: AWS::ApplicationAutoScaling::ScalableTarget + Properties: + ServiceNamespace: ecs + ResourceId: service/default/brainy + ScalableDimension: ecs:service:DesiredCount + MinCapacity: 2 + MaxCapacity: 100 + + AutoScalingPolicy: + Type: AWS::ApplicationAutoScaling::ScalingPolicy + Properties: + PolicyType: TargetTrackingScaling + TargetTrackingScalingPolicyConfiguration: + TargetValue: 70.0 + PredefinedMetricSpecification: + PredefinedMetricType: ECSServiceAverageCPUUtilization ``` ### 2. Vertical Scaling @@ -189,10 +189,10 @@ Brainy automatically adapts to available memory: ```javascript // Brainy automatically handles multi-AZ with S3 const brain = new Brainy({ - distributed: { - enabled: true, // Auto-enables with S3 storage - coordinationMethod: 'auto' // Uses S3 for coordination - } + distributed: { + enabled: true, // Auto-enables with S3 storage + coordinationMethod: 'auto' // Uses S3 for coordination + } }) ``` @@ -201,17 +201,17 @@ const brain = new Brainy({ ```bash # Application Load Balancer with health checks aws elbv2 create-load-balancer \ - --name brainy-alb \ - --subnets subnet-xxx subnet-yyy \ - --security-groups sg-xxx + --name brainy-alb \ + --subnets subnet-xxx subnet-yyy \ + --security-groups sg-xxx aws elbv2 create-target-group \ - --name brainy-targets \ - --protocol HTTP \ - --port 3000 \ - --vpc-id vpc-xxx \ - --health-check-path /health \ - --health-check-interval-seconds 30 + --name brainy-targets \ + --protocol HTTP \ + --port 3000 \ + --vpc-id vpc-xxx \ + --health-check-path /health \ + --health-check-interval-seconds 30 ``` ## Monitoring & Observability @@ -233,16 +233,16 @@ Brainy automatically sends metrics when running on AWS: ```javascript const brain = new Brainy({ - monitoring: { - enabled: true, - customMetrics: { - namespace: 'Brainy/Production', - dimensions: [ - { Name: 'Environment', Value: 'production' }, - { Name: 'Service', Value: 'api' } - ] - } - } + monitoring: { + enabled: true, + customMetrics: { + namespace: 'Brainy/Production', + dimensions: [ + { Name: 'Environment', Value: 'production' }, + { Name: 'Service', Value: 'api' } + ] + } + } }) ``` @@ -252,22 +252,22 @@ const brain = new Brainy({ ```json { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::brainy-*/*", - "arn:aws:s3:::brainy-*" - ] - } - ] + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::brainy-*/*", + "arn:aws:s3:::brainy-*" + ] + } + ] } ``` @@ -285,12 +285,12 @@ aws ec2 create-subnet --vpc-id vpc-xxx --cidr-block 10.0.2.0/24 --availability-z ```javascript // Automatic encryption with S3 const brain = new Brainy({ - storage: { - type: 's3', - options: { - encryption: 'auto' // Uses S3 SSE-S3 by default - } - } + storage: { + type: 's3', + options: { + encryption: 'auto' // Uses S3 SSE-S3 by default + } + } }) ``` @@ -300,14 +300,14 @@ const brain = new Brainy({ ```bash aws ec2 request-spot-fleet --spot-fleet-request-config '{ - "IamFleetRole": "arn:aws:iam::xxx:role/fleet-role", - "TargetCapacity": 2, - "SpotPrice": "0.05", - "LaunchSpecifications": [{ - "ImageId": "ami-xxx", - "InstanceType": "t3.medium", - "UserData": "BASE64_ENCODED_STARTUP_SCRIPT" - }] + "IamFleetRole": "arn:aws:iam::xxx:role/fleet-role", + "TargetCapacity": 2, + "SpotPrice": "0.05", + "LaunchSpecifications": [{ + "ImageId": "ami-xxx", + "InstanceType": "t3.medium", + "UserData": "BASE64_ENCODED_STARTUP_SCRIPT" + }] }' ``` @@ -316,12 +316,12 @@ aws ec2 request-spot-fleet --spot-fleet-request-config '{ ```javascript // Brainy automatically uses S3 Intelligent-Tiering const brain = new Brainy({ - storage: { - type: 's3', - options: { - storageClass: 'INTELLIGENT_TIERING' // Automatic cost optimization - } - } + storage: { + type: 's3', + options: { + storageClass: 'INTELLIGENT_TIERING' // Automatic cost optimization + } + } }) ``` @@ -329,8 +329,8 @@ const brain = new Brainy({ ```bash aws lambda put-function-concurrency \ - --function-name brainy-handler \ - --reserved-concurrent-executions 10 + --function-name brainy-handler \ + --reserved-concurrent-executions 10 ``` ## Deployment Automation @@ -340,28 +340,28 @@ aws lambda put-function-concurrency \ ```yaml name: Deploy to AWS on: - push: - branches: [main] + push: + branches: [main] jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Deploy to ECS - run: | - docker build -t brainy . - docker tag brainy:latest $ECR_URI/brainy:latest - docker push $ECR_URI/brainy:latest - aws ecs update-service --cluster default --service brainy --force-new-deployment + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Deploy to ECS + run: | + docker build -t brainy . + docker tag brainy:latest $ECR_URI/brainy:latest + docker push $ECR_URI/brainy:latest + aws ecs update-service --cluster default --service brainy --force-new-deployment ``` ## Troubleshooting @@ -369,121 +369,121 @@ jobs: ### Common Issues 1. **Storage Auto-Detection Fails** - ```javascript - // Explicitly specify storage - const brain = new Brainy({ - storage: { type: 's3', options: { bucket: 'my-bucket' } } - }) - ``` + ```javascript + // Explicitly specify storage + const brain = new Brainy({ + storage: { type: 's3', options: { bucket: 'my-bucket' } } + }) + ``` 2. **Memory Issues** - ```javascript - // Optimize for low memory - const brain = new Brainy({ - cache: { maxSize: 100 }, // Reduce cache size - index: { M: 8 } // Reduce HNSW connections - }) - ``` + ```javascript + // Optimize for low memory + const brain = new Brainy({ + cache: { maxSize: 100 }, // Reduce cache size + index: { M: 8 } // Reduce HNSW connections + }) + ``` 3. **Cold Starts (Lambda)** - **v7.3.0+ Progressive Initialization (Zero-Config)** + **Progressive Initialization (Zero-Config)** - Brainy automatically detects Lambda environments (AWS_LAMBDA_FUNCTION_NAME) - and uses progressive initialization for <200ms cold starts: + Brainy automatically detects Lambda environments (AWS_LAMBDA_FUNCTION_NAME) + and uses progressive initialization for <200ms cold starts: - ```javascript - // Zero-config - Brainy auto-detects Lambda - const brain = new Brainy({ - storage: { - type: 's3', - s3Storage: { - bucketName: 'my-bucket', - region: 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY - } - } - }) - await brain.init() // Returns in <200ms + ```javascript + // Zero-config - Brainy auto-detects Lambda + const brain = new Brainy({ + storage: { + type: 's3', + s3Storage: { + bucketName: 'my-bucket', + region: 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + } + } + }) + await brain.init() // Returns in <200ms - // First write validates bucket (lazy validation) - await brain.add('noun', { name: 'test' }) // Validates here - ``` + // First write validates bucket (lazy validation) + await brain.add('noun', { name: 'test' }) // Validates here + ``` - **Manual Override (if needed)** - ```javascript - const brain = new Brainy({ - storage: { - type: 's3', - s3Storage: { - bucketName: 'my-bucket', - // Force specific mode - initMode: 'progressive' // 'auto' | 'progressive' | 'strict' - } - } - }) - ``` + **Manual Override (if needed)** + ```javascript + const brain = new Brainy({ + storage: { + type: 's3', + s3Storage: { + bucketName: 'my-bucket', + // Force specific mode + initMode: 'progressive' // 'auto' | 'progressive' | 'strict' + } + } + }) + ``` - | Mode | Cold Start | Best For | - |------|------------|----------| - | `auto` (default) | <200ms in Lambda | Zero-config, auto-detects | - | `progressive` | <200ms always | Force fast init everywhere | - | `strict` | 100-500ms+ | Local dev, tests, debugging | + | Mode | Cold Start | Best For | + |------|------------|----------| + | `auto` (default) | <200ms in Lambda | Zero-config, auto-detects | + | `progressive` | <200ms always | Force fast init everywhere | + | `strict` | 100-500ms+ | Local dev, tests, debugging | - **Warm-up (Alternative)** - ```javascript - exports.warmup = async () => { - if (!brain) { - brain = new Brainy({ warmup: true }) - await brain.init() - } - } - ``` + **Warm-up (Alternative)** + ```javascript + exports.warmup = async () => { + if (!brain) { + brain = new Brainy({ warmup: true }) + await brain.init() + } + } + ``` - **Readiness Detection (v7.3.0+)** + **Readiness Detection** - Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests: + Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests: - ```javascript - let brain + ```javascript + let brain - exports.handler = async (event) => { - if (!brain) { - brain = new Brainy({ storage: { type: 's3', ... } }) - brain.init() // Fire and forget - } + exports.handler = async (event) => { + if (!brain) { + brain = new Brainy({ storage: { type: 's3', ... } }) + brain.init() // Fire and forget + } - // Wait for initialization to complete - await brain.ready + // Wait for initialization to complete + await brain.ready - // Now safe to use brain methods - const results = await brain.find({ query: event.queryStringParameters.q }) - return { statusCode: 200, body: JSON.stringify(results) } - } - ``` + // Now safe to use brain methods + const results = await brain.find({ query: event.queryStringParameters.q }) + return { statusCode: 200, body: JSON.stringify(results) } + } + ``` - For health checks, use `isFullyInitialized()` to verify all background tasks are complete: + For health checks, use `isFullyInitialized()` to verify all background tasks are complete: - ```javascript - exports.healthCheck = async () => { - try { - await brain.ready - return { - statusCode: 200, - body: JSON.stringify({ - status: 'ready', - fullyInitialized: brain.isFullyInitialized() - }) - } - } catch (error) { - return { - statusCode: 503, - body: JSON.stringify({ status: 'initializing' }) - } - } - } - ``` + ```javascript + exports.healthCheck = async () => { + try { + await brain.ready + return { + statusCode: 200, + body: JSON.stringify({ + status: 'ready', + fullyInitialized: brain.isFullyInitialized() + }) + } + } catch (error) { + return { + statusCode: 503, + body: JSON.stringify({ status: 'initializing' }) + } + } + } + ``` ## Production Checklist diff --git a/docs/deployment/gcp-deployment.md b/docs/deployment/gcp-deployment.md index 0e5d0f96..539966c9 100644 --- a/docs/deployment/gcp-deployment.md +++ b/docs/deployment/gcp-deployment.md @@ -10,10 +10,10 @@ Deploy Brainy on GCP with automatic scaling, global distribution, and zero-confi ```bash # Build and deploy with one command gcloud run deploy brainy \ - --source . \ - --platform managed \ - --region us-central1 \ - --allow-unauthenticated + --source . \ + --platform managed \ + --region us-central1 \ + --allow-unauthenticated # Brainy auto-detects Cloud Run and configures: # - Memory-optimized caching @@ -30,44 +30,44 @@ const { Brainy } = require('@soulcraft/brainy') let brain exports.brainyHandler = async (req, res) => { - // Zero-config - auto-adapts to Cloud Functions - if (!brain) { - brain = new Brainy() // Detects GCP environment automatically - await brain.init() - } - - const { method, ...params } = req.body - - try { - let result - switch(method) { - case 'add': - result = await brain.add(params) - break - case 'find': - result = await brain.find(params) - break - case 'relate': - result = await brain.relate(params) - break - default: - return res.status(400).json({ error: 'Unknown method' }) - } - res.json({ result }) - } catch (error) { - res.status(500).json({ error: error.message }) - } + // Zero-config - auto-adapts to Cloud Functions + if (!brain) { + brain = new Brainy() // Detects GCP environment automatically + await brain.init() + } + + const { method, ...params } = req.body + + try { + let result + switch(method) { + case 'add': + result = await brain.add(params) + break + case 'find': + result = await brain.find(params) + break + case 'relate': + result = await brain.relate(params) + break + default: + return res.status(400).json({ error: 'Unknown method' }) + } + res.json({ result }) + } catch (error) { + res.status(500).json({ error: error.message }) + } } ``` Deploy: ```bash gcloud functions deploy brainy \ - --runtime nodejs20 \ - --trigger-http \ - --entry-point brainyHandler \ - --memory 512MB \ - --timeout 60s + --runtime nodejs20 \ + --trigger-http \ + --entry-point brainyHandler \ + --memory 512MB \ + --timeout 60s ``` ### Option 3: Google Kubernetes Engine (GKE) @@ -75,7 +75,7 @@ gcloud functions deploy brainy \ ```bash # Create autopilot cluster (fully managed, zero-config) gcloud container clusters create-auto brainy-cluster \ - --region us-central1 + --region us-central1 # Deploy using Cloud Build gcloud builds submit --tag gcr.io/$PROJECT_ID/brainy @@ -85,39 +85,39 @@ kubectl apply -f - < { - // Send custom metrics to Cloud Monitoring - await monitoring.createTimeSeries({ - name: monitoring.projectPath(projectId), - timeSeries: [{ - metric: { - type: `custom.googleapis.com/brainy/${metric.name}`, - labels: metric.labels - }, - points: [{ - interval: { endTime: { seconds: Date.now() / 1000 } }, - value: { doubleValue: metric.value } - }] - }] - }) - } + onMetric: async (metric) => { + // Send custom metrics to Cloud Monitoring + await monitoring.createTimeSeries({ + name: monitoring.projectPath(projectId), + timeSeries: [{ + metric: { + type: `custom.googleapis.com/brainy/${metric.name}`, + labels: metric.labels + }, + points: [{ + interval: { endTime: { seconds: Date.now() / 1000 } }, + value: { doubleValue: metric.value } + }] + }] + }) + } }) ``` @@ -300,10 +300,10 @@ const brain = new Brainy({ ```javascript const brain = new Brainy({ - tracing: { - enabled: true, - sampleRate: 0.1 // Sample 10% of requests - } + tracing: { + enabled: true, + sampleRate: 0.1 // Sample 10% of requests + } }) ``` @@ -316,9 +316,9 @@ const brain = new Brainy({ apiVersion: v1 kind: ServiceAccount metadata: - name: brainy-sa - annotations: - iam.gke.io/gcp-service-account: brainy@PROJECT_ID.iam.gserviceaccount.com + name: brainy-sa + annotations: + iam.gke.io/gcp-service-account: brainy@PROJECT_ID.iam.gserviceaccount.com ``` ### 2. Binary Authorization @@ -328,11 +328,11 @@ metadata: apiVersion: binaryauthorization.grafeas.io/v1beta1 kind: Policy metadata: - name: brainy-policy + name: brainy-policy spec: - defaultAdmissionRule: - requireAttestationsBy: - - projects/PROJECT_ID/attestors/prod-attestor + defaultAdmissionRule: + requireAttestationsBy: + - projects/PROJECT_ID/attestors/prod-attestor ``` ### 3. VPC Service Controls @@ -340,9 +340,9 @@ spec: ```bash # Create VPC Service Perimeter gcloud access-context-manager perimeters create brainy_perimeter \ - --resources=projects/PROJECT_NUMBER \ - --restricted-services=storage.googleapis.com \ - --title="Brainy Security Perimeter" + --resources=projects/PROJECT_NUMBER \ + --restricted-services=storage.googleapis.com \ + --title="Brainy Security Perimeter" ``` ## Cost Optimization @@ -354,16 +354,16 @@ gcloud access-context-manager perimeters create brainy_perimeter \ apiVersion: container.cnrm.cloud.google.com/v1beta1 kind: ContainerNodePool metadata: - name: brainy-preemptible-pool + name: brainy-preemptible-pool spec: - clusterRef: - name: brainy-cluster - config: - preemptible: true - machineType: n2-standard-2 - autoscaling: - minNodeCount: 1 - maxNodeCount: 10 + clusterRef: + name: brainy-cluster + config: + preemptible: true + machineType: n2-standard-2 + autoscaling: + minNodeCount: 1 + maxNodeCount: 10 ``` ### 2. Cloud CDN for Static Assets @@ -371,11 +371,11 @@ spec: ```bash # Enable Cloud CDN for frequently accessed data gcloud compute backend-buckets create brainy-assets \ - --gcs-bucket-name=brainy-static + --gcs-bucket-name=brainy-static gcloud compute backend-buckets update brainy-assets \ - --enable-cdn \ - --cache-mode=CACHE_ALL_STATIC + --enable-cdn \ + --cache-mode=CACHE_ALL_STATIC ``` ### 3. Committed Use Discounts @@ -383,8 +383,8 @@ gcloud compute backend-buckets update brainy-assets \ ```bash # Purchase committed use for predictable workloads gcloud compute commitments create brainy-commitment \ - --plan=TWELVE_MONTH \ - --resources=vcpu=100,memory=400 + --plan=TWELVE_MONTH \ + --resources=vcpu=100,memory=400 ``` ## Deployment Automation @@ -394,28 +394,28 @@ gcloud compute commitments create brainy-commitment \ ```yaml # cloudbuild.yaml steps: - # Build container - - name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA', '.'] - - # Push to registry - - name: 'gcr.io/cloud-builders/docker' - args: ['push', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA'] - - # Deploy to Cloud Run - - name: 'gcr.io/cloud-builders/gcloud' - args: - - 'run' - - 'deploy' - - 'brainy' - - '--image=gcr.io/$PROJECT_ID/brainy:$SHORT_SHA' - - '--region=us-central1' - - '--platform=managed' + # Build container + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA', '.'] + + # Push to registry + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/$PROJECT_ID/brainy:$SHORT_SHA'] + + # Deploy to Cloud Run + - name: 'gcr.io/cloud-builders/gcloud' + args: + - 'run' + - 'deploy' + - 'brainy' + - '--image=gcr.io/$PROJECT_ID/brainy:$SHORT_SHA' + - '--region=us-central1' + - '--platform=managed' # Trigger on push to main trigger: - branch: - name: main + branch: + name: main ``` ### Terraform Infrastructure @@ -423,40 +423,40 @@ trigger: ```hcl # main.tf resource "google_cloud_run_service" "brainy" { - name = "brainy" - location = "us-central1" + name = "brainy" + location = "us-central1" - template { - spec { - containers { - image = "gcr.io/${var.project_id}/brainy" - - resources { - limits = { - cpu = "2" - memory = "2Gi" - } - } - - env { - name = "NODE_ENV" - value = "production" - } - } - } - } + template { + spec { + containers { + image = "gcr.io/${var.project_id}/brainy" - traffic { - percent = 100 - latest_revision = true - } + resources { + limits = { + cpu = "2" + memory = "2Gi" + } + } + + env { + name = "NODE_ENV" + value = "production" + } + } + } + } + + traffic { + percent = 100 + latest_revision = true + } } resource "google_cloud_run_service_iam_member" "public" { - service = google_cloud_run_service.brainy.name - location = google_cloud_run_service.brainy.location - role = "roles/run.invoker" - member = "allUsers" + service = google_cloud_run_service.brainy.name + location = google_cloud_run_service.brainy.location + role = "roles/run.invoker" + member = "allUsers" } ``` @@ -467,13 +467,13 @@ resource "google_cloud_run_service_iam_member" "public" { ```javascript // Brainy can use Memorystore for caching const brain = new Brainy({ - cache: { - type: 'redis', - options: { - host: process.env.REDIS_HOST || 'auto-detect', - port: 6379 - } - } + cache: { + type: 'redis', + options: { + host: process.env.REDIS_HOST || 'auto-detect', + port: 6379 + } + } }) ``` @@ -481,13 +481,13 @@ const brain = new Brainy({ ```javascript const brain = new Brainy({ - metadata: { - type: 'spanner', - options: { - instance: 'brainy-instance', - database: 'brainy-db' - } - } + metadata: { + type: 'spanner', + options: { + instance: 'brainy-instance', + database: 'brainy-db' + } + } }) ``` @@ -496,116 +496,116 @@ const brain = new Brainy({ ### Common Issues 1. **Quota Exceeded** - ```bash - # Check quotas - gcloud compute project-info describe --project=$PROJECT_ID - - # Request increase - gcloud compute project-info add-metadata \ - --metadata google-compute-default-region=us-central1 - ``` + ```bash + # Check quotas + gcloud compute project-info describe --project=$PROJECT_ID + + # Request increase + gcloud compute project-info add-metadata \ + --metadata google-compute-default-region=us-central1 + ``` 2. **Cold Starts** - **v7.3.0+ Progressive Initialization (Zero-Config)** + **Progressive Initialization (Zero-Config)** - Brainy automatically detects Cloud Run and Cloud Functions environments - and uses progressive initialization for <200ms cold starts: + Brainy automatically detects Cloud Run and Cloud Functions environments + and uses progressive initialization for <200ms cold starts: - ```javascript - // Zero-config - Brainy auto-detects Cloud Run (K_SERVICE env var) - const brain = new Brainy({ - storage: { - type: 'gcs', - gcsNativeStorage: { bucketName: 'my-bucket' } - } - }) - await brain.init() // Returns in <200ms + ```javascript + // Zero-config - Brainy auto-detects Cloud Run (K_SERVICE env var) + const brain = new Brainy({ + storage: { + type: 'gcs', + gcsNativeStorage: { bucketName: 'my-bucket' } + } + }) + await brain.init() // Returns in <200ms - // First write validates bucket (lazy validation) - await brain.add('noun', { name: 'test' }) // Validates here - ``` + // First write validates bucket (lazy validation) + await brain.add('noun', { name: 'test' }) // Validates here + ``` - **Manual Override (if needed)** - ```javascript - const brain = new Brainy({ - storage: { - type: 'gcs', - gcsNativeStorage: { - bucketName: 'my-bucket', - // Force specific mode - initMode: 'progressive' // 'auto' | 'progressive' | 'strict' - } - } - }) - ``` + **Manual Override (if needed)** + ```javascript + const brain = new Brainy({ + storage: { + type: 'gcs', + gcsNativeStorage: { + bucketName: 'my-bucket', + // Force specific mode + initMode: 'progressive' // 'auto' | 'progressive' | 'strict' + } + } + }) + ``` - | Mode | Cold Start | Best For | - |------|------------|----------| - | `auto` (default) | <200ms in cloud | Zero-config, auto-detects | - | `progressive` | <200ms always | Force fast init everywhere | - | `strict` | 100-500ms+ | Local dev, tests, debugging | + | Mode | Cold Start | Best For | + |------|------------|----------| + | `auto` (default) | <200ms in cloud | Zero-config, auto-detects | + | `progressive` | <200ms always | Force fast init everywhere | + | `strict` | 100-500ms+ | Local dev, tests, debugging | - **Keep Warm (Alternative)** - ```javascript - // Keep minimum instances warm - const brain = new Brainy({ - warmup: { - enabled: true, - interval: 60000 // Ping every minute - } - }) - ``` + **Keep Warm (Alternative)** + ```javascript + // Keep minimum instances warm + const brain = new Brainy({ + warmup: { + enabled: true, + interval: 60000 // Ping every minute + } + }) + ``` - **Readiness Detection (v7.3.0+)** + **Readiness Detection** - Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests: + Use the `brain.ready` Promise to ensure Brainy is initialized before handling requests: - ```javascript - let brain + ```javascript + let brain - async function handleRequest(req, res) { - if (!brain) { - brain = new Brainy({ storage: { type: 'gcs', ... } }) - brain.init() // Fire and forget - } + async function handleRequest(req, res) { + if (!brain) { + brain = new Brainy({ storage: { type: 'gcs', ... } }) + brain.init() // Fire and forget + } - // Wait for initialization to complete - await brain.ready + // Wait for initialization to complete + await brain.ready - // Now safe to use brain methods - const results = await brain.find({ query: req.query.q }) - res.json(results) - } - ``` + // Now safe to use brain methods + const results = await brain.find({ query: req.query.q }) + res.json(results) + } + ``` - For Cloud Run health checks, use `isFullyInitialized()` to verify all background tasks are complete: + For Cloud Run health checks, use `isFullyInitialized()` to verify all background tasks are complete: - ```javascript - // Health check endpoint for Cloud Run - app.get('/health', async (req, res) => { - try { - await brain.ready - res.json({ - status: 'ready', - fullyInitialized: brain.isFullyInitialized() - }) - } catch (error) { - res.status(503).json({ status: 'initializing' }) - } - }) - ``` + ```javascript + // Health check endpoint for Cloud Run + app.get('/health', async (req, res) => { + try { + await brain.ready + res.json({ + status: 'ready', + fullyInitialized: brain.isFullyInitialized() + }) + } catch (error) { + res.status(503).json({ status: 'initializing' }) + } + }) + ``` 3. **Memory Pressure** - ```javascript - // Optimize for GCP memory constraints - const brain = new Brainy({ - memory: { - mode: 'aggressive', // Aggressive garbage collection - maxHeap: 0.8 // Use 80% of available memory - } - }) - ``` + ```javascript + // Optimize for GCP memory constraints + const brain = new Brainy({ + memory: { + mode: 'aggressive', // Aggressive garbage collection + maxHeap: 0.8 // Use 80% of available memory + } + }) + ``` ## Production Checklist diff --git a/docs/features/instant-fork.md b/docs/features/instant-fork.md index 9bbbd740..a3b2754d 100644 --- a/docs/features/instant-fork.md +++ b/docs/features/instant-fork.md @@ -42,10 +42,10 @@ await experiment.updateAll(riskyTransformation) // Works? Great! Use the experimental branch. // Failed? Just discard. if (success) { - // Make experiment the new main branch - await brain.checkout('test-migration') + // Make experiment the new main branch + await brain.checkout('test-migration') } else { - await experiment.destroy() // No harm done + await experiment.destroy() // No harm done } ``` @@ -61,25 +61,25 @@ if (success) { Brainy uses **Snowflake-style Copy-on-Write (COW)** for instant forking: -### Architecture (v5.0.0) +### Architecture 1. **HNSW Index COW** (The Performance Bottleneck): - - **Shallow Copy**: O(1) Map reference copying (~10ms for 1M+ nodes) - - **Lazy Deep Copy**: Nodes copied only when modified (`ensureCOW()`) - - **Write Isolation**: Fork modifications don't affect parent - - **Implementation**: `src/hnsw/hnswIndex.ts` lines 2088-2150 + - **Shallow Copy**: O(1) Map reference copying (~10ms for 1M+ nodes) + - **Lazy Deep Copy**: Nodes copied only when modified (`ensureCOW()`) + - **Write Isolation**: Fork modifications don't affect parent + - **Implementation**: `src/hnsw/hnswIndex.ts` lines 2088-2150 2. **Metadata & Graph Indexes** (Fast Rebuild): - - **Rebuild from Storage**: < 500ms total for both indexes - - **Shared Storage**: Both indexes read from COW-enabled storage layer - - **Acceptable Overhead**: Fast enough not to need in-memory COW + - **Rebuild from Storage**: < 500ms total for both indexes + - **Shared Storage**: Both indexes read from COW-enabled storage layer + - **Acceptable Overhead**: Fast enough not to need in-memory COW 3. **Storage Layer** (Shared): - - **RefManager**: Manages branch references - - **BlobStorage**: Content-addressable with deduplication - - **All Adapters**: Memory, FileSystem, S3, R2, GCS, Azure, OPFS + - **RefManager**: Manages branch references + - **BlobStorage**: Content-addressable with deduplication + - **All Adapters**: Memory, FileSystem, S3, R2, GCS, Azure, OPFS -**Performance (v5.0.0)**: +**Performance**: - **Fork time**: < 100ms @ 10K entities (MEASURED in tests) - **Storage overhead**: 10-20% (shared blobs, only changed data duplicated) - **Memory overhead**: 10-20% (shared HNSW nodes, only modified nodes duplicated) @@ -87,11 +87,11 @@ Brainy uses **Snowflake-style Copy-on-Write (COW)** for instant forking: **Technical Details**: ```typescript // Shallow copy HNSW (instant) -clone.index.enableCOW(this.index) // O(1) Map reference copy +clone.index.enableCOW(this.index) // O(1) Map reference copy // Fast rebuild small indexes from shared storage -clone.metadataIndex = new MetadataIndexManager(clone.storage) // <100ms -clone.graphIndex = new GraphAdjacencyIndex(clone.storage) // <500ms +clone.metadataIndex = new MetadataIndexManager(clone.storage) // <100ms +clone.graphIndex = new GraphAdjacencyIndex(clone.storage) // <500ms ``` --- @@ -128,11 +128,11 @@ await fork.add({ noun: 'user', data: { name: 'Charlie' } }) // All APIs work const users = await fork.find({ noun: 'user' }) -console.log(users.length) // 3 (Alice, Bob, Charlie) +console.log(users.length) // 3 (Alice, Bob, Charlie) // Original brain unchanged const originalUsers = await brain.find({ noun: 'user' }) -console.log(originalUsers.length) // 2 (Alice, Bob) +console.log(originalUsers.length) // 2 (Alice, Bob) ``` ### 3. Discard When Done @@ -165,30 +165,30 @@ const migration = await brain.fork('migration-test') // Run migration on fork const users = await migration.find({ noun: 'user' }) for (const user of users) { - await migration.update(user.id, { - email: user.data.email.toLowerCase(), // Normalize emails - verified: user.data.verified ?? false // Add missing field - }) + await migration.update(user.id, { + email: user.data.email.toLowerCase(), // Normalize emails + verified: user.data.verified ?? false // Add missing field + }) } // Validate migration const allValid = (await migration.find({ noun: 'user' })) - .every(u => u.data.email === u.data.email.toLowerCase()) + .every(u => u.data.email === u.data.email.toLowerCase()) if (allValid) { - console.log('✅ Migration safe! Apply changes to production brain') - // Apply validated migration to main brain - const users = await brain.find({ noun: 'user' }) - for (const user of users) { - await brain.update(user.id, { - email: user.data.email.toLowerCase(), - verified: user.data.verified ?? false - }) - } - await migration.destroy() + console.log('✅ Migration safe! Apply changes to production brain') + // Apply validated migration to main brain + const users = await brain.find({ noun: 'user' }) + for (const user of users) { + await brain.update(user.id, { + email: user.data.email.toLowerCase(), + verified: user.data.verified ?? false + }) + } + await migration.destroy() } else { - console.log('❌ Migration failed! Discarding fork.') - await migration.destroy() + console.log('❌ Migration failed! Discarding fork.') + await migration.destroy() } ``` @@ -203,8 +203,8 @@ const brain = new Brainy({ storage: { adapter: 's3', bucket: 'production' } }) await brain.init() // Create two variants -const variantA = await brain.fork('variant-a') // Control -const variantB = await brain.fork('variant-b') // Test +const variantA = await brain.fork('variant-a') // Control +const variantB = await brain.fork('variant-b') // Test // Run different algorithms await variantA.processWithAlgorithm('current') @@ -219,14 +219,14 @@ console.log('Variant B accuracy:', metricsB.accuracy) // Choose winner and update main brain if (metricsB.accuracy > metricsA.accuracy) { - console.log('B wins! Apply algorithm B to production') - await brain.processWithAlgorithm('improved') - await variantA.destroy() - await variantB.destroy() + console.log('B wins! Apply algorithm B to production') + await brain.processWithAlgorithm('improved') + await variantA.destroy() + await variantB.destroy() } else { - console.log('A wins! Keeping current algorithm.') - await variantA.destroy() - await variantB.destroy() + console.log('A wins! Keeping current algorithm.') + await variantA.destroy() + await variantB.destroy() } ``` @@ -311,12 +311,12 @@ await brain.update(docs[0].id, { version: 2 }) // Test against original state const originalData = await snapshot.find({ noun: 'doc' }) -console.log(originalData[0].data.version) // 1 (original state!) +console.log(originalData[0].data.version) // 1 (original state!) // Clean up await snapshot.destroy() -// Note: Time-travel queries (asOf) planned for v5.1.0 +// Note: Time-travel queries (asOf) Planned ``` --- @@ -330,27 +330,27 @@ await snapshot.destroy() const fork1 = await brain.fork('my-experiment') // Auto-generated name (uses timestamp) -const fork2 = await brain.fork() // 'fork-1635789012345' +const fork2 = await brain.fork() // 'fork-1635789012345' // Fork with metadata (for tracking) const fork3 = await brain.fork('test', { - author: 'Alice', - message: 'Testing new feature' + author: 'Alice', + message: 'Testing new feature' }) ``` -### Branch Management (v5.0.0) +### Branch Management -**NEW in v5.0.0:** Full branch management now available! + Full branch management now available! ```javascript // List all branches const branches = await brain.listBranches() -console.log(branches) // ['main', 'experiment', 'test'] +console.log(branches) // ['main', 'experiment', 'test'] // Get current branch const current = await brain.getCurrentBranch() -console.log(current) // 'main' +console.log(current) // 'main' // Switch between branches await brain.checkout('experiment') @@ -359,33 +359,33 @@ await brain.checkout('experiment') await brain.deleteBranch('old-experiment') ``` -### Commit Tracking (v5.0.0) +### Commit Tracking -**NEW in v5.0.0:** Git-style commit tracking! + Git-style commit tracking! ```javascript // Create a commit (snapshot of current state) await brain.add({ type: 'user', data: { name: 'Alice' } }) const commitHash = await brain.commit({ - message: 'Add Alice user', - author: 'dev@example.com' + message: 'Add Alice user', + author: 'dev@example.com' }) -console.log(commitHash) // 'a3f2c1b9...' +console.log(commitHash) // 'a3f2c1b9...' ``` -### Commit History (v5.0.0) +### Commit History -**NEW in v5.0.0:** View commit history! + View commit history! ```javascript // Get commit history for current branch const history = await brain.getHistory({ limit: 10 }) history.forEach(commit => { - console.log(`${commit.hash}: ${commit.message}`) - console.log(` By: ${commit.author} at ${new Date(commit.timestamp)}`) + console.log(`${commit.hash}: ${commit.message}`) + console.log(` By: ${commit.author} at ${new Date(commit.timestamp)}`) }) ## Performance Characteristics @@ -425,7 +425,7 @@ Savings: 40% less memory ## Zero Configuration -**Fork is enabled by default in v5.0.0+. No setup required.** +**Fork is enabled by default. No setup required.** ```javascript // This is all you need: @@ -467,9 +467,9 @@ await new Brainy({ storage: { adapter: 's3', bucket: 'my-data' } }).fork() ```javascript const brain = new Brainy({ - storage: { adapter: 'memory' }, - vfs: { enabled: true }, - intelligence: { enabled: true } + storage: { adapter: 'memory' }, + vfs: { enabled: true }, + intelligence: { enabled: true } }) await brain.init() @@ -484,9 +484,9 @@ await brain.add({ noun: 'user', data: { name: 'Alice' } }) const fork = await brain.fork('test') // All features work on fork: -await fork.vfs.readFile('/project/README.md') // ✅ VFS -await fork.find({ noun: 'user' }) // ✅ find() -await fork.query('users named Alice') // ✅ Triple Intelligence +await fork.vfs.readFile('/project/README.md') // ✅ VFS +await fork.find({ noun: 'user' }) // ✅ find() +await fork.query('users named Alice') // ✅ Triple Intelligence ``` ### Works at Billion Scale @@ -494,8 +494,8 @@ await fork.query('users named Alice') // ✅ Triple Intelligence ```javascript // Tested at 1M entities, extrapolates to 1B const brain = new Brainy({ - storage: { adapter: 'gcs', bucket: 'billion-scale' }, - hnsw: { typeAware: true } // 87% memory reduction + storage: { adapter: 'gcs', bucket: 'billion-scale' }, + hnsw: { typeAware: true } // 87% memory reduction }) await brain.init() @@ -563,7 +563,7 @@ ALTER TABLE users_backup RENAME TO users; ```javascript const fork = await brain.fork('test') await fork.updateAll({ email: (u) => u.email.toLowerCase() }) -if (success) await brain.checkout('test') // Make test branch active +if (success) await brain.checkout('test') // Make test branch active else await fork.destroy() ``` @@ -585,7 +585,7 @@ mongoimport --db mydb --collection users_backup --file users.json **Brainy:** ```javascript -const fork = await brain.fork() // Done! +const fork = await brain.fork() // Done! ``` **Winner: Brainy** (100x faster, built-in) @@ -612,8 +612,7 @@ const fork = await brain.fork() // Done! ## What's Implemented vs. What's Next -### ✅ Available in v5.0.0: -- ✅ `fork()` - Instant clone in <100ms +### ✅ Available in - ✅ `fork()` - Instant clone in <100ms - ✅ `listBranches()` - List all forks - ✅ `getCurrentBranch()` - Get active branch - ✅ `checkout()` - Switch between branches @@ -621,14 +620,14 @@ const fork = await brain.fork() // Done! - ✅ `commit()` - Create state snapshots - ✅ `getHistory()` - View commit history -### 🔮 Planned for v5.1.0+: +### 🔮 Planned for: **Temporal Features:** -- `asOf(timestamp)` - Query data at specific time (✅ v5.0.0+) +- `asOf(timestamp)` - Query data at specific time (✅ available) - `rollback(commitHash)` - Restore to previous state - Full audit trail for all changes -These features require additional temporal infrastructure and are being carefully designed for v5.1.0+ +These features require additional temporal infrastructure and are being carefully designed --- @@ -691,4 +690,4 @@ console.log('Fork created in < 2 seconds! 🚀') --- -**Brainy v5.0.0+** | [GitHub](https://github.com/soulcraftlabs/brainy) | [npm](https://npmjs.com/package/@soulcraft/brainy) +**Brainy** | [GitHub](https://github.com/soulcraftlabs/brainy) | [npm](https://npmjs.com/package/@soulcraft/brainy) diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index be786beb..ca13cf82 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -24,8 +24,8 @@ await brain.import(anything) ```javascript // Array of objects? No problem. const people = [ - { name: 'Alice', role: 'Engineer', company: 'TechCorp' }, - { name: 'Bob', role: 'Designer', company: 'TechCorp' } + { name: 'Alice', role: 'Engineer', company: 'TechCorp' }, + { name: 'Bob', role: 'Designer', company: 'TechCorp' } ] await brain.import(people) @@ -55,7 +55,7 @@ await brain.import('sales-report.xlsx') // Or specific sheets only await brain.import('data.xlsx', { - excelSheets: ['Customers', 'Orders'] + excelSheets: ['Customers', 'Orders'] }) // ✨ Multi-sheet data becomes interconnected entities! ``` @@ -68,7 +68,7 @@ await brain.import('research-paper.pdf') // With table extraction await brain.import('report.pdf', { - pdfExtractTables: true + pdfExtractTables: true }) // ✨ Converts PDF tables to structured data automatically! ``` @@ -83,16 +83,16 @@ await brain.import('config.yaml') const yaml = ` project: AI Assistant team: - - name: Alice - role: Lead - - name: Bob - role: Dev + - name: Alice + role: Lead + - name: Bob + role: Dev ` await brain.import(yaml, { format: 'yaml' }) // ✨ Hierarchical data becomes a connected graph! ``` -### 📄 Import Word Documents (DOCX) - v4.2.0 +### 📄 Import Word Documents (DOCX) - ```javascript // From file path await brain.import('research-paper.docx') @@ -105,8 +105,8 @@ await brain.import(buffer, { format: 'docx' }) // With neural extraction await brain.import('report.docx', { - enableNeuralExtraction: true, - enableHierarchicalRelationships: true + enableNeuralExtraction: true, + enableHierarchicalRelationships: true }) // ✨ Extracts entities from paragraphs and creates relationships within sections! ``` @@ -121,25 +121,25 @@ await brain.import('https://api.example.com/data.json') await brain.import('https://data.gov/census.csv') // ✨ Fetches CSV from web, parses, imports! -// With authentication (v4.2.0) +// With authentication await brain.import({ - type: 'url', - data: 'https://api.example.com/private/data.xlsx', - auth: { - username: 'user', - password: 'pass' - } + type: 'url', + data: 'https://api.example.com/private/data.xlsx', + auth: { + username: 'user', + password: 'pass' + } }) // ✨ Supports basic authentication for protected resources! -// With custom headers (v4.2.0) +// With custom headers await brain.import({ - type: 'url', - data: 'https://api.example.com/data.json', - headers: { - 'Authorization': 'Bearer TOKEN', - 'X-API-Key': 'your-key' - } + type: 'url', + data: 'https://api.example.com/data.json', + headers: { + 'Authorization': 'Bearer TOKEN', + 'X-API-Key': 'your-key' + } }) // ✨ Full HTTP header customization support! ``` @@ -163,9 +163,9 @@ When you import data, Brainy: 2. **Intelligent parsing** - CSV (encoding/delimiter), Excel (multi-sheet), PDF (text/tables), DOCX (headings/paragraphs) 3. **Identifies entity types** - Uses AI to classify as Person, Document, Product, etc. (31 types!) 4. **Finds relationships** - Detects connections like "belongsTo", "createdBy", "references" (40 types!) -5. **Scores confidence & weight** - Every entity and relationship gets quality metrics (v4.2.0) +5. **Scores confidence & weight** - Every entity and relationship gets quality metrics 6. **Creates embeddings** - Makes everything semantically searchable -7. **Indexes metadata** - Enables lightning-fast filtering with range queries (v4.2.0) +7. **Indexes metadata** - Enables lightning-fast filtering with range queries ## Intelligent Type Detection @@ -176,7 +176,7 @@ Brainy automatically detects what TYPE of data you're importing: { name: 'John', email: 'john@example.com' } // This becomes an Organization -{ companyName: 'Acme', employees: 500 } +{ companyName: 'Acme', employees: 500 } // This becomes a Document { title: 'Report', content: '...', author: 'Jane' } @@ -193,9 +193,9 @@ Brainy finds connections in your data: ```javascript const data = [ - { id: 'u1', name: 'Alice', managerId: 'u2' }, - { id: 'u2', name: 'Bob', departmentId: 'd1' }, - { id: 'd1', name: 'Engineering' } + { id: 'u1', name: 'Alice', managerId: 'u2' }, + { id: 'u2', name: 'Bob', departmentId: 'd1' }, + { id: 'd1', name: 'Engineering' } ] await brain.import(data) @@ -204,28 +204,27 @@ await brain.import(data) // - Bob "memberOf" Engineering ``` -## Confidence & Weight Scoring - v4.2.0 - +## Confidence & Weight Scoring - Every entity and relationship gets confidence and weight scores: ```javascript // Import with confidence threshold await brain.import(data, { - confidenceThreshold: 0.8 // Only extract entities with >80% confidence + confidenceThreshold: 0.8 // Only extract entities with >80% confidence }) // Query high-confidence entities using range queries const highConfidence = await brain.find({ - where: { - confidence: { gte: 0.8 } // Get entities with confidence >= 0.8 - } + where: { + confidence: { gte: 0.8 } // Get entities with confidence >= 0.8 + } }) // Range query operators: gt, gte, lt, lte, between const mediumConfidence = await brain.find({ - where: { - confidence: { between: [0.6, 0.8] } - } + where: { + confidence: { between: [0.6, 0.8] } + } }) ``` @@ -236,24 +235,23 @@ const mediumConfidence = await brain.find({ **Weights** indicate importance/relevance within the document context. -## Per-Sheet Excel Extraction - v4.2.0 - +## Per-Sheet Excel Extraction - Excel files with multiple sheets can be organized by sheet: ```javascript // Group entities by sheet in VFS await brain.import('multi-sheet-data.xlsx', { - groupBy: 'sheet' // Creates separate directories for each sheet + groupBy: 'sheet' // Creates separate directories for each sheet }) // Result VFS structure: // /imports/data/ -// ├── Sheet1/ -// │ ├── entity1.json -// │ └── entity2.json -// └── Sheet2/ -// ├── entity3.json -// └── entity4.json +// ├── Sheet1/ +// │ ├── entity1.json +// │ └── entity2.json +// └── Sheet2/ +// ├── entity3.json +// └── entity4.json // Other groupBy options: // - 'type': Group by entity type (Person, Place, etc.) @@ -274,9 +272,9 @@ const results = await brain.find('people in engineering who joined this year') // Graph traversal + filters const connected = await brain.find({ - like: 'Alice', - connected: { depth: 2 }, - where: { department: 'Engineering' } + like: 'Alice', + connected: { depth: 2 }, + where: { department: 'Engineering' } }) ``` @@ -286,37 +284,37 @@ Everything works with zero config, but you can customize: ```javascript await brain.import(data, { - // Format detection - format: 'excel', // Force specific format (auto-detected if not specified) + // Format detection + format: 'excel', // Force specific format (auto-detected if not specified) - // VFS & Organization - vfsPath: '/imports/my-data', // Where to store in VFS (auto-generated if not specified) - groupBy: 'type', // Group entities by: 'type' | 'sheet' | 'flat' | 'custom' - preserveSource: true, // Keep original source file in VFS (default: true) + // VFS & Organization + vfsPath: '/imports/my-data', // Where to store in VFS (auto-generated if not specified) + groupBy: 'type', // Group entities by: 'type' | 'sheet' | 'flat' | 'custom' + preserveSource: true, // Keep original source file in VFS (default: true) - // Entity & Relationship Creation - createEntities: true, // Create entities in knowledge graph (default: true) - createRelationships: true, // Create relationships in knowledge graph (default: true) + // Entity & Relationship Creation + createEntities: true, // Create entities in knowledge graph (default: true) + createRelationships: true, // Create relationships in knowledge graph (default: true) - // Neural Intelligence - enableNeuralExtraction: true, // Use AI to extract entities (default: true) - enableRelationshipInference: true, // Use AI to infer relationships (default: true) - enableConceptExtraction: true, // Extract concepts from text (default: true) - confidenceThreshold: 0.6, // Minimum confidence for entities (0-1, default: 0.6) + // Neural Intelligence + enableNeuralExtraction: true, // Use AI to extract entities (default: true) + enableRelationshipInference: true, // Use AI to infer relationships (default: true) + enableConceptExtraction: true, // Extract concepts from text (default: true) + confidenceThreshold: 0.6, // Minimum confidence for entities (0-1, default: 0.6) - // Deduplication - enableDeduplication: true, // Check for duplicate entities (default: true) - deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85) - // Note: Auto-disabled for imports >100 entities + // Deduplication + enableDeduplication: true, // Check for duplicate entities (default: true) + deduplicationThreshold: 0.85, // Similarity threshold for duplicates (0-1, default: 0.85) + // Note: Auto-disabled for imports >100 entities - // Performance - chunkSize: 100, // Batch size for processing (default: varies by operation) + // Performance + chunkSize: 100, // Batch size for processing (default: varies by operation) - // History & Progress - enableHistory: true, // Track import history (default: true) - onProgress: (progress) => { // Progress callback - console.log(progress.stage, progress.message) - } + // History & Progress + enableHistory: true, // Track import history (default: true) + onProgress: (progress) => { // Progress callback + console.log(progress.stage, progress.message) + } }) ``` @@ -343,9 +341,9 @@ const results = await brain.import(problematicData) ### 🏢 Business Data ```javascript // Import ANY source - ONE method! -await brain.import('customers.csv') // File -await brain.import('https://api.co/orders') // URL -await brain.import(productsArray) // Data +await brain.import('customers.csv') // File +await brain.import('https://api.co/orders') // URL +await brain.import(productsArray) // Data // Now query across all of it! await brain.find('customers who bought products in Q4') @@ -389,8 +387,8 @@ await brain.find('posts by users following Alice with >10 comments') ```javascript // ONE method that understands EVERYTHING: -await brain.import(data) // Objects, arrays, strings -await brain.import('file.csv') // Files (auto-detected) +await brain.import(data) // Objects, arrays, strings +await brain.import('file.csv') // Files (auto-detected) await brain.import('http://..') // URLs (auto-fetched) // It ALWAYS knows what to do! ✨ diff --git a/docs/guides/import-flow.md b/docs/guides/import-flow.md index de6c3e30..6a56f3d8 100644 --- a/docs/guides/import-flow.md +++ b/docs/guides/import-flow.md @@ -39,7 +39,7 @@ Each phase adds intelligence and structure to your raw data, transforming it int --- -## 🌊 Always-On Streaming Architecture (v4.2.0+) +## 🌊 Always-On Streaming Architecture All imports use streaming with **progressive flush intervals**: @@ -1062,7 +1062,7 @@ if (!fromEntity || !toEntity) { } ``` -#### 5.3b: Check for Duplicates (v3.43.2 Critical Fix) +#### 5.3b: Check for Duplicates (Critical Fix) **The Bug**: Without duplicate checking, re-importing would create: ``` diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md index 975b9570..66f50713 100644 --- a/docs/guides/import-progress-examples.md +++ b/docs/guides/import-progress-examples.md @@ -2,7 +2,7 @@ **How to Use Progress Tracking in Your Applications** -Brainy v4.5.0+ provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX). +Brainy provides real-time progress tracking for **all 7 supported file formats** (CSV, PDF, Excel, JSON, Markdown, YAML, DOCX). > **⚠️ KEY FEATURE:** The progress API is **100% standardized**. Write your progress handler ONCE and it works for ALL formats with zero format-specific code! See [Standard Import Progress API](./standard-import-progress.md) for the complete interface documentation. @@ -20,12 +20,12 @@ const brain = await Brainy.create() // Import with progress tracking const result = await brain.import(fs.readFileSync('large-file.xlsx'), { - onProgress: (progress) => { - console.log(`Progress: ${progress.stage}`) - console.log(` Message: ${progress.message}`) - console.log(` Entities: ${progress.entities || 0}`) - console.log(` Relationships: ${progress.relationships || 0}`) - } + onProgress: (progress) => { + console.log(`Progress: ${progress.stage}`) + console.log(` Message: ${progress.message}`) + console.log(` Entities: ${progress.entities || 0}`) + console.log(` Relationships: ${progress.relationships || 0}`) + } }) console.log(`Import complete: ${result.entities.length} entities created`) @@ -34,30 +34,30 @@ console.log(`Import complete: ${result.entities.length} entities created`) **Expected Output:** ``` Progress: detecting - Message: Detecting format... - Entities: 0 - Relationships: 0 + Message: Detecting format... + Entities: 0 + Relationships: 0 Progress: extracting - Message: Loading Excel workbook... - Entities: 0 - Relationships: 0 + Message: Loading Excel workbook... + Entities: 0 + Relationships: 0 Progress: extracting - Message: Reading sheet: Sales (1/3) - Entities: 0 - Relationships: 0 + Message: Reading sheet: Sales (1/3) + Entities: 0 + Relationships: 0 Progress: extracting - Message: Parsing Excel (33%) - Entities: 0 - Relationships: 0 + Message: Parsing Excel (33%) + Entities: 0 + Relationships: 0 Progress: extracting - Message: Reading sheet: Products (2/3) - Entities: 0 - Relationships: 0 + Message: Reading sheet: Products (2/3) + Entities: 0 + Relationships: 0 ... (more progress updates) Progress: complete - Message: Import complete - Entities: 1523 - Relationships: 892 + Message: Import complete + Entities: 1523 + Relationships: 892 Import complete: 1523 entities created ``` @@ -70,19 +70,19 @@ The examples below show format-specific messages, but **you don't need format-sp ```typescript // ONE HANDLER FOR ALL FORMATS! function universalProgressHandler(progress) { - console.log(`[${progress.stage}] ${progress.message}`) + console.log(`[${progress.stage}] ${progress.message}`) - if (progress.processed && progress.total) { - console.log(` Progress: ${progress.processed}/${progress.total}`) - } + if (progress.processed && progress.total) { + console.log(` Progress: ${progress.processed}/${progress.total}`) + } - if (progress.entities || progress.relationships) { - console.log(` Extracted: ${progress.entities || 0} entities, ${progress.relationships || 0} relationships`) - } + if (progress.entities || progress.relationships) { + console.log(` Extracted: ${progress.entities || 0} entities, ${progress.relationships || 0} relationships`) + } - if (progress.throughput && progress.eta) { - console.log(` Rate: ${progress.throughput.toFixed(1)}/sec, ETA: ${Math.round(progress.eta/1000)}s`) - } + if (progress.throughput && progress.eta) { + console.log(` Rate: ${progress.throughput.toFixed(1)}/sec, ETA: ${Math.round(progress.eta/1000)}s`) + } } // Use it for ANY format! @@ -105,20 +105,20 @@ The examples below show **what messages look like** for different formats using ```typescript await brain.import(csvBuffer, { - format: 'csv', - onProgress: (progress) => { - if (progress.stage === 'extracting') { - // CSV reports: "Parsing CSV (45%)", "Extracted 1000 rows", etc. - console.log(progress.message) - } - } + format: 'csv', + onProgress: (progress) => { + if (progress.stage === 'extracting') { + // CSV reports: "Parsing CSV (45%)", "Extracted 1000 rows", etc. + console.log(progress.message) + } + } }) ``` **CSV Progress Messages:** - ✅ "Detecting CSV encoding and delimiter..." - ✅ "Parsing CSV rows (delimiter: ",")" -- ✅ "Parsed 75%" (via bytes processed) +- ✅ "Parsed 75%" (via bytes processed) - ✅ "Extracted 1000 rows" - ✅ "Converting types: 5000/10000 rows..." - ✅ "CSV processing complete: 10000 rows" @@ -129,12 +129,12 @@ await brain.import(csvBuffer, { ```typescript await brain.import(pdfBuffer, { - format: 'pdf', - onProgress: (progress) => { - // PDF reports exact page numbers - console.log(progress.message) - // Example: "Processing page 5 of 23" - } + format: 'pdf', + onProgress: (progress) => { + // PDF reports exact page numbers + console.log(progress.message) + // Example: "Processing page 5 of 23" + } }) ``` @@ -152,12 +152,12 @@ await brain.import(pdfBuffer, { ```typescript await brain.import(excelBuffer, { - format: 'excel', - onProgress: (progress) => { - // Excel reports sheet names - console.log(progress.message) - // Example: "Reading sheet: Q2 Sales (2/5)" - } + format: 'excel', + onProgress: (progress) => { + // Excel reports sheet names + console.log(progress.message) + // Example: "Reading sheet: Q2 Sales (2/5)" + } }) ``` @@ -175,11 +175,11 @@ await brain.import(excelBuffer, { ```typescript await brain.import(jsonBuffer, { - format: 'json', - onProgress: (progress) => { - // JSON reports every 10 nodes - console.log(`Processed ${progress.processed} nodes, found ${progress.entities} entities`) - } + format: 'json', + onProgress: (progress) => { + // JSON reports every 10 nodes + console.log(`Processed ${progress.processed} nodes, found ${progress.entities} entities`) + } }) ``` @@ -189,10 +189,10 @@ await brain.import(jsonBuffer, { ```typescript await brain.import(markdownString, { - format: 'markdown', - onProgress: (progress) => { - console.log(`Section ${progress.processed}/${progress.total}`) - } + format: 'markdown', + onProgress: (progress) => { + console.log(`Section ${progress.processed}/${progress.total}`) + } }) ``` @@ -204,44 +204,44 @@ await brain.import(markdownString, { ```typescript function ImportProgress({ file }: { file: File }) { - const [progress, setProgress] = useState({ - stage: 'idle', - message: '', - percent: 0, - entities: 0, - relationships: 0 - }) + const [progress, setProgress] = useState({ + stage: 'idle', + message: '', + percent: 0, + entities: 0, + relationships: 0 + }) - const handleImport = async () => { - const buffer = await file.arrayBuffer() + const handleImport = async () => { + const buffer = await file.arrayBuffer() - await brain.import(Buffer.from(buffer), { - onProgress: (p) => { - setProgress({ - stage: p.stage, - message: p.message, - // Estimate percentage from stage - percent: { - detecting: 10, - extracting: 50, - 'storing-vfs': 80, - 'storing-graph': 90, - complete: 100 - }[p.stage] || 0, - entities: p.entities || 0, - relationships: p.relationships || 0 - }) - } - }) - } + await brain.import(Buffer.from(buffer), { + onProgress: (p) => { + setProgress({ + stage: p.stage, + message: p.message, + // Estimate percentage from stage + percent: { + detecting: 10, + extracting: 50, + 'storing-vfs': 80, + 'storing-graph': 90, + complete: 100 + }[p.stage] || 0, + entities: p.entities || 0, + relationships: p.relationships || 0 + }) + } + }) + } - return ( -
- -

{progress.message}

-

Entities: {progress.entities} | Relationships: {progress.relationships}

-
- ) + return ( +
+ +

{progress.message}

+

Entities: {progress.entities} | Relationships: {progress.relationships}

+
+ ) } ``` @@ -255,13 +255,13 @@ import ora from 'ora' const spinner = ora('Starting import...').start() await brain.import(buffer, { - onProgress: (progress) => { - spinner.text = progress.message + onProgress: (progress) => { + spinner.text = progress.message - if (progress.stage === 'complete') { - spinner.succeed(`Import complete: ${progress.entities} entities`) - } - } + if (progress.stage === 'complete') { + spinner.succeed(`Import complete: ${progress.entities} entities`) + } + } }) ``` @@ -285,20 +285,20 @@ let startTime = Date.now() let lastUpdate = startTime await brain.import(buffer, { - onProgress: (progress) => { - const elapsed = Date.now() - startTime - const rate = progress.entities / (elapsed / 1000) // entities/sec + onProgress: (progress) => { + const elapsed = Date.now() - startTime + const rate = progress.entities / (elapsed / 1000) // entities/sec - console.clear() - console.log('Import Progress Dashboard') - console.log('========================') - console.log(`Stage: ${progress.stage}`) - console.log(`Status: ${progress.message}`) - console.log(`Entities: ${progress.entities}`) - console.log(`Relationships: ${progress.relationships}`) - console.log(`Rate: ${rate.toFixed(1)} entities/sec`) - console.log(`Elapsed: ${(elapsed / 1000).toFixed(1)}s`) - } + console.clear() + console.log('Import Progress Dashboard') + console.log('========================') + console.log(`Stage: ${progress.stage}`) + console.log(`Status: ${progress.message}`) + console.log(`Entities: ${progress.entities}`) + console.log(`Relationships: ${progress.relationships}`) + console.log(`Rate: ${rate.toFixed(1)} entities/sec`) + console.log(`Elapsed: ${(elapsed / 1000).toFixed(1)}s`) + } }) ``` @@ -310,21 +310,21 @@ await brain.import(buffer, { ```typescript const formatMessages = { - csv: (p) => `CSV: ${p.message}`, - pdf: (p) => `PDF: ${p.message}`, - excel: (p) => `Excel: ${p.message}`, - json: (p) => `JSON: ${p.processed} nodes, ${p.entities} entities`, - markdown: (p) => `Markdown: Section ${p.processed}/${p.total}`, - yaml: (p) => `YAML: ${p.processed} nodes`, - docx: (p) => `DOCX: ${p.processed} paragraphs` + csv: (p) => `CSV: ${p.message}`, + pdf: (p) => `PDF: ${p.message}`, + excel: (p) => `Excel: ${p.message}`, + json: (p) => `JSON: ${p.processed} nodes, ${p.entities} entities`, + markdown: (p) => `Markdown: Section ${p.processed}/${p.total}`, + yaml: (p) => `YAML: ${p.processed} nodes`, + docx: (p) => `DOCX: ${p.processed} paragraphs` } await brain.import(buffer, { - onProgress: (progress) => { - // Format is available in progress.stage metadata - const message = formatMessages[detectedFormat]?.(progress) || progress.message - console.log(message) - } + onProgress: (progress) => { + // Format is available in progress.stage metadata + const message = formatMessages[detectedFormat]?.(progress) || progress.message + console.log(message) + } }) ``` @@ -336,18 +336,18 @@ await brain.import(buffer, { ```typescript let lastUIUpdate = 0 -const THROTTLE_MS = 100 // Update UI max once per 100ms +const THROTTLE_MS = 100 // Update UI max once per 100ms await brain.import(buffer, { - onProgress: (progress) => { - const now = Date.now() - if (now - lastUIUpdate < THROTTLE_MS && progress.stage !== 'complete') { - return // Skip this update - } + onProgress: (progress) => { + const now = Date.now() + if (now - lastUIUpdate < THROTTLE_MS && progress.stage !== 'complete') { + return // Skip this update + } - lastUIUpdate = now - updateUI(progress) // Only update every 100ms - } + lastUIUpdate = now + updateUI(progress) // Only update every 100ms + } }) ``` diff --git a/docs/guides/import-progress-implementation.md b/docs/guides/import-progress-implementation.md index 2b6ac5b7..efc82be8 100644 --- a/docs/guides/import-progress-implementation.md +++ b/docs/guides/import-progress-implementation.md @@ -28,17 +28,17 @@ ```typescript // THE PUBLIC API - Same for ALL 7 formats! brain.import(buffer, { - onProgress: (progress: ImportProgress) => { - // These fields work for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX - progress.stage // 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete' - progress.message // Human-readable status (varies by format, always readable) - progress.processed // Items processed (optional) - progress.total // Total items (optional) - progress.entities // Entities extracted (optional) - progress.relationships // Relationships inferred (optional) - progress.throughput // Items/sec (optional, during extraction) - progress.eta // Time remaining in ms (optional) - } + onProgress: (progress: ImportProgress) => { + // These fields work for CSV, PDF, Excel, JSON, Markdown, YAML, DOCX + progress.stage // 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'complete' + progress.message // Human-readable status (varies by format, always readable) + progress.processed // Items processed (optional) + progress.total // Total items (optional) + progress.entities // Entities extracted (optional) + progress.relationships // Relationships inferred (optional) + progress.throughput // Items/sec (optional, during extraction) + progress.eta // Time remaining in ms (optional) + } }) ``` @@ -49,14 +49,14 @@ The table below shows how formats implement progress *internally*. Normal develo ```typescript // Internal: Binary formats use handler hooks (you added these!) interface FormatHandlerProgressHooks { - onBytesProcessed?: (bytes: number) => void - onCurrentItem?: (message: string) => void - onDataExtracted?: (count: number, total?: number) => void + onBytesProcessed?: (bytes: number) => void + onCurrentItem?: (message: string) => void + onDataExtracted?: (count: number, total?: number) => void } // Internal: Text formats use importer callbacks interface ImporterProgressCallback { - onProgress?: (stats: { processed, total, entities, relationships }) => void + onProgress?: (stats: { processed, total, entities, relationships }) => void } // Both are converted to ImportProgress by ImportCoordinator! @@ -74,7 +74,7 @@ interface ImporterProgressCallback { ## 🎯 Overview -As of v4.5.0, Brainy supports comprehensive, multi-dimensional progress tracking for imports: +Brainy supports comprehensive, multi-dimensional progress tracking for imports: - **Bytes processed** (always available, most deterministic) - **Entities extracted** (AI extraction phase) - **Stage-specific metrics** (parsing: MB/s, extraction: entities/s) @@ -91,23 +91,23 @@ All handlers follow a simple, consistent pattern using **progress hooks**. ```typescript export interface FormatHandlerProgressHooks { - /** - * Report bytes processed - * Call this as you read/parse the file - */ - onBytesProcessed?: (bytes: number) => void + /** + * Report bytes processed + * Call this as you read/parse the file + */ + onBytesProcessed?: (bytes: number) => void - /** - * Set current processing context - * Examples: "Processing page 5", "Reading sheet: Q2 Sales" - */ - onCurrentItem?: (item: string) => void + /** + * Set current processing context + * Examples: "Processing page 5", "Reading sheet: Q2 Sales" + */ + onCurrentItem?: (item: string) => void - /** - * Report structured data extraction progress - * Examples: "Extracted 100 rows", "Parsed 50 paragraphs" - */ - onDataExtracted?: (count: number, total?: number) => void + /** + * Report structured data extraction progress + * Examples: "Extracted 100 rows", "Parsed 50 paragraphs" + */ + onDataExtracted?: (count: number, total?: number) => void } ``` @@ -117,19 +117,19 @@ Progress hooks are automatically passed to your handler via `FormatHandlerOption ```typescript export interface FormatHandlerOptions { - // ... existing options ... + // ... existing options ... - /** - * Progress hooks (v4.5.0) - * Handlers call these to report progress during processing - */ - progressHooks?: FormatHandlerProgressHooks + /** + * Progress hooks + * Handlers call these to report progress during processing + */ + progressHooks?: FormatHandlerProgressHooks - /** - * Total file size in bytes (v4.5.0) - * Used for progress percentage calculation - */ - totalBytes?: number + /** + * Total file size in bytes + * Used for progress percentage calculation + */ + totalBytes?: number } ``` @@ -141,36 +141,36 @@ Every handler follows these 5 steps: ```typescript async process(data: Buffer | string, options: FormatHandlerOptions): Promise { - const progressHooks = options.progressHooks // Step 1: Get hooks + const progressHooks = options.progressHooks // Step 1: Get hooks - // Step 2: Report initial progress - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem('Starting import...') - } + // Step 2: Report initial progress + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Starting import...') + } - // Step 3: Report bytes as you process - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data) - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(0) // Start - } + // Step 3: Report bytes as you process + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(0) // Start + } - // ... do parsing ... + // ... do parsing ... - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(buffer.length) // Complete - } + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(buffer.length) // Complete + } - // Step 4: Report data extraction - if (progressHooks?.onDataExtracted) { - progressHooks.onDataExtracted(data.length, data.length) - } + // Step 4: Report data extraction + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(data.length, data.length) + } - // Step 5: Report completion - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`Complete: ${data.length} items processed`) - } + // Step 5: Report completion + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Complete: ${data.length} items processed`) + } - return { format, data, metadata } + return { format, data, metadata } } ``` @@ -182,76 +182,76 @@ Here's the **ACTUAL implementation** from CSV handler showing all the key progre ```typescript async process(data: Buffer | string, options: FormatHandlerOptions): Promise { - const startTime = Date.now() - const progressHooks = options.progressHooks // ✅ Step 1 + const startTime = Date.now() + const progressHooks = options.progressHooks // ✅ Step 1 - // Convert to buffer if string - const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8') - const totalBytes = buffer.length + // Convert to buffer if string + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8') + const totalBytes = buffer.length - // ✅ Step 2: Report start - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(0) - } - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...') - } + // ✅ Step 2: Report start + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(0) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Detecting CSV encoding and delimiter...') + } - // Detect encoding - const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer) - const text = buffer.toString(detectedEncoding as BufferEncoding) + // Detect encoding + const detectedEncoding = options.encoding || this.detectEncodingSafe(buffer) + const text = buffer.toString(detectedEncoding as BufferEncoding) - // Detect delimiter - const delimiter = options.csvDelimiter || this.detectDelimiter(text) + // Detect delimiter + const delimiter = options.csvDelimiter || this.detectDelimiter(text) - // ✅ Progress update: Parsing phase - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`) - } + // ✅ Progress update: Parsing phase + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`) + } - // Parse CSV - const records = parse(text, { /* options */ }) + // Parse CSV + const records = parse(text, { /* options */ }) - // ✅ Step 3: Report bytes processed (entire file parsed) - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(totalBytes) - } + // ✅ Step 3: Report bytes processed (entire file parsed) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } - const data = Array.isArray(records) ? records : [records] + const data = Array.isArray(records) ? records : [records] - // ✅ Step 4: Report data extraction - if (progressHooks?.onDataExtracted) { - progressHooks.onDataExtracted(data.length, data.length) - } - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`) - } + // ✅ Step 4: Report data extraction + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(data.length, data.length) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Extracted ${data.length} rows, inferring types...`) + } - // Type inference and conversion - const fields = data.length > 0 ? Object.keys(data[0]) : [] - const types = this.inferFieldTypes(data) + // Type inference and conversion + const fields = data.length > 0 ? Object.keys(data[0]) : [] + const types = this.inferFieldTypes(data) - const convertedData = data.map((row, index) => { - const converted = this.convertRow(row, types) + const convertedData = data.map((row, index) => { + const converted = this.convertRow(row, types) - // ✅ Progress update every 1000 rows (avoid spam) - if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) { - progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`) - } + // ✅ Progress update every 1000 rows (avoid spam) + if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) { + progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`) + } - return converted - }) + return converted + }) - // ✅ Step 5: Report completion - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`) - } + // ✅ Step 5: Report completion + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`) + } - return { - format: this.format, - data: convertedData, - metadata: { /* ... */ } - } + return { + format: this.format, + data: convertedData, + metadata: { /* ... */ } + } } ``` @@ -292,51 +292,51 @@ Brainy supports **7 file formats** with full progress tracking: ```typescript async process(data: Buffer, options: FormatHandlerOptions): Promise { - const progressHooks = options.progressHooks - const totalBytes = data.length + const progressHooks = options.progressHooks + const totalBytes = data.length - // Report start - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem('Loading PDF document...') - } + // Report start + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Loading PDF document...') + } - const pdfDoc = await loadPDF(data) - const totalPages = pdfDoc.numPages + const pdfDoc = await loadPDF(data) + const totalPages = pdfDoc.numPages - const extractedData: any[] = [] - let bytesProcessed = 0 + const extractedData: any[] = [] + let bytesProcessed = 0 - for (let pageNum = 1; pageNum <= totalPages; pageNum++) { - // ✅ Report current page - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`Processing page ${pageNum} of ${totalPages}`) - } + for (let pageNum = 1; pageNum <= totalPages; pageNum++) { + // ✅ Report current page + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Processing page ${pageNum} of ${totalPages}`) + } - const page = await pdfDoc.getPage(pageNum) - const text = await page.getTextContent() - extractedData.push(this.processPageText(text)) + const page = await pdfDoc.getPage(pageNum) + const text = await page.getTextContent() + extractedData.push(this.processPageText(text)) - // ✅ Estimate bytes processed (pages are sequential) - bytesProcessed = Math.floor((pageNum / totalPages) * totalBytes) - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(bytesProcessed) - } + // ✅ Estimate bytes processed (pages are sequential) + bytesProcessed = Math.floor((pageNum / totalPages) * totalBytes) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(bytesProcessed) + } - // ✅ Report extraction progress - if (progressHooks?.onDataExtracted) { - progressHooks.onDataExtracted(pageNum, totalPages) - } - } + // ✅ Report extraction progress + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(pageNum, totalPages) + } + } - // Final progress - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(totalBytes) - } - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`PDF complete: ${totalPages} pages processed`) - } + // Final progress + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`PDF complete: ${totalPages} pages processed`) + } - return { format: 'pdf', data: extractedData, metadata: { /* ... */ } } + return { format: 'pdf', data: extractedData, metadata: { /* ... */ } } } ``` @@ -344,55 +344,55 @@ async process(data: Buffer, options: FormatHandlerOptions): Promise { - const progressHooks = options.progressHooks - const totalBytes = data.length + const progressHooks = options.progressHooks + const totalBytes = data.length - // Load workbook - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem('Loading Excel workbook...') - } + // Load workbook + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem('Loading Excel workbook...') + } - const workbook = XLSX.read(data) - const sheetNames = options.excelSheets === 'all' - ? workbook.SheetNames - : (options.excelSheets || [workbook.SheetNames[0]]) + const workbook = XLSX.read(data) + const sheetNames = options.excelSheets === 'all' + ? workbook.SheetNames + : (options.excelSheets || [workbook.SheetNames[0]]) - const allData: any[] = [] - let bytesProcessed = 0 + const allData: any[] = [] + let bytesProcessed = 0 - for (let i = 0; i < sheetNames.length; i++) { - const sheetName = sheetNames[i] + for (let i = 0; i < sheetNames.length; i++) { + const sheetName = sheetNames[i] - // ✅ Report current sheet - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`Reading sheet: ${sheetName} (${i + 1}/${sheetNames.length})`) - } + // ✅ Report current sheet + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Reading sheet: ${sheetName} (${i + 1}/${sheetNames.length})`) + } - const sheet = workbook.Sheets[sheetName] - const sheetData = XLSX.utils.sheet_to_json(sheet) - allData.push(...sheetData) + const sheet = workbook.Sheets[sheetName] + const sheetData = XLSX.utils.sheet_to_json(sheet) + allData.push(...sheetData) - // ✅ Estimate bytes processed (sheets processed sequentially) - bytesProcessed = Math.floor(((i + 1) / sheetNames.length) * totalBytes) - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(bytesProcessed) - } + // ✅ Estimate bytes processed (sheets processed sequentially) + bytesProcessed = Math.floor(((i + 1) / sheetNames.length) * totalBytes) + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(bytesProcessed) + } - // ✅ Report data extraction - if (progressHooks?.onDataExtracted) { - progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until done - } - } + // ✅ Report data extraction + if (progressHooks?.onDataExtracted) { + progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until done + } + } - // Final progress - if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(totalBytes) - } - if (progressHooks?.onCurrentItem) { - progressHooks.onCurrentItem(`Excel complete: ${sheetNames.length} sheets, ${allData.length} rows`) - } + // Final progress + if (progressHooks?.onBytesProcessed) { + progressHooks.onBytesProcessed(totalBytes) + } + if (progressHooks?.onCurrentItem) { + progressHooks.onCurrentItem(`Excel complete: ${sheetNames.length} sheets, ${allData.length} rows`) + } - return { format: 'xlsx', data: allData, metadata: { /* ... */ } } + return { format: 'xlsx', data: allData, metadata: { /* ... */ } } } ``` @@ -400,44 +400,44 @@ async process(data: Buffer, options: FormatHandlerOptions): Promise { - // ✅ Report parsing start - options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + // ✅ Report parsing start + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) - // Parse JSON if string - let jsonData = typeof data === 'string' ? JSON.parse(data) : data + // Parse JSON if string + let jsonData = typeof data === 'string' ? JSON.parse(data) : data - // ✅ Report parsing complete - options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + // ✅ Report parsing complete + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) - // Traverse and extract (reports progress every 10 nodes) - const entities: ExtractedJSONEntity[] = [] - const relationships: ExtractedJSONRelationship[] = [] - let nodesProcessed = 0 + // Traverse and extract (reports progress every 10 nodes) + const entities: ExtractedJSONEntity[] = [] + const relationships: ExtractedJSONRelationship[] = [] + let nodesProcessed = 0 - await this.traverseJSON( - jsonData, - entities, - relationships, - () => { - nodesProcessed++ - if (nodesProcessed % 10 === 0) { - options.onProgress?.({ - processed: nodesProcessed, - entities: entities.length, - relationships: relationships.length - }) - } - } - ) + await this.traverseJSON( + jsonData, + entities, + relationships, + () => { + nodesProcessed++ + if (nodesProcessed % 10 === 0) { + options.onProgress?.({ + processed: nodesProcessed, + entities: entities.length, + relationships: relationships.length + }) + } + } + ) - // ✅ Report completion - options.onProgress?.({ - processed: nodesProcessed, - entities: entities.length, - relationships: relationships.length - }) + // ✅ Report completion + options.onProgress?.({ + processed: nodesProcessed, + entities: entities.length, + relationships: relationships.length + }) - return { nodesProcessed, entitiesExtracted: entities.length, ... } + return { nodesProcessed, entitiesExtracted: entities.length, ... } } ``` @@ -445,38 +445,38 @@ async extract(data: any, options: SmartJSONOptions = {}): Promise { - // ✅ Report parsing start - options.onProgress?.({ processed: 0, total: 0, entities: 0, relationships: 0 }) + // ✅ Report parsing start + options.onProgress?.({ processed: 0, total: 0, entities: 0, relationships: 0 }) - // Parse markdown into sections - const parsedSections = this.parseMarkdown(markdown, options) + // Parse markdown into sections + const parsedSections = this.parseMarkdown(markdown, options) - // ✅ Report parsing complete - options.onProgress?.({ processed: 0, total: parsedSections.length, entities: 0, relationships: 0 }) + // ✅ Report parsing complete + options.onProgress?.({ processed: 0, total: parsedSections.length, entities: 0, relationships: 0 }) - // Process each section (reports progress after each section) - const sections: MarkdownSection[] = [] - for (let i = 0; i < parsedSections.length; i++) { - const section = await this.processSection(parsedSections[i], options) - sections.push(section) + // Process each section (reports progress after each section) + const sections: MarkdownSection[] = [] + for (let i = 0; i < parsedSections.length; i++) { + const section = await this.processSection(parsedSections[i], options) + sections.push(section) - options.onProgress?.({ - processed: i + 1, - total: parsedSections.length, - entities: sections.reduce((sum, s) => sum + s.entities.length, 0), - relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0) - }) - } + options.onProgress?.({ + processed: i + 1, + total: parsedSections.length, + entities: sections.reduce((sum, s) => sum + s.entities.length, 0), + relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0) + }) + } - // ✅ Report completion - options.onProgress?.({ - processed: sections.length, - total: sections.length, - entities: sections.reduce((sum, s) => sum + s.entities.length, 0), - relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0) - }) + // ✅ Report completion + options.onProgress?.({ + processed: sections.length, + total: sections.length, + entities: sections.reduce((sum, s) => sum + s.entities.length, 0), + relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0) + }) - return { sectionsProcessed: sections.length, ... } + return { sectionsProcessed: sections.length, ... } } ``` @@ -484,27 +484,27 @@ async extract(markdown: string, options: SmartMarkdownOptions = {}): Promise { - // ✅ Report parsing start - options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + // ✅ Report parsing start + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) - // Parse YAML - const yamlString = typeof yamlContent === 'string' ? yamlContent : yamlContent.toString('utf-8') - const data = yaml.load(yamlString) + // Parse YAML + const yamlString = typeof yamlContent === 'string' ? yamlContent : yamlContent.toString('utf-8') + const data = yaml.load(yamlString) - // ✅ Report parsing complete - options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + // ✅ Report parsing complete + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) - // Traverse YAML structure (reports progress every 10 nodes) - // ... similar to JSON traversal ... + // Traverse YAML structure (reports progress every 10 nodes) + // ... similar to JSON traversal ... - // ✅ Report completion (already implemented) - options.onProgress?.({ - processed: nodesProcessed, - entities: entities.length, - relationships: relationships.length - }) + // ✅ Report completion (already implemented) + options.onProgress?.({ + processed: nodesProcessed, + entities: entities.length, + relationships: relationships.length + }) - return { nodesProcessed, entitiesExtracted: entities.length, ... } + return { nodesProcessed, entitiesExtracted: entities.length, ... } } ``` @@ -512,39 +512,39 @@ async extract(yamlContent: string | Buffer, options: SmartYAMLOptions = {}): Pro ```typescript async extract(buffer: Buffer, options: SmartDOCXOptions = {}): Promise { - // ✅ Report parsing start - options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + // ✅ Report parsing start + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) - // Extract text and HTML using Mammoth - const textResult = await mammoth.extractRawText({ buffer }) - const htmlResult = await mammoth.convertToHtml({ buffer }) + // Extract text and HTML using Mammoth + const textResult = await mammoth.extractRawText({ buffer }) + const htmlResult = await mammoth.convertToHtml({ buffer }) - // ✅ Report parsing complete - options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) + // ✅ Report parsing complete + options.onProgress?.({ processed: 0, entities: 0, relationships: 0 }) - // Process paragraphs (reports progress every 10 paragraphs) - const paragraphs = textResult.value.split(/\n\n+/).filter(p => p.trim().length >= minLength) + // Process paragraphs (reports progress every 10 paragraphs) + const paragraphs = textResult.value.split(/\n\n+/).filter(p => p.trim().length >= minLength) - for (let i = 0; i < paragraphs.length; i++) { - await this.processParagraph(paragraphs[i]) + for (let i = 0; i < paragraphs.length; i++) { + await this.processParagraph(paragraphs[i]) - if (i % 10 === 0) { - options.onProgress?.({ - processed: i + 1, - entities: entities.length, - relationships: relationships.length - }) - } - } + if (i % 10 === 0) { + options.onProgress?.({ + processed: i + 1, + entities: entities.length, + relationships: relationships.length + }) + } + } - // ✅ Report completion (already implemented) - options.onProgress?.({ - processed: paragraphs.length, - entities: entities.length, - relationships: relationships.length - }) + // ✅ Report completion (already implemented) + options.onProgress?.({ + processed: paragraphs.length, + entities: entities.length, + relationships: relationships.length + }) - return { paragraphsProcessed: paragraphs.length, ... } + return { paragraphsProcessed: paragraphs.length, ... } } ``` @@ -559,20 +559,20 @@ Progress hooks are **optional**. Always check before calling: ```typescript // ✅ Good - safe if (progressHooks?.onBytesProcessed) { - progressHooks.onBytesProcessed(bytes) + progressHooks.onBytesProcessed(bytes) } // ❌ Bad - will crash if hooks undefined -progressHooks.onBytesProcessed(bytes) // TypeError! +progressHooks.onBytesProcessed(bytes) // TypeError! ``` ### 2. Report Bytes at Start and End ```typescript // ✅ Good - clear start and end -progressHooks?.onBytesProcessed(0) // Start +progressHooks?.onBytesProcessed(0) // Start // ... processing ... -progressHooks?.onBytesProcessed(totalBytes) // End +progressHooks?.onBytesProcessed(totalBytes) // End // ❌ Bad - no clear boundaries // ... just start processing without reporting start @@ -583,17 +583,17 @@ progressHooks?.onBytesProcessed(totalBytes) // End ```typescript // ✅ Good - report every 1000 items for (let i = 0; i < items.length; i++) { - processItem(items[i]) + processItem(items[i]) - if (i > 0 && i % 1000 === 0) { - progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`) - } + if (i > 0 && i % 1000 === 0) { + progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`) + } } // ❌ Bad - report EVERY item (spam!) for (let i = 0; i < items.length; i++) { - processItem(items[i]) - progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`) // 1M callbacks! + processItem(items[i]) + progressHooks?.onCurrentItem(`Processing: ${i}/${items.length}`) // 1M callbacks! } ``` @@ -614,13 +614,13 @@ progressHooks?.onCurrentItem('Working...') ```typescript // ✅ Good - total known -progressHooks?.onDataExtracted(100, 1000) // 100 of 1000 rows +progressHooks?.onDataExtracted(100, 1000) // 100 of 1000 rows // ✅ Also good - total unknown (streaming) -progressHooks?.onDataExtracted(100, undefined) // 100 rows so far +progressHooks?.onDataExtracted(100, undefined) // 100 rows so far // ✅ Also good - complete -progressHooks?.onDataExtracted(1000, 1000) // All 1000 rows +progressHooks?.onDataExtracted(1000, 1000) // All 1000 rows ``` --- @@ -637,18 +637,18 @@ const handler = new CSVHandler() const data = fs.readFileSync('./test.csv') const result = await handler.process(data, { - filename: 'test.csv', - progressHooks: { - onBytesProcessed: (bytes) => { - console.log(`Bytes: ${bytes}`) - }, - onCurrentItem: (item) => { - console.log(`Status: ${item}`) - }, - onDataExtracted: (count, total) => { - console.log(`Extracted: ${count}${total ? `/${total}` : ''}`) - } - } + filename: 'test.csv', + progressHooks: { + onBytesProcessed: (bytes) => { + console.log(`Bytes: ${bytes}`) + }, + onCurrentItem: (item) => { + console.log(`Status: ${item}`) + }, + onDataExtracted: (count, total) => { + console.log(`Extracted: ${count}${total ? `/${total}` : ''}`) + } + } }) console.log(`Complete: ${result.data.length} rows`) @@ -673,24 +673,24 @@ Complete: 1000 rows ``` User Imports File - ↓ + ↓ ImportManager - ↓ + ↓ Creates ProgressTracker - ↓ + ↓ Calls Handler.process() with progressHooks - ↓ + ↓ Handler Reports Progress: - ├─ onBytesProcessed(0) → ProgressTracker → overall_progress calculated - ├─ onCurrentItem("Parsing...") → ProgressTracker → stage_message updated - ├─ onBytesProcessed(bytes) → ProgressTracker → bytes_per_second calculated - ├─ onDataExtracted(count) → ProgressTracker → entities_extracted updated - └─ onCurrentItem("Complete") → ProgressTracker → final progress - ↓ + ├─ onBytesProcessed(0) → ProgressTracker → overall_progress calculated + ├─ onCurrentItem("Parsing...") → ProgressTracker → stage_message updated + ├─ onBytesProcessed(bytes) → ProgressTracker → bytes_per_second calculated + ├─ onDataExtracted(count) → ProgressTracker → entities_extracted updated + └─ onCurrentItem("Complete") → ProgressTracker → final progress + ↓ ProgressTracker emits to callback (throttled 100ms) - ↓ + ↓ User sees: - "Overall: 45% | PARSING | 12.5 MB/s | Parsing CSV rows..." + "Overall: 45% | PARSING | 12.5 MB/s | Parsing CSV rows..." ``` --- diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index c337247c..1b0e3b80 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -91,7 +91,7 @@ await brain.import(file, { }) ``` -### Import Tracking (v4.10.0+) +### Import Tracking Track and organize imports by project: @@ -149,7 +149,7 @@ await brain.import(file, { }) ``` -### Always-On Streaming (v4.2.0+) +### Always-On Streaming All imports use streaming with adaptive flush intervals. Query data as it's imported: diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md index 389bfc7d..9f2e2e5b 100644 --- a/docs/guides/standard-import-progress.md +++ b/docs/guides/standard-import-progress.md @@ -65,7 +65,7 @@ interface ImportProgress { /** Estimated time remaining (milliseconds) */ eta?: number - /** Whether data is queryable at this point (v4.2.0+) */ + /** Whether data is queryable at this point */ queryable?: boolean } ``` diff --git a/docs/guides/streaming-imports.md b/docs/guides/streaming-imports.md index 025b0228..6fc80b44 100644 --- a/docs/guides/streaming-imports.md +++ b/docs/guides/streaming-imports.md @@ -300,7 +300,7 @@ interface ImportProgress { relationships?: number // Relationships inferred so far /** - * Whether data is queryable (v4.2.0+) + * Whether data is queryable * * true = Indexes flushed, queries will be fast and complete * false/undefined = Data in storage but indexes not flushed yet @@ -376,7 +376,7 @@ No changes required! Streaming is now always enabled with optimal defaults: // Before (v3.x, v4.0, v4.1): Works the same await brain.import(file) -// After (v4.2.0+): Streaming always on, zero config +// After: Streaming always on, zero config await brain.import(file) ``` diff --git a/docs/operations/capacity-planning.md b/docs/operations/capacity-planning.md index 3105c87b..263b37c0 100644 --- a/docs/operations/capacity-planning.md +++ b/docs/operations/capacity-planning.md @@ -1,6 +1,6 @@ # Capacity Planning & Operations Guide -**Brainy v3.36.0+ Enterprise Operations** +**Brainy Enterprise Operations** This guide provides production-ready capacity planning formulas, deployment strategies, and operational guidelines for scaling Brainy from development (2GB) to enterprise (128GB+) deployments. @@ -24,13 +24,13 @@ Where: ### Adaptive Caching Strategy ``` -estimatedVectorMemory = entityCount × 1536 bytes // 384 dims × 4 bytes per float -hnswCacheBudget = cacheSize × 0.80 // 80% threshold for preloading decision +estimatedVectorMemory = entityCount × 1536 bytes // 384 dims × 4 bytes per float +hnswCacheBudget = cacheSize × 0.80 // 80% threshold for preloading decision if estimatedVectorMemory < hnswCacheBudget: - cachingStrategy = 'preloaded' // All vectors loaded at init + cachingStrategy = 'preloaded' // All vectors loaded at init else: - cachingStrategy = 'on-demand' // Vectors loaded adaptively via UnifiedCache + cachingStrategy = 'on-demand' // Vectors loaded adaptively via UnifiedCache ``` --- @@ -46,19 +46,19 @@ else: **Memory Breakdown:** ``` -System Memory: 2048 MB -OS Reserved (20%): -410 MB -Available: 1638 MB -Model Memory: -140 MB - ├─ WASM + Weights: 90 MB - └─ Workspace: 50 MB +System Memory: 2048 MB +OS Reserved (20%): -410 MB +Available: 1638 MB +Model Memory: -140 MB + ├─ WASM + Weights: 90 MB + └─ Workspace: 50 MB ─────────────────────────── -Available for Cache: 1488 MB +Available for Cache: 1488 MB Dev Allocation (25%): 372 MB UnifiedCache - ├─ HNSW (30%): 112 MB - ├─ Metadata (40%): 149 MB - ├─ Search (20%): 74 MB - └─ Shared (10%): 37 MB + ├─ HNSW (30%): 112 MB + ├─ Metadata (40%): 149 MB + ├─ Search (20%): 74 MB + └─ Shared (10%): 37 MB ``` **Capacity:** @@ -75,9 +75,9 @@ Dev Allocation (25%): 372 MB UnifiedCache **Configuration:** ```typescript const brain = new Brainy({ - storage: { type: 'filesystem', path: './brainy-data' }, - model: { precision: 'q8' }, - cache: { /* auto-sized to 372MB */ } + storage: { type: 'filesystem', path: './brainy-data' }, + model: { precision: 'q8' }, + cache: { /* auto-sized to 372MB */ } }) ``` @@ -92,17 +92,17 @@ const brain = new Brainy({ **Memory Breakdown:** ``` -System Memory: 8192 MB -OS Reserved (20%): -1638 MB -Available: 6554 MB -Model Memory (Q8): -150 MB +System Memory: 8192 MB +OS Reserved (20%): -1638 MB +Available: 6554 MB +Model Memory (Q8): -150 MB ─────────────────────────── -Available for Cache: 6404 MB +Available for Cache: 6404 MB Prod Allocation (50%): 3202 MB UnifiedCache - ├─ HNSW (30%): 961 MB - ├─ Metadata (40%): 1281 MB - ├─ Search (20%): 640 MB - └─ Shared (10%): 320 MB + ├─ HNSW (30%): 961 MB + ├─ Metadata (40%): 1281 MB + ├─ Search (20%): 640 MB + └─ Shared (10%): 320 MB ``` **Capacity:** @@ -119,9 +119,9 @@ Prod Allocation (50%): 3202 MB UnifiedCache **Configuration:** ```typescript const brain = new Brainy({ - storage: { type: 'filesystem', path: '/var/lib/brainy' }, - model: { precision: 'q8' }, - // Auto-sized cache: 3202MB + storage: { type: 'filesystem', path: '/var/lib/brainy' }, + model: { precision: 'q8' }, + // Auto-sized cache: 3202MB }) // Monitor health @@ -141,17 +141,17 @@ console.log(`Caching strategy: ${stats.cachingStrategy}`) **Memory Breakdown:** ``` -System Memory: 32768 MB -OS Reserved (20%): -6554 MB -Available: 26214 MB -Model Memory (Q8): -150 MB +System Memory: 32768 MB +OS Reserved (20%): -6554 MB +Available: 26214 MB +Model Memory (Q8): -150 MB ─────────────────────────── -Available for Cache: 26064 MB +Available for Cache: 26064 MB Prod Allocation (50%): 13032 MB UnifiedCache - ├─ HNSW (30%): 3910 MB - ├─ Metadata (40%): 5213 MB - ├─ Search (20%): 2606 MB - └─ Shared (10%): 1303 MB + ├─ HNSW (30%): 3910 MB + ├─ Metadata (40%): 5213 MB + ├─ Search (20%): 2606 MB + └─ Shared (10%): 1303 MB ``` **Capacity:** @@ -168,11 +168,11 @@ Prod Allocation (50%): 13032 MB UnifiedCache **Configuration:** ```typescript const brain = new Brainy({ - storage: { - type: 'gcs-native', - gcsNativeStorage: { bucketName: 'production-data' } - }, - model: { precision: 'q8' } // or 'fp32' for +0.5% accuracy + storage: { + type: 'gcs-native', + gcsNativeStorage: { bucketName: 'production-data' } + }, + model: { precision: 'q8' } // or 'fp32' for +0.5% accuracy }) // Verify allocation @@ -192,17 +192,17 @@ console.log(`Environment: ${memoryInfo.memoryInfo.environment}`) **Memory Breakdown:** ``` -System Memory: 131072 MB -OS Reserved (20%): -26214 MB -Available: 104858 MB -Model Memory (FP32): -250 MB +System Memory: 131072 MB +OS Reserved (20%): -26214 MB +Available: 104858 MB +Model Memory (FP32): -250 MB ─────────────────────────────── -Available for Cache: 104608 MB +Available for Cache: 104608 MB Prod Allocation (50%): 52304 MB UnifiedCache (logarithmic scaling applies) - ├─ HNSW (30%): 15691 MB - ├─ Metadata (40%): 20922 MB - ├─ Search (20%): 10461 MB - └─ Shared (10%): 5230 MB + ├─ HNSW (30%): 15691 MB + ├─ Metadata (40%): 20922 MB + ├─ Search (20%): 10461 MB + └─ Shared (10%): 5230 MB ``` **Logarithmic Scaling Applied:** @@ -227,30 +227,30 @@ Actual cache size: ~40GB (prevents waste on 128GB systems) **Configuration:** ```typescript const brain = new Brainy({ - storage: { - type: 's3', - s3Storage: { - bucketName: 'enterprise-data', - region: 'us-east-1' - } - }, - model: { precision: 'fp32' } // Maximum accuracy + storage: { + type: 's3', + s3Storage: { + bucketName: 'enterprise-data', + region: 'us-east-1' + } + }, + model: { precision: 'fp32' } // Maximum accuracy }) // Enterprise monitoring setInterval(() => { - const stats = brain.hnsw.getCacheStats() + const stats = brain.hnsw.getCacheStats() - if (stats.fairness.fairnessViolation) { - console.warn('FAIRNESS VIOLATION: HNSW using too much cache') - console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access, ${stats.hnswCache.sizePercent}% size`) - } + if (stats.fairness.fairnessViolation) { + console.warn('FAIRNESS VIOLATION: HNSW using too much cache') + console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access, ${stats.hnswCache.sizePercent}% size`) + } - if (stats.unifiedCache.hitRatePercent < 75) { - console.warn(`Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) - console.warn('Recommendations:', stats.recommendations) - } -}, 60000) // Check every minute + if (stats.unifiedCache.hitRatePercent < 75) { + console.warn(`Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) + console.warn('Recommendations:', stats.recommendations) + } +}, 60000) // Check every minute ``` --- @@ -263,12 +263,12 @@ Brainy auto-detects container memory limits via cgroups v1/v2: ```typescript // Automatic detection -const brain = new Brainy() // Detects cgroup limits automatically +const brain = new Brainy() // Detects cgroup limits automatically // Verify detection const memoryInfo = brain.hnsw.unifiedCache.getMemoryInfo() console.log(`Container: ${memoryInfo.memoryInfo.isContainer}`) -console.log(`Source: ${memoryInfo.memoryInfo.source}`) // 'cgroup-v2' or 'cgroup-v1' +console.log(`Source: ${memoryInfo.memoryInfo.source}`) // 'cgroup-v2' or 'cgroup-v1' console.log(`Limit: ${Math.round(memoryInfo.memoryInfo.available / 1024 / 1024)}MB`) ``` @@ -294,37 +294,37 @@ CMD ["node", "dist/index.js"] ```bash docker run \ - --memory="2g" \ - --memory-reservation="1.5g" \ - --cpus="2" \ - my-brainy-app + --memory="2g" \ + --memory-reservation="1.5g" \ + --cpus="2" \ + my-brainy-app ``` **Expected allocation:** ``` -Container Limit: 2048 MB -Available: 1638 MB (80% usable) -Model Memory: -150 MB -Available for Cache: 1488 MB +Container Limit: 2048 MB +Available: 1638 MB (80% usable) +Model Memory: -150 MB +Available for Cache: 1488 MB Container Ratio (40%): 595 MB UnifiedCache ``` **Medium Container (8GB)** ```bash docker run \ - --memory="8g" \ - --memory-reservation="6g" \ - --cpus="4" \ - -e NODE_OPTIONS="--max-old-space-size=6144" \ - my-brainy-app + --memory="8g" \ + --memory-reservation="6g" \ + --cpus="4" \ + -e NODE_OPTIONS="--max-old-space-size=6144" \ + my-brainy-app ``` **Expected allocation:** ``` -Container Limit: 8192 MB -Available: 6554 MB -Model Memory: -150 MB -Available for Cache: 6404 MB +Container Limit: 8192 MB +Available: 6554 MB +Model Memory: -150 MB +Available for Cache: 6404 MB Container Ratio (40%): 2562 MB UnifiedCache ``` @@ -335,24 +335,24 @@ Container Ratio (40%): 2562 MB UnifiedCache apiVersion: apps/v1 kind: Deployment metadata: - name: brainy-api + name: brainy-api spec: - replicas: 3 - template: - spec: - containers: - - name: brainy - image: my-brainy-app:latest - resources: - requests: - memory: "1.5Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "1000m" - env: - - name: NODE_OPTIONS - value: "--max-old-space-size=1536" + replicas: 3 + template: + spec: + containers: + - name: brainy + image: my-brainy-app:latest + resources: + requests: + memory: "1.5Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1000m" + env: + - name: NODE_OPTIONS + value: "--max-old-space-size=1536" ``` **Medium Pod (8GB)** @@ -360,24 +360,24 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: brainy-api + name: brainy-api spec: - replicas: 2 - template: - spec: - containers: - - name: brainy - image: my-brainy-app:latest - resources: - requests: - memory: "6Gi" - cpu: "2000m" - limits: - memory: "8Gi" - cpu: "4000m" - env: - - name: NODE_OPTIONS - value: "--max-old-space-size=6144" + replicas: 2 + template: + spec: + containers: + - name: brainy + image: my-brainy-app:latest + resources: + requests: + memory: "6Gi" + cpu: "2000m" + limits: + memory: "8Gi" + cpu: "4000m" + env: + - name: NODE_OPTIONS + value: "--max-old-space-size=6144" ``` **Best Practices:** @@ -399,15 +399,15 @@ The system automatically chooses the optimal caching strategy: **Auto-detection logic:** ```typescript -const vectorMemoryNeeded = entityCount × 1536 // bytes +const vectorMemoryNeeded = entityCount × 1536 // bytes const hnswCacheAvailable = unifiedCache.maxSize × 0.80 if (vectorMemoryNeeded < hnswCacheAvailable) { - // Preload strategy: all vectors loaded at init - console.log('Caching strategy: preloaded (all vectors in memory)') + // Preload strategy: all vectors loaded at init + console.log('Caching strategy: preloaded (all vectors in memory)') } else { - // On-demand strategy: vectors loaded adaptively - console.log('Caching strategy: on-demand (adaptive loading via UnifiedCache)') + // On-demand strategy: vectors loaded adaptively + console.log('Caching strategy: on-demand (adaptive loading via UnifiedCache)') } ``` @@ -422,12 +422,12 @@ Consider increasing RAM when: **Decision tree:** ``` If cache hit rate < 70%: - └─> Is working set < 50% of total entities? - ├─> YES: Increase cache size (add RAM) - └─> NO: Working set too large, consider: - ├─> Application-level caching - ├─> Query optimization - └─> Sharding dataset + └─> Is working set < 50% of total entities? + ├─> YES: Increase cache size (add RAM) + └─> NO: Working set too large, consider: + ├─> Application-level caching + ├─> Query optimization + └─> Sharding dataset ``` ### When to Shard/Distribute @@ -442,17 +442,17 @@ Consider sharding when: ```typescript // Example: Geographic sharding const usEastBrain = new Brainy({ - storage: { type: 's3', s3Storage: { bucket: 'us-east-data' } } + storage: { type: 's3', s3Storage: { bucket: 'us-east-data' } } }) const euWestBrain = new Brainy({ - storage: { type: 's3', s3Storage: { bucket: 'eu-west-data' } } + storage: { type: 's3', s3Storage: { bucket: 'eu-west-data' } } }) // Route queries based on user location async function search(query, userRegion) { - const brain = userRegion === 'US' ? usEastBrain : euWestBrain - return await brain.search(query) + const brain = userRegion === 'US' ? usEastBrain : euWestBrain + return await brain.search(query) } ``` @@ -482,7 +482,7 @@ console.log(`Pressure: ${memoryInfo.currentPressure.pressure}`) // Values: 'low', 'moderate', 'high', 'critical' if (memoryInfo.currentPressure.warnings.length > 0) { - console.warn('Memory warnings:', memoryInfo.currentPressure.warnings) + console.warn('Memory warnings:', memoryInfo.currentPressure.warnings) } ``` @@ -491,9 +491,9 @@ if (memoryInfo.currentPressure.warnings.length > 0) { const stats = brain.hnsw.getCacheStats() if (stats.fairness.fairnessViolation) { - console.warn('Cache fairness violation detected') - console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access`) - console.warn(`HNSW: ${stats.hnswCache.sizePercent}% of cache`) + console.warn('Cache fairness violation detected') + console.warn(`HNSW: ${stats.fairness.hnswAccessPercent}% access`) + console.warn(`HNSW: ${stats.hnswCache.sizePercent}% of cache`) } ``` @@ -502,7 +502,7 @@ if (stats.fairness.fairnessViolation) { // Track search latency console.time('search') const results = await brain.search('query') -console.timeEnd('search') // Target: <10ms for hot queries +console.timeEnd('search') // Target: <10ms for hot queries ``` ### Alerting Thresholds @@ -516,26 +516,26 @@ Set up alerts for: **Example monitoring script:** ```typescript async function monitorHealth() { - const stats = brain.hnsw.getCacheStats() + const stats = brain.hnsw.getCacheStats() - // Alert on low cache hit rate - if (stats.unifiedCache.hitRatePercent < 70) { - await sendAlert({ - severity: 'warning', - message: `Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`, - recommendations: stats.recommendations - }) - } + // Alert on low cache hit rate + if (stats.unifiedCache.hitRatePercent < 70) { + await sendAlert({ + severity: 'warning', + message: `Low cache hit rate: ${stats.unifiedCache.hitRatePercent}%`, + recommendations: stats.recommendations + }) + } - // Alert on memory pressure - const memoryInfo = brain.hnsw.unifiedCache.getMemoryInfo() - if (memoryInfo.currentPressure.pressure === 'high') { - await sendAlert({ - severity: 'critical', - message: 'High memory pressure detected', - warnings: memoryInfo.currentPressure.warnings - }) - } + // Alert on memory pressure + const memoryInfo = brain.hnsw.unifiedCache.getMemoryInfo() + if (memoryInfo.currentPressure.pressure === 'high') { + await sendAlert({ + severity: 'critical', + message: 'High memory pressure detected', + warnings: memoryInfo.currentPressure.warnings + }) + } } // Run every 60 seconds @@ -552,9 +552,9 @@ setInterval(monitorHealth, 60000) **Sizing:** ``` -Products: 500,000 -Vector memory needed: 500K × 1536 bytes = 768 MB -HNSW cache available: (16GB × 0.8 - 150MB) × 0.5 × 0.3 = 1,915 MB +Products: 500,000 +Vector memory needed: 500K × 1536 bytes = 768 MB +HNSW cache available: (16GB × 0.8 - 150MB) × 0.5 × 0.3 = 1,915 MB Result: Standard mode (all vectors fit in HNSW cache) ``` @@ -562,16 +562,16 @@ Result: Standard mode (all vectors fit in HNSW cache) **Configuration:** ```typescript const brain = new Brainy({ - storage: { type: 'filesystem', path: '/var/lib/brainy' }, - model: { precision: 'q8' } + storage: { type: 'filesystem', path: '/var/lib/brainy' }, + model: { precision: 'q8' } }) await brain.init() // Verify preloaded strategy (all vectors in memory) const stats = brain.hnsw.getCacheStats() -console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'preloaded' -console.log(`Search latency: ${stats.performance.avgSearchMs}ms`) // ~3ms +console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'preloaded' +console.log(`Search latency: ${stats.performance.avgSearchMs}ms`) // ~3ms ``` ### Example 2: Document Search (5M documents) @@ -580,9 +580,9 @@ console.log(`Search latency: ${stats.performance.avgSearchMs}ms`) // ~3ms **Sizing:** ``` -Documents: 5,000,000 -Vector memory needed: 5M × 1536 bytes = 7,680 MB -HNSW cache available: (32GB × 0.8 - 150MB) × 0.5 × 0.3 = 3,910 MB +Documents: 5,000,000 +Vector memory needed: 5M × 1536 bytes = 7,680 MB +HNSW cache available: (32GB × 0.8 - 150MB) × 0.5 × 0.3 = 3,910 MB Result: On-demand caching (vectors loaded adaptively) ``` @@ -590,20 +590,20 @@ Result: On-demand caching (vectors loaded adaptively) **Configuration:** ```typescript const brain = new Brainy({ - storage: { - type: 'gcs-native', - gcsNativeStorage: { bucketName: 'docs-production' } - }, - model: { precision: 'q8' } + storage: { + type: 'gcs-native', + gcsNativeStorage: { bucketName: 'docs-production' } + }, + model: { precision: 'q8' } }) await brain.init() // Monitor cache performance const stats = brain.hnsw.getCacheStats() -console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand' -console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >80% -console.log(`Cold search latency: ${stats.performance.avgSearchMs}ms`) // ~12ms +console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand' +console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >80% +console.log(`Cold search latency: ${stats.performance.avgSearchMs}ms`) // ~12ms // Recommendations console.log('Recommendations:', stats.recommendations) @@ -616,9 +616,9 @@ console.log('Recommendations:', stats.recommendations) **Sizing:** ``` -Entities: 20,000,000 -Vector memory needed: 20M × 1536 bytes = 30,720 MB -HNSW cache available: ~15,691 MB (after logarithmic scaling) +Entities: 20,000,000 +Vector memory needed: 20M × 1536 bytes = 30,720 MB +HNSW cache available: ~15,691 MB (after logarithmic scaling) Result: On-demand caching with high-performance adaptive loading ``` @@ -626,14 +626,14 @@ Result: On-demand caching with high-performance adaptive loading **Configuration:** ```typescript const brain = new Brainy({ - storage: { - type: 's3', - s3Storage: { - bucketName: 'knowledge-graph-prod', - region: 'us-east-1' - } - }, - model: { precision: 'fp32' } // Maximum accuracy + storage: { + type: 's3', + s3Storage: { + bucketName: 'knowledge-graph-prod', + region: 'us-east-1' + } + }, + model: { precision: 'fp32' } // Maximum accuracy }) await brain.init() @@ -641,13 +641,13 @@ await brain.init() // Enterprise monitoring const stats = brain.hnsw.getCacheStats() console.log(`Entities: ${stats.autoDetection.entityCount.toLocaleString()}`) -console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand' -console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >85% +console.log(`Caching strategy: ${stats.cachingStrategy}`) // 'on-demand' +console.log(`Cache hit rate: ${stats.unifiedCache.hitRatePercent}%`) // Target >85% console.log(`HNSW cache: ${stats.hnswCache.estimatedMemoryMB}MB`) // Fairness check if (stats.fairness.fairnessViolation) { - console.warn('HNSW dominating cache - consider tuning eviction policies') + console.warn('HNSW dominating cache - consider tuning eviction policies') } ``` @@ -690,8 +690,8 @@ console.log(`Warnings:`, memoryInfo.currentPressure.warnings) ```typescript const stats = brain.hnsw.getCacheStats() if (stats.fairness.fairnessViolation) { - console.log(`HNSW access: ${stats.fairness.hnswAccessPercent}%`) - console.log(`HNSW cache: ${stats.hnswCache.sizePercent}%`) + console.log(`HNSW access: ${stats.fairness.hnswAccessPercent}%`) + console.log(`HNSW cache: ${stats.hnswCache.sizePercent}%`) } ``` @@ -704,8 +704,7 @@ if (stats.fairness.fairnessViolation) { ## 📚 Additional Resources -- **[Migration Guide](../guides/migration-3.36.0.md)** - Upgrading to v3.36.0 -- **[Architecture Overview](../architecture/data-storage-architecture.md)** - Deep dive into storage and caching +- **[Migration Guide](../guides/migration-3.36.0.md)** - Upgrading to **[Architecture Overview](../architecture/data-storage-architecture.md)** - Deep dive into storage and caching - **[GitHub Issues](https://github.com/soulcraftlabs/brainy/issues)** - Report problems or ask questions --- diff --git a/docs/operations/cost-optimization-aws-s3.md b/docs/operations/cost-optimization-aws-s3.md index 1fa412de..4bb379d2 100644 --- a/docs/operations/cost-optimization-aws-s3.md +++ b/docs/operations/cost-optimization-aws-s3.md @@ -1,10 +1,9 @@ -# AWS S3 Cost Optimization Guide for Brainy v4.0.0 - +# AWS S3 Cost Optimization Guide for Brainy > **Cost Impact**: Reduce storage costs from $138k/year to $5.9k/year at 500TB scale (**96% savings**) ## Overview -Brainy v4.0.0 provides enterprise-grade cost optimization features for AWS S3 storage, enabling automatic tier transitions that dramatically reduce storage costs while maintaining performance. +Brainy provides enterprise-grade cost optimization features for AWS S3 storage, enabling automatic tier transitions that dramatically reduce storage costs while maintaining performance. ## Cost Breakdown (Before Optimization) @@ -39,10 +38,10 @@ import { S3CompatibleStorage } from '@soulcraft/brainy/storage' // Initialize Brainy with S3 storage const storage = new S3CompatibleStorage({ - bucket: 'my-brainy-data', - region: 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + bucket: 'my-brainy-data', + region: 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY }) const brain = new Brainy({ storage }) @@ -50,24 +49,24 @@ await brain.init() // Set lifecycle policy for automatic archival await storage.setLifecyclePolicy({ - rules: [{ - id: 'optimize-vectors', - prefix: 'entities/nouns/vectors/', - status: 'Enabled', - transitions: [ - { days: 30, storageClass: 'STANDARD_IA' }, // Infrequent Access after 30 days - { days: 90, storageClass: 'GLACIER' }, // Glacier after 90 days - { days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 1 year - ] - }, { - id: 'optimize-metadata', - prefix: 'entities/nouns/metadata/', - status: 'Enabled', - transitions: [ - { days: 30, storageClass: 'STANDARD_IA' }, - { days: 180, storageClass: 'GLACIER' } - ] - }] + rules: [{ + id: 'optimize-vectors', + prefix: 'entities/nouns/vectors/', + status: 'Enabled', + transitions: [ + { days: 30, storageClass: 'STANDARD_IA' }, // Infrequent Access after 30 days + { days: 90, storageClass: 'GLACIER' }, // Glacier after 90 days + { days: 365, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 1 year + ] + }, { + id: 'optimize-metadata', + prefix: 'entities/nouns/metadata/', + status: 'Enabled', + transitions: [ + { days: 30, storageClass: 'STANDARD_IA' }, + { days: 180, storageClass: 'GLACIER' } + ] + }] }) // Verify lifecycle policy @@ -84,10 +83,10 @@ console.log('Lifecycle policy active:', policy.rules.length, 'rules') - 10% of data 365+ days old (Deep Archive) ``` -Standard (200TB): 200TB × $0.023/GB × 12 = $55,200/year -Standard-IA (150TB): 150TB × $0.0125/GB × 12 = $22,500/year -Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year -Deep Archive (50TB): 50TB × $0.00099/GB × 12 = $594/year +Standard (200TB): 200TB × $0.023/GB × 12 = $55,200/year +Standard-IA (150TB): 150TB × $0.0125/GB × 12 = $22,500/year +Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year +Deep Archive (50TB): 50TB × $0.00099/GB × 12 = $594/year Total Storage Cost: $83,094/year (instead of $138,000) Total with Operations: ~$88,000/year @@ -133,11 +132,11 @@ Intelligent-Tiering automatically moves objects between: - 30% Deep Archive Access ``` -Frequent (75TB): 75TB × $0.023/GB × 12 = $20,700/year -Infrequent (100TB): 100TB × $0.0125/GB × 12 = $15,000/year -Archive (175TB): 175TB × $0.004/GB × 12 = $8,400/year -Deep Archive (150TB): 150TB × $0.00099/GB × 12 = $1,782/year -Monitoring: ~$300/year (minimal) +Frequent (75TB): 75TB × $0.023/GB × 12 = $20,700/year +Infrequent (100TB): 100TB × $0.0125/GB × 12 = $15,000/year +Archive (175TB): 175TB × $0.004/GB × 12 = $8,400/year +Deep Archive (150TB): 150TB × $0.00099/GB × 12 = $1,782/year +Monitoring: ~$300/year (minimal) Total Storage Cost: $46,182/year Total with Operations: ~$51,000/year @@ -157,21 +156,21 @@ await storage.enableIntelligentTiering('entities/verbs/vectors/', 'verbs-auto') // Set lifecycle policy for metadata (less frequently accessed) await storage.setLifecyclePolicy({ - rules: [{ - id: 'archive-old-metadata', - prefix: 'entities/nouns/metadata/', - status: 'Enabled', - transitions: [ - { days: 30, storageClass: 'STANDARD_IA' }, - { days: 60, storageClass: 'GLACIER' }, - { days: 180, storageClass: 'DEEP_ARCHIVE' } - ] - }, { - id: 'cleanup-old-system-data', - prefix: '_system/', - status: 'Enabled', - expiration: { days: 365 } // Delete old statistics - }] + rules: [{ + id: 'archive-old-metadata', + prefix: 'entities/nouns/metadata/', + status: 'Enabled', + transitions: [ + { days: 30, storageClass: 'STANDARD_IA' }, + { days: 60, storageClass: 'GLACIER' }, + { days: 180, storageClass: 'DEEP_ARCHIVE' } + ] + }, { + id: 'cleanup-old-system-data', + prefix: '_system/', + status: 'Enabled', + expiration: { days: 365 } // Delete old statistics + }] }) ``` @@ -179,19 +178,19 @@ await storage.setLifecyclePolicy({ **Vectors (300TB with Intelligent-Tiering):** ``` -Frequent (45TB): 45TB × $0.023/GB × 12 = $12,420/year -Infrequent (60TB): 60TB × $0.0125/GB × 12 = $9,000/year -Archive (105TB): 105TB × $0.004/GB × 12 = $5,040/year -Deep Archive (90TB): 90TB × $0.00099/GB × 12 = $1,069/year +Frequent (45TB): 45TB × $0.023/GB × 12 = $12,420/year +Infrequent (60TB): 60TB × $0.0125/GB × 12 = $9,000/year +Archive (105TB): 105TB × $0.004/GB × 12 = $5,040/year +Deep Archive (90TB): 90TB × $0.00099/GB × 12 = $1,069/year Subtotal: $27,529/year ``` **Metadata (200TB with Lifecycle Policy):** ``` -Standard (60TB): 60TB × $0.023/GB × 12 = $16,560/year -Standard-IA (40TB): 40TB × $0.0125/GB × 12 = $6,000/year -Glacier (60TB): 60TB × $0.004/GB × 12 = $2,880/year -Deep Archive (40TB): 40TB × $0.00099/GB × 12 = $475/year +Standard (60TB): 60TB × $0.023/GB × 12 = $16,560/year +Standard-IA (40TB): 40TB × $0.0125/GB × 12 = $6,000/year +Glacier (60TB): 60TB × $0.004/GB × 12 = $2,880/year +Deep Archive (40TB): 40TB × $0.00099/GB × 12 = $475/year Subtotal: $25,915/year ``` @@ -206,16 +205,16 @@ Subtotal: $25,915/year ```typescript await storage.setLifecyclePolicy({ - rules: [{ - id: 'aggressive-archival', - prefix: 'entities/', - status: 'Enabled', - transitions: [ - { days: 14, storageClass: 'STANDARD_IA' }, // IA after 2 weeks - { days: 30, storageClass: 'GLACIER' }, // Glacier after 1 month - { days: 90, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 3 months - ] - }] + rules: [{ + id: 'aggressive-archival', + prefix: 'entities/', + status: 'Enabled', + transitions: [ + { days: 14, storageClass: 'STANDARD_IA' }, // IA after 2 weeks + { days: 30, storageClass: 'GLACIER' }, // Glacier after 1 month + { days: 90, storageClass: 'DEEP_ARCHIVE' } // Deep Archive after 3 months + ] + }] }) ``` @@ -223,10 +222,10 @@ await storage.setLifecyclePolicy({ **After 1 year:** ``` -Standard (50TB): 50TB × $0.023/GB × 12 = $13,800/year -Standard-IA (50TB): 50TB × $0.0125/GB × 12 = $7,500/year -Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year -Deep Archive (300TB): 300TB × $0.00099/GB × 12 = $3,564/year +Standard (50TB): 50TB × $0.023/GB × 12 = $13,800/year +Standard-IA (50TB): 50TB × $0.0125/GB × 12 = $7,500/year +Glacier (100TB): 100TB × $0.004/GB × 12 = $4,800/year +Deep Archive (300TB): 300TB × $0.00099/GB × 12 = $3,564/year Total Storage Cost: $29,664/year Total with Operations: ~$34,000/year @@ -250,16 +249,16 @@ Note: Retrieval costs may be significant if archived data is accessed frequently ### Efficient Cleanup ```typescript -// v4.0.0: Batch delete (1000 objects per request) +// Batch delete (1000 objects per request) const idsToDelete = [/* array of entity IDs */] // Generate paths for both vector and metadata files const paths = idsToDelete.flatMap(id => { - const shard = id.substring(0, 2) - return [ - `entities/nouns/vectors/${shard}/${id}.json`, - `entities/nouns/metadata/${shard}/${id}.json` - ] + const shard = id.substring(0, 2) + return [ + `entities/nouns/vectors/${shard}/${id}.json`, + `entities/nouns/metadata/${shard}/${id}.json` + ] }) // Batch delete (much faster and cheaper than individual deletes) @@ -280,14 +279,14 @@ console.log('Active rules:', policy.rules) // Example output: // { -// rules: [ -// { -// id: 'optimize-vectors', -// prefix: 'entities/nouns/vectors/', -// status: 'Enabled', -// transitions: [...] -// } -// ] +// rules: [ +// { +// id: 'optimize-vectors', +// prefix: 'entities/nouns/vectors/', +// status: 'Enabled', +// transitions: [...] +// } +// ] // } ``` @@ -396,6 +395,5 @@ await storage.enableIntelligentTiering('entities/', 'new-config') --- -**Version**: v4.0.0 **Last Updated**: 2025-10-17 **Cloud Provider**: AWS S3 diff --git a/docs/operations/cost-optimization-azure.md b/docs/operations/cost-optimization-azure.md index 601ee04a..bae4370d 100644 --- a/docs/operations/cost-optimization-azure.md +++ b/docs/operations/cost-optimization-azure.md @@ -1,10 +1,9 @@ -# Azure Blob Storage Cost Optimization Guide for Brainy v4.0.0 - +# Azure Blob Storage Cost Optimization Guide for Brainy > **Cost Impact**: Reduce storage costs from $107k/year to $5k/year at 500TB scale (**95% savings**) ## Overview -Brainy v4.0.0 provides enterprise-grade cost optimization features for Azure Blob Storage, including manual tier management, lifecycle policies, and batch operations. +Brainy provides enterprise-grade cost optimization features for Azure Blob Storage, including manual tier management, lifecycle policies, and batch operations. ## Cost Breakdown (Before Optimization) @@ -41,8 +40,8 @@ import { AzureBlobStorage } from '@soulcraft/brainy/storage' // Initialize Brainy with Azure storage const storage = new AzureBlobStorage({ - connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, - containerName: 'brainy-data' + connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, + containerName: 'brainy-data' }) const brain = new Brainy({ storage }) @@ -50,19 +49,19 @@ await brain.init() // Change tier for a single blob await storage.changeBlobTier( - 'entities/nouns/vectors/00/00123456-uuid.json', - 'Cool' + 'entities/nouns/vectors/00/00123456-uuid.json', + 'Cool' ) // Batch tier changes (efficient for thousands of blobs) const blobsToMove = [ - 'entities/nouns/vectors/01/...', - 'entities/nouns/vectors/02/...', - // ... up to thousands of blobs + 'entities/nouns/vectors/01/...', + 'entities/nouns/vectors/02/...', + // ... up to thousands of blobs ] -await storage.batchChangeTier(blobsToMove, 'Cool') // Hot → Cool -await storage.batchChangeTier(oldBlobs, 'Archive') // Cool → Archive +await storage.batchChangeTier(blobsToMove, 'Cool') // Hot → Cool +await storage.batchChangeTier(oldBlobs, 'Archive') // Cool → Archive ``` ### Immediate Cost Impact @@ -90,54 +89,54 @@ Savings: $20,346/year (95% savings on moved data) ```typescript // Set lifecycle policy for automatic tier management await storage.setLifecyclePolicy({ - rules: [{ - name: 'optimizeVectors', - enabled: true, - type: 'Lifecycle', - definition: { - filters: { - blobTypes: ['blockBlob'], - prefixMatch: ['entities/nouns/vectors/'] - }, - actions: { - baseBlob: { - tierToCool: { daysAfterModificationGreaterThan: 30 }, - tierToArchive: { daysAfterModificationGreaterThan: 90 } - } - } - } - }, { - name: 'optimizeMetadata', - enabled: true, - type: 'Lifecycle', - definition: { - filters: { - blobTypes: ['blockBlob'], - prefixMatch: ['entities/nouns/metadata/'] - }, - actions: { - baseBlob: { - tierToCool: { daysAfterModificationGreaterThan: 30 }, - tierToArchive: { daysAfterModificationGreaterThan: 180 } - } - } - } - }, { - name: 'cleanupOldSystemFiles', - enabled: true, - type: 'Lifecycle', - definition: { - filters: { - blobTypes: ['blockBlob'], - prefixMatch: ['_system/'] - }, - actions: { - baseBlob: { - delete: { daysAfterModificationGreaterThan: 365 } - } - } - } - }] + rules: [{ + name: 'optimizeVectors', + enabled: true, + type: 'Lifecycle', + definition: { + filters: { + blobTypes: ['blockBlob'], + prefixMatch: ['entities/nouns/vectors/'] + }, + actions: { + baseBlob: { + tierToCool: { daysAfterModificationGreaterThan: 30 }, + tierToArchive: { daysAfterModificationGreaterThan: 90 } + } + } + } + }, { + name: 'optimizeMetadata', + enabled: true, + type: 'Lifecycle', + definition: { + filters: { + blobTypes: ['blockBlob'], + prefixMatch: ['entities/nouns/metadata/'] + }, + actions: { + baseBlob: { + tierToCool: { daysAfterModificationGreaterThan: 30 }, + tierToArchive: { daysAfterModificationGreaterThan: 180 } + } + } + } + }, { + name: 'cleanupOldSystemFiles', + enabled: true, + type: 'Lifecycle', + definition: { + filters: { + blobTypes: ['blockBlob'], + prefixMatch: ['_system/'] + }, + actions: { + baseBlob: { + delete: { daysAfterModificationGreaterThan: 365 } + } + } + } + }] }) // Verify lifecycle policy @@ -153,9 +152,9 @@ console.log('Lifecycle policy active:', policy.rules.length, 'rules') - 30% of data in Archive tier (90+ days old) ``` -Hot (150TB): 150TB × $0.0184/GB × 12 = $32,256/year -Cool (200TB): 200TB × $0.0115/GB × 12 = $26,880/year -Archive (150TB): 150TB × $0.00099/GB × 12 = $1,732/year +Hot (150TB): 150TB × $0.0184/GB × 12 = $32,256/year +Cool (200TB): 200TB × $0.0115/GB × 12 = $26,880/year +Archive (150TB): 150TB × $0.00099/GB × 12 = $1,732/year Total Storage Cost: $60,868/year Total with Operations: ~$65,500/year @@ -170,23 +169,23 @@ Savings: $47,000/year (42%) ```typescript await storage.setLifecyclePolicy({ - rules: [{ - name: 'aggressiveArchival', - enabled: true, - type: 'Lifecycle', - definition: { - filters: { - blobTypes: ['blockBlob'], - prefixMatch: ['entities/'] - }, - actions: { - baseBlob: { - tierToCool: { daysAfterModificationGreaterThan: 14 }, - tierToArchive: { daysAfterModificationGreaterThan: 30 } - } - } - } - }] + rules: [{ + name: 'aggressiveArchival', + enabled: true, + type: 'Lifecycle', + definition: { + filters: { + blobTypes: ['blockBlob'], + prefixMatch: ['entities/'] + }, + actions: { + baseBlob: { + tierToCool: { daysAfterModificationGreaterThan: 14 }, + tierToArchive: { daysAfterModificationGreaterThan: 30 } + } + } + } + }] }) ``` @@ -194,9 +193,9 @@ await storage.setLifecyclePolicy({ **After 6 months:** ``` -Hot (50TB): 50TB × $0.0184/GB × 12 = $10,752/year -Cool (100TB): 100TB × $0.0115/GB × 12 = $13,440/year -Archive (350TB): 350TB × $0.00099/GB × 12 = $4,039/year +Hot (50TB): 50TB × $0.0184/GB × 12 = $10,752/year +Cool (100TB): 100TB × $0.0115/GB × 12 = $13,440/year +Archive (350TB): 350TB × $0.00099/GB × 12 = $4,039/year Total Storage Cost: $28,231/year Total with Operations: ~$33,000/year @@ -211,41 +210,41 @@ Warning: Archive rehydration takes 1-15 hours ```typescript await storage.setLifecyclePolicy({ - rules: [{ - // Vectors: Keep in Hot/Cool for search performance - name: 'vectors-moderate', - enabled: true, - type: 'Lifecycle', - definition: { - filters: { - blobTypes: ['blockBlob'], - prefixMatch: ['entities/nouns/vectors/', 'entities/verbs/vectors/'] - }, - actions: { - baseBlob: { - tierToCool: { daysAfterModificationGreaterThan: 60 } - // Don't archive vectors - keep searchable - } - } - } - }, { - // Metadata: Aggressive archival - name: 'metadata-aggressive', - enabled: true, - type: 'Lifecycle', - definition: { - filters: { - blobTypes: ['blockBlob'], - prefixMatch: ['entities/nouns/metadata/', 'entities/verbs/metadata/'] - }, - actions: { - baseBlob: { - tierToCool: { daysAfterModificationGreaterThan: 30 }, - tierToArchive: { daysAfterModificationGreaterThan: 90 } - } - } - } - }] + rules: [{ + // Vectors: Keep in Hot/Cool for search performance + name: 'vectors-moderate', + enabled: true, + type: 'Lifecycle', + definition: { + filters: { + blobTypes: ['blockBlob'], + prefixMatch: ['entities/nouns/vectors/', 'entities/verbs/vectors/'] + }, + actions: { + baseBlob: { + tierToCool: { daysAfterModificationGreaterThan: 60 } + // Don't archive vectors - keep searchable + } + } + } + }, { + // Metadata: Aggressive archival + name: 'metadata-aggressive', + enabled: true, + type: 'Lifecycle', + definition: { + filters: { + blobTypes: ['blockBlob'], + prefixMatch: ['entities/nouns/metadata/', 'entities/verbs/metadata/'] + }, + actions: { + baseBlob: { + tierToCool: { daysAfterModificationGreaterThan: 30 }, + tierToArchive: { daysAfterModificationGreaterThan: 90 } + } + } + } + }] }) ``` @@ -253,16 +252,16 @@ await storage.setLifecyclePolicy({ **Vectors (300TB):** ``` -Hot (90TB): 90TB × $0.0184/GB × 12 = $19,354/year -Cool (210TB): 210TB × $0.0115/GB × 12 = $28,980/year +Hot (90TB): 90TB × $0.0184/GB × 12 = $19,354/year +Cool (210TB): 210TB × $0.0115/GB × 12 = $28,980/year Subtotal: $48,334/year ``` **Metadata (200TB):** ``` -Hot (30TB): 30TB × $0.0184/GB × 12 = $6,451/year -Cool (70TB): 70TB × $0.0115/GB × 12 = $9,660/year -Archive (100TB): 100TB × $0.00099/GB × 12 = $1,158/year +Hot (30TB): 30TB × $0.0184/GB × 12 = $6,451/year +Cool (70TB): 70TB × $0.0115/GB × 12 = $9,660/year +Archive (100TB): 100TB × $0.00099/GB × 12 = $1,158/year Subtotal: $17,269/year ``` @@ -288,8 +287,8 @@ Subtotal: $17,269/year ```typescript // Rehydrate blob from Archive to Hot (high priority) await storage.rehydrateBlob( - 'entities/nouns/vectors/00/00123456-uuid.json', - 'High' // 'Standard' or 'High' priority + 'entities/nouns/vectors/00/00123456-uuid.json', + 'High' // 'Standard' or 'High' priority ) // Rehydration time: @@ -309,7 +308,7 @@ console.log('Archive status:', metadata.archiveStatus) const blobsToRehydrate = [/* array of blob paths */] for (const blobPath of blobsToRehydrate) { - await storage.rehydrateBlob(blobPath, 'High') + await storage.rehydrateBlob(blobPath, 'High') } // Wait for rehydration to complete (1-15 hours) @@ -333,15 +332,15 @@ Examples: ### Efficient Bulk Deletions ```typescript -// v4.0.0: Batch delete (256 blobs per request) +// Batch delete (256 blobs per request) const idsToDelete = [/* array of entity IDs */] const paths = idsToDelete.flatMap(id => { - const shard = id.substring(0, 2) - return [ - `entities/nouns/vectors/${shard}/${id}.json`, - `entities/nouns/metadata/${shard}/${id}.json` - ] + const shard = id.substring(0, 2) + return [ + `entities/nouns/vectors/${shard}/${id}.json`, + `entities/nouns/metadata/${shard}/${id}.json` + ] }) // Batch delete via BlobBatchClient @@ -375,14 +374,14 @@ console.log('Active rules:', policy.rules) // Example output: // { -// rules: [ -// { -// name: 'optimizeVectors', -// enabled: true, -// type: 'Lifecycle', -// definition: {...} -// } -// ] +// rules: [ +// { +// name: 'optimizeVectors', +// enabled: true, +// type: 'Lifecycle', +// definition: {...} +// } +// ] // } ``` @@ -452,8 +451,8 @@ Monthly cost trend: Decreasing 5-8% per month as data transitions // Check lifecycle policy status const policy = await storage.getLifecyclePolicy() console.log('Policy rules:', policy.rules.map(r => ({ - name: r.name, - enabled: r.enabled + name: r.name, + enabled: r.enabled }))) // Azure lifecycle policies run once per day @@ -471,9 +470,9 @@ await storage.rehydrateBlob(blobPath, 'High') // Wait for rehydration (check status) let status do { - await new Promise(resolve => setTimeout(resolve, 60000)) // Wait 1 minute - const metadata = await storage.getBlobMetadata(blobPath) - status = metadata.archiveStatus + await new Promise(resolve => setTimeout(resolve, 60000)) // Wait 1 minute + const metadata = await storage.getBlobMetadata(blobPath) + status = metadata.archiveStatus } while (status && status.includes('pending')) // Now access the blob @@ -494,8 +493,8 @@ const data = await storage.get(blobPath) ```typescript const storage = new AzureBlobStorage({ - connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, - containerName: 'brainy-data' + connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING, + containerName: 'brainy-data' }) ``` @@ -503,9 +502,9 @@ const storage = new AzureBlobStorage({ ```typescript const storage = new AzureBlobStorage({ - accountName: process.env.AZURE_STORAGE_ACCOUNT, - accountKey: process.env.AZURE_STORAGE_KEY, - containerName: 'brainy-data' + accountName: process.env.AZURE_STORAGE_ACCOUNT, + accountKey: process.env.AZURE_STORAGE_KEY, + containerName: 'brainy-data' }) ``` @@ -513,9 +512,9 @@ const storage = new AzureBlobStorage({ ```typescript const storage = new AzureBlobStorage({ - sasToken: process.env.AZURE_STORAGE_SAS_TOKEN, - accountName: process.env.AZURE_STORAGE_ACCOUNT, - containerName: 'brainy-data' + sasToken: process.env.AZURE_STORAGE_SAS_TOKEN, + accountName: process.env.AZURE_STORAGE_ACCOUNT, + containerName: 'brainy-data' }) ``` @@ -549,6 +548,5 @@ const storage = new AzureBlobStorage({ --- -**Version**: v4.0.0 **Last Updated**: 2025-10-17 **Cloud Provider**: Azure Blob Storage diff --git a/docs/operations/cost-optimization-cloudflare-r2.md b/docs/operations/cost-optimization-cloudflare-r2.md index b8718521..3bdc5490 100644 --- a/docs/operations/cost-optimization-cloudflare-r2.md +++ b/docs/operations/cost-optimization-cloudflare-r2.md @@ -1,10 +1,9 @@ -# Cloudflare R2 Cost Optimization Guide for Brainy v4.0.0 - +# Cloudflare R2 Cost Optimization Guide for Brainy > **Cost Impact**: $0 egress fees + 96% storage savings = Lowest cloud storage costs ## Overview -Cloudflare R2 is an S3-compatible object storage with **zero egress fees**, making it ideal for high-traffic applications. Brainy v4.0.0 fully supports R2 with lifecycle policies and batch operations. +Cloudflare R2 is an S3-compatible object storage with **zero egress fees**, making it ideal for high-traffic applications. Brainy fully supports R2 with lifecycle policies and batch operations. ## Cost Breakdown @@ -49,7 +48,7 @@ Savings vs AWS: $156,000/year (62%) - Cloudflare plans to add infrequent access tiers **When lifecycle features arrive:** -- Brainy v4.0.0 is already prepared with `setLifecyclePolicy()` support +- Brainy is already prepared with `setLifecyclePolicy()` support - Will work seamlessly once Cloudflare enables lifecycle management ## Strategy 1: Use R2 Standard (Current Best Practice) @@ -62,11 +61,11 @@ import { S3CompatibleStorage } from '@soulcraft/brainy/storage' // R2 uses S3-compatible API const storage = new S3CompatibleStorage({ - endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, - bucket: 'my-brainy-data', - region: 'auto', // R2 uses 'auto' region - accessKeyId: process.env.R2_ACCESS_KEY_ID, - secretAccessKey: process.env.R2_SECRET_ACCESS_KEY + endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, + bucket: 'my-brainy-data', + region: 'auto', // R2 uses 'auto' region + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY }) const brain = new Brainy({ storage }) @@ -96,23 +95,23 @@ Total: $90,081/year ```typescript // Cloudflare Worker (runs at edge) export default { - async fetch(request, env) { - // Initialize Brainy with R2 - const storage = new S3CompatibleStorage({ - endpoint: env.R2_ENDPOINT, - bucket: env.R2_BUCKET, - accessKeyId: env.R2_ACCESS_KEY, - secretAccessKey: env.R2_SECRET_KEY, - region: 'auto' - }) + async fetch(request, env) { + // Initialize Brainy with R2 + const storage = new S3CompatibleStorage({ + endpoint: env.R2_ENDPOINT, + bucket: env.R2_BUCKET, + accessKeyId: env.R2_ACCESS_KEY, + secretAccessKey: env.R2_SECRET_KEY, + region: 'auto' + }) - const brain = new Brainy({ storage }) - await brain.init() + const brain = new Brainy({ storage }) + await brain.init() - // Process at edge (no origin server needed) - const results = await brain.search(request.query) - return new Response(JSON.stringify(results)) - } + // Process at edge (no origin server needed) + const results = await brain.search(request.query) + return new Response(JSON.stringify(results)) + } } ``` @@ -121,18 +120,18 @@ export default { ``` R2 Storage (500TB): $90,000/year Workers (10M requests/day): - - First 100k requests/day: FREE - - Additional 350M requests/month: $1,750/year - - CPU time (50ms avg): $5,000/year + - First 100k requests/day: FREE + - Additional 350M requests/month: $1,750/year + - CPU time (50ms avg): $5,000/year Total: $96,750/year vs Traditional Setup (AWS S3 + EC2 + CloudFront): - - S3: $138,000/year - - EC2 (t3.xlarge × 4): $24,000/year - - CloudFront egress: $50,000/year - - Load balancer: $3,000/year - Total: $215,000/year + - S3: $138,000/year + - EC2 (t3.xlarge × 4): $24,000/year + - CloudFront egress: $50,000/year + - Load balancer: $3,000/year + Total: $215,000/year Savings: $118,250/year (55%) ``` @@ -144,20 +143,20 @@ Savings: $118,250/year (55%) ```typescript // Use R2 for frequently accessed data (zero egress) const hotStorage = new S3CompatibleStorage({ - endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, - bucket: 'brainy-hot', - region: 'auto', - accessKeyId: process.env.R2_ACCESS_KEY_ID, - secretAccessKey: process.env.R2_SECRET_ACCESS_KEY + endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, + bucket: 'brainy-hot', + region: 'auto', + accessKeyId: process.env.R2_ACCESS_KEY_ID, + secretAccessKey: process.env.R2_SECRET_ACCESS_KEY }) // Use AWS S3 with Intelligent-Tiering for cold data const coldStorage = new S3CompatibleStorage({ - endpoint: 's3.amazonaws.com', - bucket: 'brainy-archive', - region: 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + endpoint: 's3.amazonaws.com', + bucket: 'brainy-archive', + region: 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY }) // Initialize separate Brainy instances or implement tiering logic @@ -167,17 +166,17 @@ const coldStorage = new S3CompatibleStorage({ ``` R2 Hot Data (300TB): - Storage: 300TB × $0.015/GB × 12 = $54,000/year - Egress: $0 + Storage: 300TB × $0.015/GB × 12 = $54,000/year + Egress: $0 S3 Cold Data (200TB with lifecycle → Deep Archive): - Storage: 200TB × $0.00099/GB × 12 = $2,376/year - Ops: $1,000/year + Storage: 200TB × $0.00099/GB × 12 = $2,376/year + Ops: $1,000/year Total: $57,376/year vs All AWS S3: - - S3 storage + egress: $251,000/year + - S3 storage + egress: $251,000/year Savings: $193,624/year (77%) ``` @@ -187,15 +186,15 @@ Savings: $193,624/year (77%) ### Efficient Bulk Deletions ```typescript -// v4.0.0: R2 supports S3 batch delete API +// R2 supports S3 batch delete API const idsToDelete = [/* array of entity IDs */] const paths = idsToDelete.flatMap(id => { - const shard = id.substring(0, 2) - return [ - `entities/nouns/vectors/${shard}/${id}.json`, - `entities/nouns/metadata/${shard}/${id}.json` - ] + const shard = id.substring(0, 2) + return [ + `entities/nouns/vectors/${shard}/${id}.json`, + `entities/nouns/metadata/${shard}/${id}.json` + ] }) // Batch delete (1000 objects per request) @@ -231,10 +230,10 @@ https://storage.yourdomain.com/entities/nouns/vectors/... ```typescript // Worker triggered on R2 object upload export default { - async fetch(request, env) { - // Process new objects automatically - // E.g., index new entities, generate thumbnails, etc. - } + async fetch(request, env) { + // Process new objects automatically + // E.g., index new entities, generate thumbnails, etc. + } } // Cost: Only pay for Worker execution (no polling needed) @@ -244,7 +243,7 @@ export default { ```typescript // Generate presigned URL for direct browser uploads -const url = await storage.getPresignedUrl('upload-path', 3600) // 1 hour expiry +const url = await storage.getPresignedUrl('upload-path', 3600) // 1 hour expiry // Client uploads directly to R2 (no server bandwidth used) ``` @@ -255,7 +254,7 @@ const url = await storage.getPresignedUrl('upload-path', 3600) // 1 hour expiry ```typescript const status = await storage.getStorageStatus() -console.log('Storage type:', status.type) // 's3-compatible' +console.log('Storage type:', status.type) // 's3-compatible' console.log('Bucket:', status.details.bucket) console.log('Endpoint:', status.details.endpoint) ``` @@ -318,20 +317,20 @@ Egress: Unlimited (always free) ```bash # Install rclone -brew install rclone # or apt-get install rclone +brew install rclone # or apt-get install rclone # Configure S3 source rclone config create s3-source s3 \ - access_key_id=$AWS_ACCESS_KEY \ - secret_access_key=$AWS_SECRET_KEY \ - region=us-east-1 + access_key_id=$AWS_ACCESS_KEY \ + secret_access_key=$AWS_SECRET_KEY \ + region=us-east-1 # Configure R2 destination rclone config create r2-dest s3 \ - access_key_id=$R2_ACCESS_KEY \ - secret_access_key=$R2_SECRET_KEY \ - endpoint=https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com \ - region=auto + access_key_id=$R2_ACCESS_KEY \ + secret_access_key=$R2_SECRET_KEY \ + endpoint=https://$R2_ACCOUNT_ID.r2.cloudflarestorage.com \ + region=auto # Copy data rclone copy s3-source:my-bucket r2-dest:my-bucket --progress @@ -361,17 +360,17 @@ ROI: 3.4 months ### Prepared for Future Features ```typescript -// Brainy v4.0.0 is ready for R2 lifecycle features +// Brainy is ready for R2 lifecycle features await storage.setLifecyclePolicy({ - rules: [{ - id: 'archive-old-data', - prefix: 'entities/', - status: 'Enabled', - transitions: [ - { days: 30, storageClass: 'INFREQUENT_ACCESS' }, // When available - { days: 90, storageClass: 'ARCHIVE' } - ] - }] + rules: [{ + id: 'archive-old-data', + prefix: 'entities/', + status: 'Enabled', + transitions: [ + { days: 30, storageClass: 'INFREQUENT_ACCESS' }, // When available + { days: 90, storageClass: 'ARCHIVE' } + ] + }] }) // Expected cost impact (when lifecycle is available): @@ -388,11 +387,11 @@ await storage.setLifecyclePolicy({ ```typescript // Ensure correct endpoint format const storage = new S3CompatibleStorage({ - endpoint: `https://${accountId}.r2.cloudflarestorage.com`, - // NOT: `https://r2.cloudflarestorage.com/${accountId}` + endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + // NOT: `https://r2.cloudflarestorage.com/${accountId}` - region: 'auto', // R2 requires 'auto' - forcePathStyle: false // R2 uses virtual-hosted-style + region: 'auto', // R2 requires 'auto' + forcePathStyle: false // R2 uses virtual-hosted-style }) ``` @@ -446,7 +445,6 @@ const storage = new S3CompatibleStorage({ --- -**Version**: v4.0.0 **Last Updated**: 2025-10-17 **Cloud Provider**: Cloudflare R2 **Key Advantage**: **$0 egress fees forever** diff --git a/docs/operations/cost-optimization-gcs.md b/docs/operations/cost-optimization-gcs.md index f08597b6..59ab3ecf 100644 --- a/docs/operations/cost-optimization-gcs.md +++ b/docs/operations/cost-optimization-gcs.md @@ -1,10 +1,9 @@ -# Google Cloud Storage Cost Optimization Guide for Brainy v4.0.0 - +# Google Cloud Storage Cost Optimization Guide for Brainy > **Cost Impact**: Reduce storage costs from $138k/year to $8.3k/year at 500TB scale (**94% savings**) ## Overview -Brainy v4.0.0 provides enterprise-grade cost optimization features for Google Cloud Storage, including lifecycle policies and Autoclass for automatic tier management. +Brainy provides enterprise-grade cost optimization features for Google Cloud Storage, including lifecycle policies and Autoclass for automatic tier management. ## Cost Breakdown (Before Optimization) @@ -35,8 +34,8 @@ import { GcsStorage } from '@soulcraft/brainy/storage' // Initialize Brainy with GCS storage const storage = new GcsStorage({ - bucketName: 'my-brainy-data', - keyFilename: './service-account.json' // Or use ADC + bucketName: 'my-brainy-data', + keyFilename: './service-account.json' // Or use ADC }) const brain = new Brainy({ storage }) @@ -44,16 +43,16 @@ await brain.init() // Set lifecycle policy for automatic archival await storage.setLifecyclePolicy({ - rules: [{ - condition: { age: 30 }, - action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } - }, { - condition: { age: 90 }, - action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } - }, { - condition: { age: 365 }, - action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } - }] + rules: [{ + condition: { age: 30 }, + action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } + }, { + condition: { age: 90 }, + action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } + }, { + condition: { age: 365 }, + action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } + }] }) // Verify lifecycle policy @@ -70,10 +69,10 @@ console.log('Lifecycle policy active:', policy.rules.length, 'rules') - 10% of data 365+ days old (Archive) ``` -Standard (200TB): 200TB × $0.020/GB × 12 = $48,000/year -Nearline (150TB): 150TB × $0.010/GB × 12 = $18,000/year -Coldline (100TB): 100TB × $0.004/GB × 12 = $4,800/year -Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year +Standard (200TB): 200TB × $0.020/GB × 12 = $48,000/year +Nearline (150TB): 150TB × $0.010/GB × 12 = $18,000/year +Coldline (100TB): 100TB × $0.004/GB × 12 = $4,800/year +Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year Total Storage Cost: $71,520/year Total with Operations: ~$76,500/year @@ -89,7 +88,7 @@ Savings: $66,500/year (46%) ```typescript // Enable Autoclass for automatic tier management await storage.enableAutoclass({ - terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier + terminalStorageClass: 'ARCHIVE' // Optional: Set lowest tier }) // Benefits: @@ -116,10 +115,10 @@ await storage.enableAutoclass({ - 40% Archive (cold data) ``` -Standard (50TB): 50TB × $0.020/GB × 12 = $12,000/year -Nearline (75TB): 75TB × $0.010/GB × 12 = $9,000/year -Coldline (175TB): 175TB × $0.004/GB × 12 = $8,400/year -Archive (200TB): 200TB × $0.0012/GB × 12 = $2,880/year +Standard (50TB): 50TB × $0.020/GB × 12 = $12,000/year +Nearline (75TB): 75TB × $0.010/GB × 12 = $9,000/year +Coldline (175TB): 175TB × $0.004/GB × 12 = $8,400/year +Archive (200TB): 200TB × $0.0012/GB × 12 = $2,880/year Total Storage Cost: $32,280/year Total with Operations: ~$37,000/year @@ -133,24 +132,24 @@ Savings vs Standard: $106,000/year (74%) ```typescript // Enable Autoclass for vectors (frequently searched) await storage.enableAutoclass({ - terminalStorageClass: 'COLDLINE' // Don't archive vectors deeply + terminalStorageClass: 'COLDLINE' // Don't archive vectors deeply }) // Set lifecycle policy for metadata (less frequently accessed) await storage.setLifecyclePolicy({ - rules: [{ - condition: { age: 30, matchesPrefix: ['entities/nouns/metadata/'] }, - action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } - }, { - condition: { age: 60, matchesPrefix: ['entities/nouns/metadata/'] }, - action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } - }, { - condition: { age: 180, matchesPrefix: ['entities/nouns/metadata/'] }, - action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } - }, { - condition: { age: 730, matchesPrefix: ['_system/'] }, - action: { type: 'Delete' } // Delete old system files after 2 years - }] + rules: [{ + condition: { age: 30, matchesPrefix: ['entities/nouns/metadata/'] }, + action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } + }, { + condition: { age: 60, matchesPrefix: ['entities/nouns/metadata/'] }, + action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } + }, { + condition: { age: 180, matchesPrefix: ['entities/nouns/metadata/'] }, + action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } + }, { + condition: { age: 730, matchesPrefix: ['_system/'] }, + action: { type: 'Delete' } // Delete old system files after 2 years + }] }) ``` @@ -158,18 +157,18 @@ await storage.setLifecyclePolicy({ **Vectors (300TB with Autoclass):** ``` -Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year -Nearline (45TB): 45TB × $0.010/GB × 12 = $5,400/year -Coldline (225TB): 225TB × $0.004/GB × 12 = $10,800/year +Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year +Nearline (45TB): 45TB × $0.010/GB × 12 = $5,400/year +Coldline (225TB): 225TB × $0.004/GB × 12 = $10,800/year Subtotal: $23,400/year ``` **Metadata (200TB with Lifecycle Policy):** ``` -Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year -Nearline (40TB): 40TB × $0.010/GB × 12 = $4,800/year -Coldline (80TB): 80TB × $0.004/GB × 12 = $3,840/year -Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year +Standard (30TB): 30TB × $0.020/GB × 12 = $7,200/year +Nearline (40TB): 40TB × $0.010/GB × 12 = $4,800/year +Coldline (80TB): 80TB × $0.004/GB × 12 = $3,840/year +Archive (50TB): 50TB × $0.0012/GB × 12 = $720/year Subtotal: $16,560/year ``` @@ -182,16 +181,16 @@ Subtotal: $16,560/year ```typescript await storage.setLifecyclePolicy({ - rules: [{ - condition: { age: 14 }, - action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } - }, { - condition: { age: 30 }, - action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } - }, { - condition: { age: 90 }, - action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } - }] + rules: [{ + condition: { age: 14 }, + action: { type: 'SetStorageClass', storageClass: 'NEARLINE' } + }, { + condition: { age: 30 }, + action: { type: 'SetStorageClass', storageClass: 'COLDLINE' } + }, { + condition: { age: 90 }, + action: { type: 'SetStorageClass', storageClass: 'ARCHIVE' } + }] }) // Note: Archive class has 365-day minimum storage duration @@ -202,10 +201,10 @@ await storage.setLifecyclePolicy({ **After 1 year:** ``` -Standard (25TB): 25TB × $0.020/GB × 12 = $6,000/year -Nearline (50TB): 50TB × $0.010/GB × 12 = $6,000/year -Coldline (75TB): 75TB × $0.004/GB × 12 = $3,600/year -Archive (350TB): 350TB × $0.0012/GB × 12 = $5,040/year +Standard (25TB): 25TB × $0.020/GB × 12 = $6,000/year +Nearline (50TB): 50TB × $0.010/GB × 12 = $6,000/year +Coldline (75TB): 75TB × $0.004/GB × 12 = $3,600/year +Archive (350TB): 350TB × $0.0012/GB × 12 = $5,040/year Total Storage Cost: $20,640/year Total with Operations: ~$25,500/year @@ -241,15 +240,15 @@ Warning: High retrieval costs if archived data is accessed frequently ### Efficient Cleanup ```typescript -// v4.0.0: Batch delete (100 objects per request for GCS) +// Batch delete (100 objects per request for GCS) const idsToDelete = [/* array of entity IDs */] const paths = idsToDelete.flatMap(id => { - const shard = id.substring(0, 2) - return [ - `entities/nouns/vectors/${shard}/${id}.json`, - `entities/nouns/metadata/${shard}/${id}.json` - ] + const shard = id.substring(0, 2) + return [ + `entities/nouns/vectors/${shard}/${id}.json`, + `entities/nouns/metadata/${shard}/${id}.json` + ] }) // Batch delete @@ -271,9 +270,9 @@ console.log('Terminal class:', status.terminalStorageClass) // Example output: // { -// enabled: true, -// terminalStorageClass: 'ARCHIVE', -// toggleTime: '2025-01-15T10:30:00Z' +// enabled: true, +// terminalStorageClass: 'ARCHIVE', +// toggleTime: '2025-01-15T10:30:00Z' // } ``` @@ -336,7 +335,7 @@ Monthly cost trend: Decreasing 8-12% per month as data ages into cheaper classes // Check Autoclass status const status = await storage.getAutoclassStatus() if (!status.enabled) { - await storage.enableAutoclass({ terminalStorageClass: 'ARCHIVE' }) + await storage.enableAutoclass({ terminalStorageClass: 'ARCHIVE' }) } // Autoclass requires 24-48 hours for initial transitions @@ -365,8 +364,8 @@ if (!status.enabled) { ```typescript // Use ADC instead of service account key file const storage = new GcsStorage({ - bucketName: 'my-brainy-data' - // No keyFilename needed - uses ADC automatically + bucketName: 'my-brainy-data' + // No keyFilename needed - uses ADC automatically }) // ADC authentication order: @@ -417,6 +416,5 @@ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" --- -**Version**: v4.0.0 **Last Updated**: 2025-10-17 **Cloud Provider**: Google Cloud Storage diff --git a/docs/transactions.md b/docs/transactions.md index f406e154..34a98285 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -1,6 +1,6 @@ # Transaction System -**Status:** ✅ Production Ready (v5.8.0+) +**Status:** ✅ Production Ready ## Overview @@ -17,11 +17,11 @@ Brainy's transaction system provides **atomic operations** with automatic rollba ``` User Code (brain.add(), brain.update(), etc.) - ↓ + ↓ Transaction Manager (orchestration) - ↓ + ↓ Operations (SaveNounMetadataOperation, SaveNounOperation, etc.) - ↓ + ↓ Storage Adapter (COW, sharding, type-aware routing) ``` @@ -32,8 +32,8 @@ Every write operation in Brainy automatically uses transactions: ```typescript // Internally, this uses a transaction const id = await brain.add({ - data: { name: 'Alice', role: 'Engineer' }, - type: NounType.Person + data: { name: 'Alice', role: 'Engineer' }, + type: NounType.Person }) ``` @@ -51,19 +51,19 @@ Each operation implements both **execute** and **undo**: ```typescript class SaveNounMetadataOperation { - async execute(): Promise { - // Save new metadata - await this.storage.saveNounMetadata(this.id, this.metadata) - } + async execute(): Promise { + // Save new metadata + await this.storage.saveNounMetadata(this.id, this.metadata) + } - async undo(): Promise { - // Restore previous metadata (or delete if new entity) - if (this.previousMetadata) { - await this.storage.saveNounMetadata(this.id, this.previousMetadata) - } else { - await this.storage.deleteNounMetadata(this.id) - } - } + async undo(): Promise { + // Restore previous metadata (or delete if new entity) + if (this.previousMetadata) { + await this.storage.saveNounMetadata(this.id, this.previousMetadata) + } else { + await this.storage.deleteNounMetadata(this.id) + } + } } ``` @@ -87,8 +87,8 @@ await brain.cow.checkout('feature-branch') // Add entity (uses transaction on this branch) const id = await brain.add({ - data: { name: 'Feature Entity' }, - type: NounType.Thing + data: { name: 'Feature Entity' }, + type: NounType.Thing }) // On rollback: Branch remains clean, no partial commits @@ -123,7 +123,7 @@ await brain.relate({ from: id1, to: id2, type: VerbType.RelatesTo }) - Transaction operations don't need to know about shards - Rollback works across all shards involved -### ID-First Storage (v6.0.0+) +### ID-First Storage ✅ **Fully Compatible** @@ -132,27 +132,27 @@ Transactions work with direct ID-first paths - no type routing needed! ```typescript // Entities stored with direct ID-first paths const personId = await brain.add({ - data: { name: 'John Doe' }, - type: NounType.Person // → entities/nouns/{shard}/{id}/metadata.json + data: { name: 'John Doe' }, + type: NounType.Person // → entities/nouns/{shard}/{id}/metadata.json }) const orgId = await brain.add({ - data: { name: 'Acme Corp' }, - type: NounType.Organization // → entities/nouns/{shard}/{id}/metadata.json + data: { name: 'Acme Corp' }, + type: NounType.Organization // → entities/nouns/{shard}/{id}/metadata.json }) // Type changes handled atomically (type is just metadata) await brain.update({ - id: personId, - type: NounType.Organization, // Type change - data: { name: 'Doe Corp' } + id: personId, + type: NounType.Organization, // Type change + data: { name: 'Doe Corp' } }) ``` **How It Works:** - Type information stored in metadata.noun field - Storage layer uses O(1) ID-first path construction -- No type cache needed (removed in v6.0.0) +- No type cache needed (removed in a previous version) - Type counters adjusted on commit/rollback - 40x faster on cloud storage (eliminates 42-type search) @@ -165,10 +165,10 @@ Transactions work with distributed/remote storage: ```typescript // Works with S3, Azure, GCS, etc. const brain = new Brainy({ - storage: { - type: 's3Compatible', - config: { /* S3 config */ } - } + storage: { + type: 's3Compatible', + config: { /* S3 config */ } + } }) // Transactions ensure atomicity at write coordinator level @@ -199,8 +199,8 @@ await brain.init() // Automatically uses transaction const id = await brain.add({ - data: { name: 'Alice', role: 'Engineer' }, - type: NounType.Person + data: { name: 'Alice', role: 'Engineer' }, + type: NounType.Person }) // If add fails, all changes rolled back automatically @@ -211,15 +211,15 @@ const id = await brain.add({ ```typescript // Original entity const id = await brain.add({ - data: { name: 'John Smith', category: 'individual' }, - type: NounType.Person + data: { name: 'John Smith', category: 'individual' }, + type: NounType.Person }) // Update with type change (atomic) await brain.update({ - id, - type: NounType.Organization, // Type change - data: { name: 'Smith Corp', category: 'business' } + id, + type: NounType.Organization, // Type change + data: { name: 'Smith Corp', category: 'business' } }) // If update fails, original type and data restored @@ -229,20 +229,20 @@ await brain.update({ ```typescript const personId = await brain.add({ - data: { name: 'Alice' }, - type: NounType.Person + data: { name: 'Alice' }, + type: NounType.Person }) const projectId = await brain.add({ - data: { name: 'Project X' }, - type: NounType.Thing + data: { name: 'Project X' }, + type: NounType.Thing }) // Create relationship (atomic) await brain.relate({ - from: personId, - to: projectId, - type: VerbType.WorksOn + from: personId, + to: projectId, + type: VerbType.WorksOn }) // If relate fails, no partial relationship created @@ -253,10 +253,10 @@ await brain.relate({ ```typescript // Multiple operations, all atomic for (let i = 0; i < 100; i++) { - await brain.add({ - data: { name: `Entity ${i}`, index: i }, - type: NounType.Thing - }) + await brain.add({ + data: { name: `Entity ${i}`, index: i }, + type: NounType.Thing + }) } // Each add() is a separate transaction @@ -267,19 +267,19 @@ for (let i = 0; i < 100; i++) { ```typescript const personId = await brain.add({ - data: { name: 'Bob' }, - type: NounType.Person + data: { name: 'Bob' }, + type: NounType.Person }) const projectId = await brain.add({ - data: { name: 'Project Y' }, - type: NounType.Thing + data: { name: 'Project Y' }, + type: NounType.Thing }) await brain.relate({ - from: personId, - to: projectId, - type: VerbType.WorksOn + from: personId, + to: projectId, + type: VerbType.WorksOn }) // Delete person (atomic - deletes entity + relationships) @@ -294,15 +294,15 @@ Transactions automatically handle errors and rollback: ```typescript try { - await brain.add({ - data: { name: 'Test Entity' }, - type: NounType.Thing, - vector: [1, 2, 3] // Wrong dimension → error - }) + await brain.add({ + data: { name: 'Test Entity' }, + type: NounType.Thing, + vector: [1, 2, 3] // Wrong dimension → error + }) } catch (error) { - // Transaction automatically rolled back - // No partial data in storage or indexes - console.error('Add failed:', error.message) + // Transaction automatically rolled back + // No partial data in storage or indexes + console.error('Add failed:', error.message) } ``` @@ -333,11 +333,11 @@ try { const stats = brain.transactionManager?.getStats() console.log(stats) // { -// totalTransactions: 1234, -// successfulTransactions: 1200, -// failedTransactions: 34, -// rollbacks: 34, -// averageOperationsPerTransaction: 4.2 +// totalTransactions: 1234, +// successfulTransactions: 1200, +// failedTransactions: 34, +// rollbacks: 34, +// averageOperationsPerTransaction: 4.2 // } ``` @@ -359,7 +359,7 @@ await brain.update({ id, data }) await brain.delete(id) // ❌ Avoid: Direct storage access bypasses transactions -await brain.storage.saveNoun(noun) // No transaction protection +await brain.storage.saveNoun(noun) // No transaction protection ``` ### 2. Handle Errors Gracefully @@ -367,11 +367,11 @@ await brain.storage.saveNoun(noun) // No transaction protection ```typescript // ✅ Recommended: Catch errors, transaction rolls back automatically try { - const id = await brain.add({ data, type }) - return id + const id = await brain.add({ data, type }) + return id } catch (error) { - console.error('Add failed, rolled back:', error) - // Decide how to handle (retry, log, alert user) + console.error('Add failed, rolled back:', error) + // Decide how to handle (retry, log, alert user) } ``` @@ -380,7 +380,7 @@ try { ```typescript // ✅ Recommended: Validate early to avoid unnecessary rollbacks if (!isValidVector(vector, brain.dimension)) { - throw new Error(`Vector must have ${brain.dimension} dimensions`) + throw new Error(`Vector must have ${brain.dimension} dimensions`) } await brain.add({ data, type, vector }) @@ -391,14 +391,14 @@ await brain.add({ data, type, vector }) ```typescript // ✅ Recommended: Monitor in production setInterval(() => { - const stats = brain.transactionManager?.getStats() - if (stats) { - const failureRate = stats.failedTransactions / stats.totalTransactions - if (failureRate > 0.05) { // > 5% failure rate - console.warn('High transaction failure rate:', failureRate) - } - } -}, 60000) // Check every minute + const stats = brain.transactionManager?.getStats() + if (stats) { + const failureRate = stats.failedTransactions / stats.totalTransactions + if (failureRate > 0.05) { // > 5% failure rate + console.warn('High transaction failure rate:', failureRate) + } + } +}, 60000) // Check every minute ``` ### 5. Understand Atomicity Guarantees @@ -425,25 +425,25 @@ import { describe, it, expect } from 'vitest' import { Brainy } from '@soulcraft/brainy' describe('Transaction Tests', () => { - it('should rollback on failure', async () => { - const brain = new Brainy() - await brain.init() + it('should rollback on failure', async () => { + const brain = new Brainy() + await brain.init() - const id1 = await brain.add({ data: { name: 'Entity 1' }, type: NounType.Thing }) + const id1 = await brain.add({ data: { name: 'Entity 1' }, type: NounType.Thing }) - try { - await brain.add({ - data: null as any, // Invalid - will fail - type: NounType.Thing - }) - } catch (e) { - // Expected failure - } + try { + await brain.add({ + data: null as any, // Invalid - will fail + type: NounType.Thing + }) + } catch (e) { + // Expected failure + } - // First entity should still exist (rollback didn't affect it) - const entity1 = await brain.get(id1) - expect(entity1).toBeTruthy() - }) + // First entity should still exist (rollback didn't affect it) + const entity1 = await brain.get(id1) + expect(entity1).toBeTruthy() + }) }) ``` @@ -510,21 +510,21 @@ const stats = brain.transactionManager?.getStats() ``` 1. BEGIN - ↓ + ↓ 2. ADD OPERATIONS - - SaveNounMetadataOperation - - SaveNounOperation - - UpdateGraphIndexOperation - ↓ + - SaveNounMetadataOperation + - SaveNounOperation + - UpdateGraphIndexOperation + ↓ 3. EXECUTE (sequential) - - Execute operation 1 → Success - - Execute operation 2 → Success - - Execute operation 3 → FAILURE - ↓ + - Execute operation 1 → Success + - Execute operation 2 → Success + - Execute operation 3 → FAILURE + ↓ 4. ROLLBACK (reverse order) - - Undo operation 2 - - Undo operation 1 - ↓ + - Undo operation 2 + - Undo operation 1 + ↓ 5. THROW ERROR ``` @@ -544,11 +544,11 @@ Transactions use the `StorageAdapter` interface: ```typescript interface StorageAdapter { - saveNounMetadata(id: string, metadata: NounMetadata): Promise - saveNoun(noun: Noun): Promise - deleteNounMetadata(id: string): Promise - deleteNoun(id: string): Promise - // ... other methods + saveNounMetadata(id: string, metadata: NounMetadata): Promise + saveNoun(noun: Noun): Promise + deleteNounMetadata(id: string): Promise + deleteNoun(id: string): Promise + // ... other methods } ``` @@ -563,11 +563,11 @@ interface StorageAdapter { ## Version History -- **v5.8.0**: Initial transaction system release - - Atomic operations with rollback - - Compatible with COW, sharding, type-aware, distributed - - 36/36 unit tests passing - - 35 integration test scenarios +- Initial transaction system release + - Atomic operations with rollback + - Compatible with COW, sharding, type-aware, distributed + - 36/36 unit tests passing + - 35 integration test scenarios ## Summary diff --git a/docs/vfs/QUICK_START.md b/docs/vfs/QUICK_START.md index 4b68e9c7..8b0efce6 100644 --- a/docs/vfs/QUICK_START.md +++ b/docs/vfs/QUICK_START.md @@ -22,13 +22,13 @@ import { Brainy } from '@soulcraft/brainy' // ✅ CORRECT: Use filesystem storage for production const brain = new Brainy({ - storage: { - type: 'filesystem', - path: './brainy-data' // Your data directory - } + storage: { + type: 'filesystem', + path: './brainy-data' // Your data directory + } }) -await brain.init() // VFS auto-initialized! +await brain.init() // VFS auto-initialized! console.log('🎉 VFS ready!') ``` @@ -47,40 +47,40 @@ const badItems = allNodes.filter(node => node.path.startsWith(dirPath)) ```typescript // ✅ Method 1: Get direct children (recommended for UI) async function loadDirectoryContents(brain: Brainy, path: string) { - try { - const children = await brain.vfs.getDirectChildren(path) + try { + const children = await brain.vfs.getDirectChildren(path) - // Sort directories first, then files - return children.sort((a, b) => { - if (a.metadata.vfsType === 'directory' && b.metadata.vfsType === 'file') return -1 - if (a.metadata.vfsType === 'file' && b.metadata.vfsType === 'directory') return 1 - return a.metadata.name.localeCompare(b.metadata.name) - }) - } catch (error) { - console.error(`Failed to load ${path}:`, error.message) - return [] - } + // Sort directories first, then files + return children.sort((a, b) => { + if (a.metadata.vfsType === 'directory' && b.metadata.vfsType === 'file') return -1 + if (a.metadata.vfsType === 'file' && b.metadata.vfsType === 'directory') return 1 + return a.metadata.name.localeCompare(b.metadata.name) + }) + } catch (error) { + console.error(`Failed to load ${path}:`, error.message) + return [] + } } // ✅ Method 2: Get complete tree structure (for full trees) async function loadFullTree(brain: Brainy, path: string) { - const tree = await brain.vfs.getTreeStructure(path, { - maxDepth: 3, // Prevent deep recursion - includeHidden: false, // Skip hidden files - sort: 'name' - }) - return tree + const tree = await brain.vfs.getTreeStructure(path, { + maxDepth: 3, // Prevent deep recursion + includeHidden: false, // Skip hidden files + sort: 'name' + }) + return tree } // ✅ Method 3: Get detailed path info async function inspectPath(brain: Brainy, path: string) { - const info = await brain.vfs.inspect(path) - return { - isDirectory: info.node.metadata.vfsType === 'directory', - children: info.children, - parent: info.parent, - stats: info.stats - } + const info = await brain.vfs.inspect(path) + return { + isDirectory: info.node.metadata.vfsType === 'directory', + children: info.children, + parent: info.parent, + stats: info.stats + } } ``` @@ -89,19 +89,19 @@ async function inspectPath(brain: Brainy, path: string) { ```typescript // ✅ Find files by content, not just filename async function searchFiles(brain: Brainy, query: string, basePath: string = '/') { - const results = await brain.vfs.search(query, { - path: basePath, // Limit search to specific directory - limit: 50, // Reasonable limit - type: 'file' // Only search files, not directories - }) + const results = await brain.vfs.search(query, { + path: basePath, // Limit search to specific directory + limit: 50, // Reasonable limit + type: 'file' // Only search files, not directories + }) - return results.map(result => ({ - path: result.path, - score: result.score, - type: result.type, - size: result.size, - modified: result.modified - })) + return results.map(result => ({ + path: result.path, + score: result.score, + type: result.type, + size: result.size, + modified: result.modified + })) } // Example usage @@ -118,149 +118,149 @@ import React, { useState, useEffect } from 'react' import { Brainy } from '@soulcraft/brainy' export function FileExplorer() { - const [brain, setBrain] = useState(null) - const [currentPath, setCurrentPath] = useState('/') - const [items, setItems] = useState([]) - const [loading, setLoading] = useState(true) - const [searchQuery, setSearchQuery] = useState('') + const [brain, setBrain] = useState(null) + const [currentPath, setCurrentPath] = useState('/') + const [items, setItems] = useState([]) + const [loading, setLoading] = useState(true) + const [searchQuery, setSearchQuery] = useState('') - // Initialize Brainy (VFS auto-initialized!) - useEffect(() => { - async function initBrainy() { - const brainInstance = new Brainy({ - storage: { type: 'filesystem', path: './brainy-data' } - }) - await brainInstance.init() // VFS ready after this! + // Initialize Brainy (VFS auto-initialized!) + useEffect(() => { + async function initBrainy() { + const brainInstance = new Brainy({ + storage: { type: 'filesystem', path: './brainy-data' } + }) + await brainInstance.init() // VFS ready after this! - setBrain(brainInstance) - setLoading(false) - } - initBrainy() - }, []) + setBrain(brainInstance) + setLoading(false) + } + initBrainy() + }, []) - // Load directory contents - const loadDirectory = async (path: string) => { - if (!brain) return + // Load directory contents + const loadDirectory = async (path: string) => { + if (!brain) return - setLoading(true) - try { - // ✅ CORRECT: Use getDirectChildren to prevent recursion - const children = await brain.vfs.getDirectChildren(path) + setLoading(true) + try { + // ✅ CORRECT: Use getDirectChildren to prevent recursion + const children = await brain.vfs.getDirectChildren(path) - // Sort directories first - const sorted = children.sort((a, b) => { - if (a.metadata.vfsType === 'directory' && b.metadata.vfsType === 'file') return -1 - if (a.metadata.vfsType === 'file' && b.metadata.vfsType === 'directory') return 1 - return a.metadata.name.localeCompare(b.metadata.name) - }) + // Sort directories first + const sorted = children.sort((a, b) => { + if (a.metadata.vfsType === 'directory' && b.metadata.vfsType === 'file') return -1 + if (a.metadata.vfsType === 'file' && b.metadata.vfsType === 'directory') return 1 + return a.metadata.name.localeCompare(b.metadata.name) + }) - setItems(sorted) - setCurrentPath(path) - } catch (error) { - console.error('Failed to load directory:', error) - setItems([]) - } finally { - setLoading(false) - } - } + setItems(sorted) + setCurrentPath(path) + } catch (error) { + console.error('Failed to load directory:', error) + setItems([]) + } finally { + setLoading(false) + } + } - // Search files - const handleSearch = async () => { - if (!brain || !searchQuery.trim()) { - loadDirectory(currentPath) - return - } + // Search files + const handleSearch = async () => { + if (!brain || !searchQuery.trim()) { + loadDirectory(currentPath) + return + } - setLoading(true) - try { - const results = await brain.vfs.search(searchQuery, { - path: currentPath, - limit: 100 - }) - setItems(results) - } catch (error) { - console.error('Search failed:', error) - } finally { - setLoading(false) - } - } + setLoading(true) + try { + const results = await brain.vfs.search(searchQuery, { + path: currentPath, + limit: 100 + }) + setItems(results) + } catch (error) { + console.error('Search failed:', error) + } finally { + setLoading(false) + } + } - // Initial load - useEffect(() => { - if (brain) { - loadDirectory('/') - } - }, [brain]) + // Initial load + useEffect(() => { + if (brain) { + loadDirectory('/') + } + }, [brain]) - if (loading && !brain) { - return
Initializing Brainy...
- } + if (loading && !brain) { + return
Initializing Brainy...
+ } - return ( -
- {/* Search bar */} -
- setSearchQuery(e.target.value)} - placeholder="Search files by content..." - onKeyPress={(e) => e.key === 'Enter' && handleSearch()} - /> - - {searchQuery && ( - - )} -
+ return ( +
+ {/* Search bar */} +
+ setSearchQuery(e.target.value)} + placeholder="Search files by content..." + onKeyPress={(e) => e.key === 'Enter' && handleSearch()} + /> + + {searchQuery && ( + + )} +
- {/* Current path */} -
- 📁 {currentPath} - {currentPath !== '/' && ( - - )} -
+ {/* Current path */} +
+ 📁 {currentPath} + {currentPath !== '/' && ( + + )} +
- {/* File list */} - {loading ? ( -
Loading...
- ) : ( -
- {items.map((item) => ( -
{ - if (item.metadata.vfsType === 'directory') { - loadDirectory(item.metadata.path) - } else { - console.log('Open file:', item.metadata.path) - // Add your file opening logic here - } - }} - > - - {item.metadata.vfsType === 'directory' ? '📁' : '📄'} - - {item.metadata.name} - - {item.metadata.size ? `${Math.round(item.metadata.size / 1024)}KB` : ''} - -
- ))} - {items.length === 0 && ( -
- {searchQuery ? 'No results found' : 'Empty directory'} -
- )} -
- )} -
- ) + {/* File list */} + {loading ? ( +
Loading...
+ ) : ( +
+ {items.map((item) => ( +
{ + if (item.metadata.vfsType === 'directory') { + loadDirectory(item.metadata.path) + } else { + console.log('Open file:', item.metadata.path) + // Add your file opening logic here + } + }} + > + + {item.metadata.vfsType === 'directory' ? '📁' : '📄'} + + {item.metadata.name} + + {item.metadata.size ? `${Math.round(item.metadata.size / 1024)}KB` : ''} + +
+ ))} + {items.length === 0 && ( +
+ {searchQuery ? 'No results found' : 'Empty directory'} +
+ )} +
+ )} +
+ ) } ``` @@ -288,13 +288,13 @@ Your file explorer is now working! Here's what to explore next: ### "Module not found" errors ```bash # Make sure you're using the right import -npm ls @soulcraft/brainy # Check version -npm install @soulcraft/brainy@latest # Update if needed +npm ls @soulcraft/brainy # Check version +npm install @soulcraft/brainy@latest # Update if needed ``` ### "VFS not initialized" errors ```typescript -// v5.1.0+: Just await brain.init() - VFS is auto-initialized! +// Just await brain.init() - VFS is auto-initialized! await brain.init() // VFS ready - use brain.vfs directly await brain.vfs.writeFile('/test.txt', 'data') @@ -304,8 +304,8 @@ await brain.vfs.writeFile('/test.txt', 'data') ```typescript // Add pagination for large directories const children = await brain.vfs.getDirectChildren(path, { - limit: 100, // Load only first 100 items - offset: 0 // Start from beginning + limit: 100, // Load only first 100 items + offset: 0 // Start from beginning }) ``` @@ -313,8 +313,8 @@ const children = await brain.vfs.getDirectChildren(path, { ```typescript // Make sure files are imported into VFS first await brain.vfs.importDirectory('./my-files', { - recursive: true, - extractMetadata: true // Enable content understanding + recursive: true, + extractMetadata: true // Enable content understanding }) ``` diff --git a/docs/vfs/README.md b/docs/vfs/README.md index 2618b6ef..3bb9b1e1 100644 --- a/docs/vfs/README.md +++ b/docs/vfs/README.md @@ -28,15 +28,15 @@ import { VirtualFileSystem } from '@soulcraft/brainy/vfs' // Initialize the VFS const vfs = new VirtualFileSystem({ - root: '/my-brain', - intelligent: true // Enable AI features + root: '/my-brain', + intelligent: true // Enable AI features }) await vfs.init() // Write a file - it automatically becomes intelligent await vfs.writeFile('/projects/my-app/index.js', - 'console.log("Hello, World!")') + 'console.log("Hello, World!")') // Find similar files using semantic search const similar = await vfs.findSimilar('/projects/my-app/index.js') @@ -48,11 +48,11 @@ const results = await vfs.search('files about authentication') await vfs.addRelationship('/docs/spec.md', '/projects/my-app/', 'implements') ``` -## ⚡ Performance (v5.11.1) +## ⚡ Performance **75% Faster File Operations!** VFS now automatically benefits from brain.get() metadata-only optimization: -| Operation | Before (v5.11.0) | After (v5.11.1) | Speedup | +| Operation | Before | After | Speedup | |-----------|------------------|-----------------|---------| | `readFile()` | 53ms | **~13ms** | **75%** | | `stat()` | 53ms | **~13ms** | **75%** | @@ -60,7 +60,7 @@ await vfs.addRelationship('/docs/spec.md', '/projects/my-app/', 'implements') **Zero configuration** - automatic optimization for all VFS operations! -VFS operations only need metadata (path, size, timestamps), not 384-dimensional vector embeddings. The v5.11.1 optimization automatically uses metadata-only reads, saving 95% bandwidth and 76-81% time. +VFS operations only need metadata (path, size, timestamps), not 384-dimensional vector embeddings. The optimization automatically uses metadata-only reads, saving 95% bandwidth and 76-81% time. ## Core Features @@ -110,20 +110,20 @@ const docs = await vfs.search('technical documentation for API endpoints') // Find similar files const similar = await vfs.findSimilar('/code/auth.js', { - limit: 5, - threshold: 0.8 // 80% similarity + limit: 5, + threshold: 0.8 // 80% similarity }) // Get related files through the knowledge graph const related = await vfs.getRelated('/proposal.pdf', { - depth: 2 // Include relationships of relationships + depth: 2 // Include relationships of relationships }) // Auto-organization suggestions const suggestions = await vfs.suggestOrganization([ - '/downloads/doc1.pdf', - '/downloads/image.jpg', - '/downloads/code.py' + '/downloads/doc1.pdf', + '/downloads/image.jpg', + '/downloads/code.py' ]) // Returns: suggested folders and categorization ``` @@ -144,10 +144,10 @@ const connections = await vfs.getConnections('/code/impl.js') // Traverse the graph const implementations = await vfs.search('', { - connected: { - to: '/spec.md', - via: 'implements' - } + connected: { + to: '/spec.md', + via: 'implements' + } }) ``` @@ -158,8 +158,8 @@ Store anything alongside your files: ```javascript // Add todos to files await vfs.setTodos('/projects/app/index.js', [ - { task: 'Add error handling', priority: 'high', due: '2024-01-20' }, - { task: 'Optimize performance', priority: 'medium' } + { task: 'Add error handling', priority: 'high', due: '2024-01-20' }, + { task: 'Optimize performance', priority: 'medium' } ]) // Set custom attributes @@ -169,11 +169,11 @@ await vfs.setxattr('/video.mp4', 'tags', ['tutorial', 'react', 'hooks']) // Query by metadata const urgent = await vfs.search('', { - where: { 'todos.priority': 'high' } + where: { 'todos.priority': 'high' } }) const parisPhotos = await vfs.search('', { - where: { location: 'Paris, France' } + where: { location: 'Paris, France' } }) ``` @@ -184,20 +184,20 @@ Access files through semantic dimensions (see [Semantic VFS](./SEMANTIC_VFS.md)) ```javascript // Query-based path access (current functionality) const authFiles = await vfs.search('', { - where: { concepts: { contains: 'authentication' }} + where: { concepts: { contains: 'authentication' }} }) // Find files by custom metadata const recent = await vfs.search('', { - where: { - type: 'document', - modified: { greaterThan: Date.now() - 7*24*60*60*1000 } - } + where: { + type: 'document', + modified: { greaterThan: Date.now() - 7*24*60*60*1000 } + } }) // Find similar files const similar = await vfs.findSimilar('/examples/good-code.js', { - threshold: 0.7 + threshold: 0.7 }) ``` @@ -210,24 +210,24 @@ const similar = await vfs.findSimilar('/examples/good-code.js', { ```javascript // Store a research paper with automatic analysis await vfs.writeFile('/research/quantum-computing.pdf', pdfBuffer, { - metadata: { - authors: ['Dr. Alice Smith', 'Dr. Bob Jones'], - year: 2024, - topics: ['quantum', 'computing', 'algorithms'], - citations: 42 - } + metadata: { + authors: ['Dr. Alice Smith', 'Dr. Bob Jones'], + year: 2024, + topics: ['quantum', 'computing', 'algorithms'], + citations: 42 + } }) // Find all papers on similar topics const related = await vfs.search('quantum algorithms', { - type: 'document', - where: { year: { $gte: 2020 } } + type: 'document', + where: { year: { $gte: 2020 } } }) // Find papers that cite this one const citations = await vfs.getConnections('/research/quantum-computing.pdf', { - type: 'cites', - direction: 'incoming' + type: 'cites', + direction: 'incoming' }) ``` @@ -245,12 +245,12 @@ await vfs.writeFile('/src/utils/auth.js', authCode) // Find all files that import this module const importers = await vfs.search('', { - where: { dependencies: 'utils/auth.js' } + where: { dependencies: 'utils/auth.js' } }) // Find test files for this code const tests = await vfs.getRelated('/src/utils/auth.js', { - type: 'tests' + type: 'tests' }) // Find similar implementations @@ -262,12 +262,12 @@ const similar = await vfs.findSimilar('/src/utils/auth.js') ```javascript // Store media with rich metadata await vfs.writeFile('/photos/sunset.jpg', imageBuffer, { - metadata: { - camera: 'Canon R5', - location: { lat: 37.7749, lng: -122.4194 }, - tags: ['sunset', 'golden-gate', 'landscape'], - album: 'San Francisco 2024' - } + metadata: { + camera: 'Canon R5', + location: { lat: 37.7749, lng: -122.4194 }, + tags: ['sunset', 'golden-gate', 'landscape'], + album: 'San Francisco 2024' + } }) // Find similar images @@ -275,18 +275,18 @@ const similar = await vfs.findSimilar('/photos/sunset.jpg') // Find photos by location const nearby = await vfs.search('', { - type: 'image', - where: { - 'location.lat': { $between: [37.7, 37.8] }, - 'location.lng': { $between: [-122.5, -122.3] } - } + type: 'image', + where: { + 'location.lat': { $between: [37.7, 37.8] }, + 'location.lng': { $between: [-122.5, -122.3] } + } }) // Smart albums await vfs.createVirtualDirectory('/albums/best-sunsets', { - query: 'sunset', - type: 'image', - where: { rating: { $gte: 4 } } + query: 'sunset', + type: 'image', + where: { rating: { $gte: 4 } } }) ``` @@ -308,25 +308,25 @@ await vfs.setxattr(projectPath, 'status', 'in-progress') // Add todos to specific files await vfs.setTodos(`${projectPath}/src/index.js`, [ - { task: 'Implement user authentication', assignee: 'Alice' }, - { task: 'Add error handling', assignee: 'Bob' } + { task: 'Implement user authentication', assignee: 'Alice' }, + { task: 'Add error handling', assignee: 'Bob' } ]) // Find all files with pending todos const pending = await vfs.search('', { - where: { - path: { $startsWith: projectPath }, - 'todos.status': 'pending' - } + where: { + path: { $startsWith: projectPath }, + 'todos.status': 'pending' + } }) // Find projects nearing deadline const urgent = await vfs.search('', { - where: { - type: 'directory', - 'deadline': { $lte: '2024-02-01' }, - 'status': 'in-progress' - } + where: { + type: 'directory', + 'deadline': { $lte: '2024-02-01' }, + 'status': 'in-progress' + } }) ``` @@ -417,9 +417,9 @@ The VFS is built with production scalability in mind: #### **Storage Strategy** ```typescript // Adaptive storage based on file size -< 100KB: Inline storage (entity.data) -< 10MB: External reference (S3/R2 key) -> 10MB: Chunked storage (parallel chunks) +< 100KB: Inline storage (entity.data) +< 10MB: External reference (S3/R2 key) +> 10MB: Chunked storage (parallel chunks) ``` #### **Performance Metrics** @@ -434,24 +434,24 @@ The VFS is built with production scalability in mind: #### **How It Handles Scale** 1. **Hierarchical Caching** - - 100K+ path cache entries - - Parent-child relationship caching - - Hot path detection and optimization + - 100K+ path cache entries + - Parent-child relationship caching + - Hot path detection and optimization 2. **Distributed Architecture** - - Sharding by path prefix - - Read replicas for hot directories - - CDN integration for static files + - Sharding by path prefix + - Read replicas for hot directories + - CDN integration for static files 3. **Intelligent Indexing** - - Compound indexes on (parent, name) - - Vector indexes for semantic search - - Graph indexes for relationships + - Compound indexes on (parent, name) + - Vector indexes for semantic search + - Graph indexes for relationships 4. **Streaming Everything** - - Large files never fully in memory - - Progressive loading - - Chunked transfers + - Large files never fully in memory + - Progressive loading + - Chunked transfers ### Real Production Scenarios @@ -462,29 +462,29 @@ Store build artifacts with automatic relationships: ```javascript // Store build output with metadata await vfs.writeFile('/builds/v1.2.3/app.js', buildOutput, { - metadata: { - commit: 'abc123', - branch: 'main', - timestamp: Date.now(), - tests: 'passing', - coverage: 0.92 - } + metadata: { + commit: 'abc123', + branch: 'main', + timestamp: Date.now(), + tests: 'passing', + coverage: 0.92 + } }) // Find all builds for a commit const builds = await vfs.search('', { - where: { commit: 'abc123' } + where: { commit: 'abc123' } }) // Get latest passing build const latest = await vfs.search('', { - where: { - branch: 'main', - tests: 'passing' - }, - sort: 'modified', - order: 'desc', - limit: 1 + where: { + branch: 'main', + tests: 'passing' + }, + sort: 'modified', + order: 'desc', + limit: 1 }) ``` @@ -495,8 +495,8 @@ Isolate customer data with semantic understanding: ```javascript // Each tenant gets their own root const tenantVfs = new VirtualFileSystem({ - root: `/tenants/${tenantId}`, - service: tenantId // Isolate at Brainy level too + root: `/tenants/${tenantId}`, + service: tenantId // Isolate at Brainy level too }) // Tenant uploads document @@ -505,11 +505,11 @@ await tenantVfs.writeFile('/documents/contract.pdf', pdfBuffer) // Cross-tenant analytics (admin only) const adminVfs = new VirtualFileSystem({ root: '/tenants' }) const stats = await adminVfs.search('contract', { - recursive: true, - aggregations: { - byTenant: { field: 'service' }, - byType: { field: 'mimeType' } - } + recursive: true, + aggregations: { + byTenant: { field: 'service' }, + byType: { field: 'mimeType' } + } }) ``` @@ -520,38 +520,38 @@ Connect datasets, models, and results: ```javascript // Store training data await vfs.writeFile('/datasets/train.csv', csvData, { - metadata: { - samples: 100000, - features: 50, - labels: 10 - } + metadata: { + samples: 100000, + features: 50, + labels: 10 + } }) // Store trained model await vfs.writeFile('/models/v1/model.pkl', modelBuffer, { - metadata: { - algorithm: 'random-forest', - accuracy: 0.95, - trainedOn: '/datasets/train.csv', - hyperparameters: { trees: 100, depth: 10 } - } + metadata: { + algorithm: 'random-forest', + accuracy: 0.95, + trainedOn: '/datasets/train.csv', + hyperparameters: { trees: 100, depth: 10 } + } }) // Connect model to its training data await vfs.addRelationship( - '/models/v1/model.pkl', - '/datasets/train.csv', - 'trained-on' + '/models/v1/model.pkl', + '/datasets/train.csv', + 'trained-on' ) // Find best model for a dataset const models = await vfs.search('', { - connected: { - to: '/datasets/train.csv', - via: 'trained-on' - }, - sort: 'accuracy', - order: 'desc' + connected: { + to: '/datasets/train.csv', + via: 'trained-on' + }, + sort: 'accuracy', + order: 'desc' }) ``` @@ -562,30 +562,30 @@ Intelligent content organization: ```javascript // Auto-organize uploads vfs.on('file:added', async (path) => { - if (path.startsWith('/uploads/')) { - const file = await vfs.getEntity(path) + if (path.startsWith('/uploads/')) { + const file = await vfs.getEntity(path) - // Auto-categorize by AI - const category = await detectCategory(file) - const newPath = `/content/${category}/${file.metadata.name}` + // Auto-categorize by AI + const category = await detectCategory(file) + const newPath = `/content/${category}/${file.metadata.name}` - await vfs.move(path, newPath) + await vfs.move(path, newPath) - // Auto-tag - const tags = await extractTags(file) - await vfs.setxattr(newPath, 'tags', tags) + // Auto-tag + const tags = await extractTags(file) + await vfs.setxattr(newPath, 'tags', tags) - // Find related content - const related = await vfs.findSimilar(newPath, { - limit: 5, - threshold: 0.8 - }) + // Find related content + const related = await vfs.findSimilar(newPath, { + limit: 5, + threshold: 0.8 + }) - // Create relationships - for (const rel of related) { - await vfs.addRelationship(newPath, rel.path, 'related-to') - } - } + // Create relationships + for (const rel of related) { + await vfs.addRelationship(newPath, rel.path, 'related-to') + } + } }) ``` @@ -596,39 +596,39 @@ Collaborative file management: ```javascript // Track file ownership and access await vfs.writeFile('/projects/alpha/spec.md', content, { - metadata: { - owner: userId, - team: 'engineering', - permissions: { - [userId]: 'rw', - 'team:engineering': 'r', - 'others': '-' - } - } + metadata: { + owner: userId, + team: 'engineering', + permissions: { + [userId]: 'rw', + 'team:engineering': 'r', + 'others': '-' + } + } }) // Add collaborative features await vfs.addTodo('/projects/alpha/spec.md', { - task: 'Review security section', - assignee: 'alice@company.com', - due: '2024-02-01', - priority: 'high' + task: 'Review security section', + assignee: 'alice@company.com', + due: '2024-02-01', + priority: 'high' }) // Track who's working on what await vfs.setxattr('/projects/alpha/spec.md', 'locks', { - section3: { - user: 'bob@company.com', - since: Date.now() - } + section3: { + user: 'bob@company.com', + since: Date.now() + } }) // Find all files assigned to a user const assigned = await vfs.search('', { - where: { - 'todos.assignee': 'alice@company.com', - 'todos.status': 'pending' - } + where: { + 'todos.assignee': 'alice@company.com', + 'todos.status': 'pending' + } }) ``` @@ -643,15 +643,15 @@ const assigned = await vfs.search('', { #### **Standalone Mode** ```javascript const vfs = new VirtualFileSystem() -await vfs.init() // Uses in-memory storage +await vfs.init() // Uses in-memory storage ``` #### **Local Persistence** ```javascript const vfs = new VirtualFileSystem() await vfs.init({ - storage: 'filesystem', - dataDir: '/var/lib/brainy-vfs' + storage: 'filesystem', + dataDir: '/var/lib/brainy-vfs' }) ``` @@ -659,9 +659,9 @@ await vfs.init({ ```javascript const vfs = new VirtualFileSystem() await vfs.init({ - storage: 's3', - bucket: 'my-vfs-data', - region: 'us-west-2' + storage: 's3', + bucket: 'my-vfs-data', + region: 'us-west-2' }) ``` @@ -669,14 +669,14 @@ await vfs.init({ ```javascript const vfs = new VirtualFileSystem() await vfs.init({ - distributed: true, - nodes: [ - 'vfs1.internal:8080', - 'vfs2.internal:8080', - 'vfs3.internal:8080' - ], - replication: 3, - consistency: 'eventual' + distributed: true, + nodes: [ + 'vfs1.internal:8080', + 'vfs2.internal:8080', + 'vfs3.internal:8080' + ], + replication: 3, + consistency: 'eventual' }) ``` diff --git a/docs/vfs/TROUBLESHOOTING.md b/docs/vfs/TROUBLESHOOTING.md index d3c75144..f1bbd120 100644 --- a/docs/vfs/TROUBLESHOOTING.md +++ b/docs/vfs/TROUBLESHOOTING.md @@ -12,8 +12,8 @@ **Root Cause:** The root directory entity exists but doesn't have the proper metadata structure. -**Solution (v3.15.0+):** -This issue has been fixed in v3.15.0. The VFS now: +**Solution:** +This issue has been fixed. The VFS now: 1. Ensures root directory has `vfsType: 'directory'` metadata 2. Adds compatibility layer for entities with malformed metadata 3. Automatically repairs metadata on entity retrieval @@ -21,17 +21,17 @@ This issue has been fixed in v3.15.0. The VFS now: **Manual Fix (if needed):** ```javascript // Force re-initialization of root directory -await vfs.init() // Will repair root if needed +await vfs.init() // Will repair root if needed // Or manually update root entity const rootId = vfs.rootEntityId await brain.update({ - id: rootId, - metadata: { - path: '/', - vfsType: 'directory', - // ... other metadata - } + id: rootId, + metadata: { + path: '/', + vfsType: 'directory', + // ... other metadata + } }) ``` @@ -47,7 +47,7 @@ await brain.update({ **Root Cause:** Contains relationships are missing between parent directories and files. -**Solution (v3.15.0+):** +**Solution:** This issue has been fixed. The VFS now: 1. Creates Contains relationships when writing new files 2. Ensures Contains relationships exist when updating files @@ -60,9 +60,9 @@ const parentId = await vfs.resolvePath('/directory') const fileId = await vfs.resolvePath('/directory/file.txt') await brain.relate({ - from: parentId, - to: fileId, - type: VerbType.Contains + from: parentId, + to: fileId, + type: VerbType.Contains }) ``` @@ -80,8 +80,8 @@ The VFS `init()` method wasn't called after getting the VFS instance. **Solution:** ```javascript // ✅ CORRECT -const vfs = brain.vfs() // Get instance -await vfs.init() // Initialize (REQUIRED!) +const vfs = brain.vfs() // Get instance +await vfs.init() // Initialize (REQUIRED!) // ❌ WRONG const vfs = brain.vfs() @@ -104,15 +104,15 @@ Using in-memory storage instead of persistent storage. ```javascript // ✅ Use persistent storage const brain = new Brainy({ - storage: { - type: 'filesystem', // Persistent - path: './brainy-data' - } + storage: { + type: 'filesystem', // Persistent + path: './brainy-data' + } }) // ❌ Don't use memory for production const brain = new Brainy({ - storage: { type: 'memory' } // Data lost on restart! + storage: { type: 'memory' } // Data lost on restart! }) ``` @@ -136,7 +136,7 @@ const children = await vfs.getDirectChildren('/directory') // ❌ WRONG - Path prefix matching causes recursion const allNodes = await brain.find({}) const children = allNodes.filter(n => - n.metadata.path.startsWith('/directory/')) // Directory matches itself! + n.metadata.path.startsWith('/directory/')) // Directory matches itself! ``` --- @@ -153,13 +153,13 @@ Files aren't being properly embedded or indexed. **Solution:** ```javascript // Ensure files have content for embedding -await vfs.writeFile('/doc.txt', 'Actual content here') // Not empty! +await vfs.writeFile('/doc.txt', 'Actual content here') // Not empty! // Use semantic search correctly const results = await vfs.search('machine learning', { - path: '/documents', // Search within path - limit: 10, - type: 'file' + path: '/documents', // Search within path + limit: 10, + type: 'file' }) ``` @@ -184,8 +184,8 @@ console.log('VFS type:', entity.metadata.vfsType) ```javascript const parentId = await vfs.resolvePath('/directory') const relations = await brain.getRelations({ - from: parentId, - type: VerbType.Contains + from: parentId, + type: VerbType.Contains }) console.log('Child count:', relations.length) ``` @@ -193,11 +193,11 @@ console.log('Child count:', relations.length) ### 4. Enable Debug Logging ```javascript const brain = new Brainy({ - storage: { type: 'filesystem' }, - logger: { - level: 'debug', - enabled: true - } + storage: { type: 'filesystem' }, + logger: { + level: 'debug', + enabled: true + } }) ``` @@ -209,11 +209,11 @@ const brain = new Brainy({ ```javascript // Enable caching in VFS config const vfs = brain.vfs({ - cache: { - enabled: true, - ttl: 300000, // 5 minutes - maxSize: 1000 - } + cache: { + enabled: true, + ttl: 300000, // 5 minutes + maxSize: 1000 + } }) ``` @@ -221,12 +221,12 @@ const vfs = brain.vfs({ ```javascript // Write multiple files efficiently const files = [ - { path: '/file1.txt', content: 'content1' }, - { path: '/file2.txt', content: 'content2' } + { path: '/file1.txt', content: 'content1' }, + { path: '/file2.txt', content: 'content2' } ] await Promise.all( - files.map(f => vfs.writeFile(f.path, f.content)) + files.map(f => vfs.writeFile(f.path, f.content)) ) ``` @@ -234,8 +234,8 @@ await Promise.all( ```javascript // Don't traverse too deep const tree = await vfs.getTreeStructure('/', { - maxDepth: 3, // Limit recursion - includeHidden: false + maxDepth: 3, // Limit recursion + includeHidden: false }) ``` diff --git a/docs/vfs/VFS_CORE.md b/docs/vfs/VFS_CORE.md index 5634bef2..892ee819 100644 --- a/docs/vfs/VFS_CORE.md +++ b/docs/vfs/VFS_CORE.md @@ -418,7 +418,7 @@ await vfs.importDirectory()// Import directory from filesystem ## Bulk Write Operations -**v6.5.0**: `bulkWrite` efficiently processes multiple VFS operations with automatic ordering to prevent race conditions. +`bulkWrite` efficiently processes multiple VFS operations with automatic ordering to prevent race conditions. ```javascript const result = await vfs.bulkWrite([ @@ -439,7 +439,7 @@ console.log(`Successful: ${result.successful}, Failed: ${result.failed.length}`) - `delete` - Delete file - `update` - Update file metadata only -**Operation ordering (v6.5.0):** +**Operation ordering:** 1. `mkdir` operations run **first**, sequentially, sorted by path depth (shallowest first) 2. Other operations (`write`, `delete`, `update`) run **after** in parallel batches of 10 diff --git a/docs/vfs/VFS_INITIALIZATION.md b/docs/vfs/VFS_INITIALIZATION.md index 86581e71..f6b91e0e 100644 --- a/docs/vfs/VFS_INITIALIZATION.md +++ b/docs/vfs/VFS_INITIALIZATION.md @@ -1,4 +1,4 @@ -# VFS Initialization Guide (v5.1.0+) +# VFS Initialization Guide ## Quick Start @@ -30,7 +30,7 @@ await vfs.init() // ❌ Separate initialization await vfs.writeFile(...) ``` -### After (v5.1.0+): +### After: ```javascript const brain = new Brainy(...) await brain.init() // VFS auto-initialized! @@ -53,7 +53,7 @@ await vfs.init() await vfs.writeFile('/test.txt', 'data') ``` -### New Pattern (v5.1.0+): +### New Pattern: ```javascript // Just remove the () and init() call await brain.vfs.writeFile('/test.txt', 'data') @@ -142,7 +142,7 @@ await brain.init() // Required! await brain.vfs.writeFile(...) // Now this works ``` -## Fork Support (v5.0.0+) +## Fork Support VFS works seamlessly with Brainy's Copy-on-Write (COW) fork feature: diff --git a/src/api/ConfigAPI.ts b/src/api/ConfigAPI.ts index dbaaa56c..7e7647c5 100644 --- a/src/api/ConfigAPI.ts +++ b/src/api/ConfigAPI.ts @@ -60,7 +60,7 @@ export class ConfigAPI { // Store in cache this.configCache.set(key, entry) - // v4.0.0: Persist to storage as NounMetadata + // Persist to storage as NounMetadata const configId = this.CONFIG_NOUN_PREFIX + key const nounMetadata = { noun: 'config' as any, @@ -94,7 +94,7 @@ export class ConfigAPI { return defaultValue } - // v4.0.0: Extract ConfigEntry from NounMetadata + // Extract ConfigEntry from NounMetadata const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt ? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000) : ((metadata.createdAt as unknown as number) || Date.now()) @@ -140,7 +140,7 @@ export class ConfigAPI { // Remove from cache this.configCache.delete(key) - // v4.0.0: Remove from storage + // Remove from storage const configId = this.CONFIG_NOUN_PREFIX + key await this.storage.deleteNounMetadata(configId) } @@ -149,7 +149,7 @@ export class ConfigAPI { * List all configuration keys */ async list(): Promise { - // v4.0.0: Get all nouns and filter for config entries + // Get all nouns and filter for config entries const result = await this.storage.getNouns({ pagination: { limit: 10000 } }) const configKeys: string[] = [] @@ -213,7 +213,7 @@ export class ConfigAPI { for (const [key, entry] of Object.entries(config)) { this.configCache.set(key, entry) const configId = this.CONFIG_NOUN_PREFIX + key - // v4.0.0: Convert ConfigEntry to NounMetadata + // Convert ConfigEntry to NounMetadata const nounMetadata = { noun: 'config' as any, ...entry, @@ -240,7 +240,7 @@ export class ConfigAPI { return null } - // v4.0.0: Extract ConfigEntry from NounMetadata + // Extract ConfigEntry from NounMetadata const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt ? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000) : ((metadata.createdAt as unknown as number) || Date.now()) diff --git a/src/api/DataAPI.ts b/src/api/DataAPI.ts index 7e342aa9..8ba1a49f 100644 --- a/src/api/DataAPI.ts +++ b/src/api/DataAPI.ts @@ -210,7 +210,7 @@ export class DataAPI { this.validateImportItem(mapped) } - // v4.0.0: Save entity - separate vector and metadata + // Save entity - separate vector and metadata const id = mapped.id || this.generateId() const noun: HNSWNoun = { id, diff --git a/src/augmentations/brainyAugmentation.ts b/src/augmentations/brainyAugmentation.ts index 38232524..7b94f170 100644 --- a/src/augmentations/brainyAugmentation.ts +++ b/src/augmentations/brainyAugmentation.ts @@ -375,7 +375,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation { } /** - * Get CommitLog for temporal features (v5.0.0+) + * Get CommitLog for temporal features * * Provides access to commit history for time-travel queries, audit trails, * and branch management. Available after initialize() is called. @@ -414,7 +414,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation { } /** - * Get BlobStorage for content-addressable storage (v5.0.0+) + * Get BlobStorage for content-addressable storage * * Provides access to the underlying blob storage system for storing * and retrieving content-addressed data. Available after initialize() is called. @@ -454,7 +454,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation { } /** - * Get RefManager for branch/ref management (v5.0.0+) + * Get RefManager for branch/ref management * * Provides access to the reference manager for creating, updating, * and managing Git-style branches and refs. Available after initialize() is called. @@ -496,7 +496,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation { } /** - * Get current branch name (v5.0.0+) + * Get current branch name * * Convenience helper for getting the current branch from the Brainy instance. * Available after initialize() is called. @@ -525,7 +525,7 @@ export abstract class BaseAugmentation implements BrainyAugmentation { if (typeof brain.getCurrentBranch !== 'function') { throw new Error( `${this.name}: getCurrentBranch() not available on Brainy instance. ` + - `This method requires Brainy v5.0.0+.` + `This method requires Brainy with VFS support.` ) } diff --git a/src/augmentations/intelligentImport/FormatHandlerRegistry.ts b/src/augmentations/intelligentImport/FormatHandlerRegistry.ts index 71973b94..4daff432 100644 --- a/src/augmentations/intelligentImport/FormatHandlerRegistry.ts +++ b/src/augmentations/intelligentImport/FormatHandlerRegistry.ts @@ -1,5 +1,5 @@ /** - * Format Handler Registry (v5.2.0) + * Format Handler Registry * * Central registry for format handlers with: * - MIME type-based routing diff --git a/src/augmentations/intelligentImport/IntelligentImportAugmentation.ts b/src/augmentations/intelligentImport/IntelligentImportAugmentation.ts index aeececcf..0e21b585 100644 --- a/src/augmentations/intelligentImport/IntelligentImportAugmentation.ts +++ b/src/augmentations/intelligentImport/IntelligentImportAugmentation.ts @@ -35,7 +35,7 @@ export class IntelligentImportAugmentation extends BaseAugmentation { enableCSV: true, enableExcel: true, enablePDF: true, - enableImage: true, // v5.2.0: Image handler enabled by default + enableImage: true, // Image handler enabled by default maxFileSize: 100 * 1024 * 1024, // 100MB default enableCache: true, cacheTTL: 24 * 60 * 60 * 1000, // 24 hours diff --git a/src/augmentations/intelligentImport/handlers/csvHandler.ts b/src/augmentations/intelligentImport/handlers/csvHandler.ts index 87951ad4..3287e258 100644 --- a/src/augmentations/intelligentImport/handlers/csvHandler.ts +++ b/src/augmentations/intelligentImport/handlers/csvHandler.ts @@ -40,7 +40,7 @@ export class CSVHandler extends BaseFormatHandler { const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf-8') const totalBytes = buffer.length - // v4.5.0: Report total bytes for progress tracking + // Report total bytes for progress tracking if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(0) } @@ -55,7 +55,7 @@ export class CSVHandler extends BaseFormatHandler { // Detect delimiter if not specified const delimiter = options.csvDelimiter || this.detectDelimiter(text) - // v4.5.0: Report progress - parsing started + // Report progress - parsing started if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem(`Parsing CSV rows (delimiter: "${delimiter}")...`) } @@ -75,7 +75,7 @@ export class CSVHandler extends BaseFormatHandler { cast: false // We'll do type inference ourselves }) - // v4.5.0: Report bytes processed (entire file parsed) + // Report bytes processed (entire file parsed) if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(totalBytes) } @@ -83,7 +83,7 @@ export class CSVHandler extends BaseFormatHandler { // Convert to array of objects const data = Array.isArray(records) ? records : [records] - // v4.5.0: Report data extraction progress + // Report data extraction progress if (progressHooks?.onDataExtracted) { progressHooks.onDataExtracted(data.length, data.length) } @@ -101,7 +101,7 @@ export class CSVHandler extends BaseFormatHandler { converted[key] = this.convertValue(value, types[key] || 'string') } - // v4.5.0: Report progress every 1000 rows + // Report progress every 1000 rows if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) { progressHooks.onCurrentItem(`Converting types: ${index}/${data.length} rows...`) } @@ -111,7 +111,7 @@ export class CSVHandler extends BaseFormatHandler { const processingTime = Date.now() - startTime - // v4.5.0: Final progress update + // Final progress update if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem(`CSV processing complete: ${convertedData.length} rows`) } diff --git a/src/augmentations/intelligentImport/handlers/excelHandler.ts b/src/augmentations/intelligentImport/handlers/excelHandler.ts index f5f7e314..64a6aaa5 100644 --- a/src/augmentations/intelligentImport/handlers/excelHandler.ts +++ b/src/augmentations/intelligentImport/handlers/excelHandler.ts @@ -27,7 +27,7 @@ export class ExcelHandler extends BaseFormatHandler { const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary') const totalBytes = buffer.length - // v4.5.0: Report start + // Report start if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(0) } @@ -47,7 +47,7 @@ export class ExcelHandler extends BaseFormatHandler { // Determine which sheets to process const sheetsToProcess = this.getSheetsToProcess(workbook, options) - // v4.5.0: Report workbook loaded + // Report workbook loaded if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem(`Processing ${sheetsToProcess.length} sheets...`) } @@ -59,7 +59,7 @@ export class ExcelHandler extends BaseFormatHandler { for (let sheetIndex = 0; sheetIndex < sheetsToProcess.length; sheetIndex++) { const sheetName = sheetsToProcess[sheetIndex] - // v4.5.0: Report current sheet + // Report current sheet if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem( `Reading sheet: ${sheetName} (${sheetIndex + 1}/${sheetsToProcess.length})` @@ -117,19 +117,19 @@ export class ExcelHandler extends BaseFormatHandler { headers } - // v4.5.0: Estimate bytes processed (sheets are sequential) + // Estimate bytes processed (sheets are sequential) const bytesProcessed = Math.floor(((sheetIndex + 1) / sheetsToProcess.length) * totalBytes) if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(bytesProcessed) } - // v4.5.0: Report extraction progress + // Report extraction progress if (progressHooks?.onDataExtracted) { progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete } } - // v4.5.0: Report data extraction complete + // Report data extraction complete if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem(`Extracted ${allData.length} rows, inferring types...`) } @@ -152,7 +152,7 @@ export class ExcelHandler extends BaseFormatHandler { } } - // v4.5.0: Report progress every 1000 rows (avoid spam) + // Report progress every 1000 rows (avoid spam) if (progressHooks?.onCurrentItem && index > 0 && index % 1000 === 0) { progressHooks.onCurrentItem(`Converting types: ${index}/${allData.length} rows...`) } @@ -160,14 +160,14 @@ export class ExcelHandler extends BaseFormatHandler { return converted }) - // v4.5.0: Final progress - all bytes processed + // Final progress - all bytes processed if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(totalBytes) } const processingTime = Date.now() - startTime - // v4.5.0: Report completion + // Report completion if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem( `Excel complete: ${sheetsToProcess.length} sheets, ${convertedData.length} rows` diff --git a/src/augmentations/intelligentImport/handlers/imageHandler.ts b/src/augmentations/intelligentImport/handlers/imageHandler.ts index 8d529646..f896e489 100644 --- a/src/augmentations/intelligentImport/handlers/imageHandler.ts +++ b/src/augmentations/intelligentImport/handlers/imageHandler.ts @@ -1,5 +1,5 @@ /** - * Image Import Handler (v5.8.0 - Pure JavaScript) + * Image Import Handler * * Handles image files with: * - EXIF metadata extraction (camera, GPS, timestamps) via exifr @@ -7,7 +7,7 @@ * - Support for JPEG, PNG, WebP, GIF, TIFF, BMP, SVG * * NO NATIVE DEPENDENCIES - Pure JavaScript implementation - * Replaces Sharp (v5.7.x) with lightweight pure-JS alternatives + * Replaces Sharp with lightweight pure-JS alternatives */ import { BaseFormatHandler } from './base.js' @@ -96,7 +96,7 @@ export interface ImageHandlerOptions extends FormatHandlerOptions { * Enables developers to import images into the knowledge graph with * full metadata extraction. * - * v5.8.0: Pure JavaScript implementation (no native dependencies) + * Pure JavaScript implementation (no native dependencies) */ export class ImageHandler extends BaseFormatHandler { readonly format = 'image' diff --git a/src/augmentations/intelligentImport/handlers/pdfHandler.ts b/src/augmentations/intelligentImport/handlers/pdfHandler.ts index f8bc9838..91ee70e8 100644 --- a/src/augmentations/intelligentImport/handlers/pdfHandler.ts +++ b/src/augmentations/intelligentImport/handlers/pdfHandler.ts @@ -52,7 +52,7 @@ export class PDFHandler extends BaseFormatHandler { const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'binary') const totalBytes = buffer.length - // v4.5.0: Report start + // Report start if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(0) } @@ -74,7 +74,7 @@ export class PDFHandler extends BaseFormatHandler { const metadata = await pdfDoc.getMetadata() const numPages = pdfDoc.numPages - // v4.5.0: Report document loaded + // Report document loaded if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem(`Processing ${numPages} pages...`) } @@ -85,7 +85,7 @@ export class PDFHandler extends BaseFormatHandler { let detectedTables = 0 for (let pageNum = 1; pageNum <= numPages; pageNum++) { - // v4.5.0: Report current page + // Report current page if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem(`Processing page ${pageNum} of ${numPages}`) } @@ -131,19 +131,19 @@ export class PDFHandler extends BaseFormatHandler { } } - // v4.5.0: Estimate bytes processed (pages are sequential) + // Estimate bytes processed (pages are sequential) const bytesProcessed = Math.floor((pageNum / numPages) * totalBytes) if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(bytesProcessed) } - // v4.5.0: Report extraction progress + // Report extraction progress if (progressHooks?.onDataExtracted) { progressHooks.onDataExtracted(allData.length, undefined) // Total unknown until complete } } - // v4.5.0: Final progress - all bytes processed + // Final progress - all bytes processed if (progressHooks?.onBytesProcessed) { progressHooks.onBytesProcessed(totalBytes) } @@ -153,7 +153,7 @@ export class PDFHandler extends BaseFormatHandler { const processingTime = Date.now() - startTime - // v4.5.0: Report completion + // Report completion if (progressHooks?.onCurrentItem) { progressHooks.onCurrentItem( `PDF complete: ${numPages} pages, ${allData.length} items extracted` diff --git a/src/augmentations/intelligentImport/index.ts b/src/augmentations/intelligentImport/index.ts index d6f378bf..5c8c182d 100644 --- a/src/augmentations/intelligentImport/index.ts +++ b/src/augmentations/intelligentImport/index.ts @@ -18,14 +18,14 @@ export { ExcelHandler } from './handlers/excelHandler.js' export { PDFHandler } from './handlers/pdfHandler.js' export { ImageHandler } from './handlers/imageHandler.js' -// Format Handler Registry (v5.2.0) +// Format Handler Registry export { FormatHandlerRegistry, globalHandlerRegistry } from './FormatHandlerRegistry.js' export type { HandlerRegistration } from './FormatHandlerRegistry.js' -// Image Handler Types (v5.2.0) +// Image Handler Types export type { ImageMetadata, EXIFData, diff --git a/src/augmentations/intelligentImport/types.ts b/src/augmentations/intelligentImport/types.ts index f4073b28..2d847140 100644 --- a/src/augmentations/intelligentImport/types.ts +++ b/src/augmentations/intelligentImport/types.ts @@ -88,13 +88,13 @@ export interface FormatHandlerOptions { streaming?: boolean /** - * Progress hooks (v4.5.0) + * Progress hooks * Handlers call these to report progress during processing */ progressHooks?: FormatHandlerProgressHooks /** - * Total file size in bytes (v4.5.0) + * Total file size in bytes * Used for progress percentage calculation */ totalBytes?: number @@ -168,7 +168,7 @@ export interface IntelligentImportConfig { /** Enable PDF handler */ enablePDF: boolean - /** Enable Image handler (v5.2.0) */ + /** Enable Image handler */ enableImage: boolean /** Default options for CSV */ @@ -180,7 +180,7 @@ export interface IntelligentImportConfig { /** Default options for PDF */ pdfDefaults?: Partial - /** Default options for Image (v5.2.0) */ + /** Default options for Image */ imageDefaults?: Partial /** Maximum file size to process (bytes) */ diff --git a/src/augmentations/typeMatching/brainyTypes.ts b/src/augmentations/typeMatching/brainyTypes.ts index d904351c..ea7d55f9 100644 --- a/src/augmentations/typeMatching/brainyTypes.ts +++ b/src/augmentations/typeMatching/brainyTypes.ts @@ -168,7 +168,7 @@ export interface TypeMatchResult { /** * BrainyTypes - Intelligent type detection for nouns and verbs - * PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed type embeddings + * PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings * Type embeddings are loaded instantly; only input objects are embedded at runtime */ export class BrainyTypes { diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index 5b2f8473..97437cf7 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -172,7 +172,7 @@ export const coreCommands = { metadata } - // v4.3.x: Add confidence and weight if provided + // Add confidence and weight if provided if (options.confidence) { addParams.confidence = parseFloat(options.confidence) } @@ -347,7 +347,7 @@ export const coreCommands = { searchParams.includeRelations = true } - // v4.7.0: VFS is now part of the knowledge graph (included by default) + // VFS is now part of the knowledge graph (included by default) // Users can exclude VFS with --where vfsType exists:false if needed // Triple Intelligence Fusion - custom weighting diff --git a/src/cli/commands/cow.ts b/src/cli/commands/cow.ts index d1483dde..d6078be3 100644 --- a/src/cli/commands/cow.ts +++ b/src/cli/commands/cow.ts @@ -312,7 +312,7 @@ ${chalk.cyan('Fork Statistics:')} }, /** - * Migrate from v4.x to v5.0.0 (one-time) + * Migrate storage format (one-time) */ async migrate(options: MigrateOptions) { let spinner: any = null @@ -388,7 +388,7 @@ ${chalk.cyan('Fork Statistics:')} await oldBrain.init() - // Create new brain (v5.0.0) + // Create new brain const newBrain = new Brainy({ storage: { type: 'filesystem', diff --git a/src/cli/commands/neural.ts b/src/cli/commands/neural.ts index 5f775857..2c878071 100644 --- a/src/cli/commands/neural.ts +++ b/src/cli/commands/neural.ts @@ -117,7 +117,7 @@ export const neuralCommand = { await handleNeighborsCommand(neural, argv) break case 'path': - console.log(chalk.yellow('\n⚠️ Semantic path finding coming in v3.21.0')) + console.log(chalk.yellow('\n⚠️ Semantic path finding coming soon')) console.log(chalk.dim('This feature requires implementing graph traversal algorithms')) console.log(chalk.dim('Use "neighbors" and "hierarchy" commands to explore connections')) break @@ -148,7 +148,7 @@ async function promptForAction(): Promise { { name: '🎯 Find semantic clusters', value: 'clusters' }, { name: '🌳 Show item hierarchy', value: 'hierarchy' }, { name: '🕸️ Find semantic neighbors', value: 'neighbors' }, - { name: '🛣️ Find semantic path between items (v3.21.0)', value: 'path', disabled: true }, + { name: '🛣️ Find semantic path between items (coming soon)', value: 'path', disabled: true }, { name: '🚨 Detect outliers', value: 'outliers' }, { name: '📊 Generate visualization data', value: 'visualize' } ] diff --git a/src/cli/commands/storage.ts b/src/cli/commands/storage.ts index 2cdb3df2..54a19044 100644 --- a/src/cli/commands/storage.ts +++ b/src/cli/commands/storage.ts @@ -1,5 +1,5 @@ /** - * 💾 Storage Management Commands - v4.0.0 + * 💾 Storage Management Commands * * Modern interactive CLI for storage lifecycle, cost optimization, and management */ diff --git a/src/cli/commands/utility.ts b/src/cli/commands/utility.ts index 7520f914..45864547 100644 --- a/src/cli/commands/utility.ts +++ b/src/cli/commands/utility.ts @@ -133,7 +133,7 @@ export const utilityCommands = { // removeOrphans and rebuildIndex would require new Brainy APIs if (options.removeOrphans || options.rebuildIndex) { spinner.warn('Advanced cleanup options not yet implemented') - console.log(chalk.yellow('\n⚠️ Advanced cleanup features coming in v3.21.0:')) + console.log(chalk.yellow('\n⚠️ Advanced cleanup features coming soon:')) console.log(chalk.dim(' • --remove-orphans: Remove disconnected items')) console.log(chalk.dim(' • --rebuild-index: Rebuild vector index')) console.log(chalk.dim('\nUse "brainy clean" without options to clear the database')) diff --git a/src/cli/commands/vfs.ts b/src/cli/commands/vfs.ts index 6478aa26..32132670 100644 --- a/src/cli/commands/vfs.ts +++ b/src/cli/commands/vfs.ts @@ -21,7 +21,7 @@ let brainyInstance: Brainy | null = null const getBrainy = async (): Promise => { if (!brainyInstance) { brainyInstance = new Brainy() - await brainyInstance.init() // v5.0.1: Initialize brain (VFS auto-initialized here!) + await brainyInstance.init() // Initialize brain (VFS auto-initialized here!) } return brainyInstance } @@ -52,8 +52,8 @@ export const vfsCommands = { const spinner = ora('Reading file...').start() try { - const brain = await getBrainy() // v5.0.1: Await async getBrainy - // v5.0.1: VFS auto-initialized, no need for vfs.init() + const brain = await getBrainy() // Await async getBrainy + // VFS auto-initialized, no need for vfs.init() const buffer = await brain.vfs.readFile(path, { encoding: options.encoding as any }) diff --git a/src/cli/index.ts b/src/cli/index.ts index 92915f9d..9894830d 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -68,7 +68,7 @@ ${chalk.cyan('Examples:')} $ brainy vfs search "React components" $ brainy vfs similar /code/Button.tsx - ${chalk.dim('# Storage management (v4.0.0)')} + ${chalk.dim('# Storage management')} $ brainy storage status --quota $ brainy storage lifecycle set ${chalk.dim('# Interactive mode')} $ brainy storage cost-estimate @@ -217,11 +217,11 @@ program program .command('path ') - .description('Find semantic path between items (v3.21.0)') + .description('Find semantic path between items') .option('--steps', 'Show step-by-step path') .option('--max-hops ', 'Maximum path length', '5') .action(() => { - console.log(chalk.yellow('\n⚠️ Semantic path finding coming in v3.21.0')) + console.log(chalk.yellow('\n⚠️ Semantic path finding coming soon')) console.log(chalk.dim('This feature requires implementing graph traversal algorithms')) console.log(chalk.dim('Use "brainy neighbors" and "brainy hierarchy" to explore connections')) }) @@ -446,7 +446,7 @@ program vfsCommands.tree(path, options) }) -// ===== Storage Management Commands (v4.0.0) ===== +// ===== Storage Management Commands ===== program .command('storage') @@ -610,7 +610,7 @@ program .option('--iterations ', 'Number of iterations', '100') .action(utilityCommands.benchmark) -// ===== COW Commands (v5.0.0) - Instant Fork & Branching ===== +// ===== COW Commands - Instant Fork & Branching ===== program .command('fork [name]') diff --git a/src/cli/interactive.ts b/src/cli/interactive.ts index 005eea7e..52f4461a 100644 --- a/src/cli/interactive.ts +++ b/src/cli/interactive.ts @@ -625,7 +625,7 @@ export async function promptCommand(): Promise { */ export async function startInteractiveMode() { console.log(chalk.cyan('\n🧠 Brainy Interactive Mode\n')) - console.log(chalk.yellow('Interactive REPL mode coming in v3.20.0\n')) + console.log(chalk.yellow('Interactive REPL mode coming soon\n')) console.log(chalk.dim('Use specific commands for now: brainy add, brainy search, etc.')) process.exit(0) } diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 55bd4fae..f6122d00 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -57,7 +57,7 @@ export type DistanceFunction = (a: Vector, b: Vector) => number /** * Embedding function for converting data to vectors - * v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any` + * Now properly typed - accepts string, string array (batch), or object, no `any` */ export type EmbeddingFunction = (data: string | string[] | Record) => Promise @@ -72,7 +72,7 @@ export interface EmbeddingModel { /** * Embed data into a vector - * v4.0.0: Now properly typed - accepts string, string array (batch), or object, no `any` + * Now properly typed - accepts string, string array (batch), or object, no `any` */ embed(data: string | string[] | Record): Promise @@ -83,9 +83,9 @@ export interface EmbeddingModel { } /** - * HNSW graph noun - Pure vector structure (v4.0.0) + * HNSW graph noun - Pure vector structure * - * v4.0.0 BREAKING CHANGE: metadata field removed + * metadata field removed * - Stores ONLY vector data for optimal memory usage * - Metadata stored separately and combined on retrieval * - 25% memory reduction @ 1B scale (no in-memory metadata) @@ -100,15 +100,15 @@ export interface HNSWNoun { } /** - * Lightweight verb for HNSW index storage - Core relational structure (v4.0.0) + * Lightweight verb for HNSW index storage - Core relational structure * - * Core fields (v3.50.1+): verb/sourceId/targetId are first-class fields + * Core fields: verb/sourceId/targetId are first-class fields * These are NOT metadata - they're the essence of what a verb IS: * - verb: The relationship type (creates, contains, etc.) - needed for routing & display * - sourceId: What entity this verb connects FROM - needed for graph traversal * - targetId: What entity this verb connects TO - needed for graph traversal * - * v4.0.0 BREAKING CHANGE: metadata field removed + * metadata field removed * - Stores ONLY vector + core relational data * - User metadata (weight, custom fields) stored separately * - 10x faster metadata-only updates (skip HNSW rebuild) @@ -134,9 +134,9 @@ export interface HNSWVerb { } /** - * Noun metadata structure (v4.8.0) + * Noun metadata structure * - * v4.8.0 BREAKING CHANGE: Now contains ONLY custom user-defined fields + * Now contains ONLY custom user-defined fields * - Standard fields (confidence, weight, timestamps, etc.) moved to top-level in HNSWNounWithMetadata * - This interface represents custom metadata stored separately from vector data * - Storage format unchanged (backward compatible at storage layer) @@ -162,9 +162,9 @@ export interface NounMetadata { } /** - * Verb metadata structure (v4.8.0) + * Verb metadata structure * - * v4.8.0 BREAKING CHANGE: Now contains ONLY custom user-defined fields + * Now contains ONLY custom user-defined fields * - Standard fields (weight, confidence, timestamps, etc.) moved to top-level in HNSWVerbWithMetadata * - This interface represents custom metadata stored separately from vector + core relational data * - Storage format unchanged (backward compatible at storage layer) @@ -190,9 +190,9 @@ export interface VerbMetadata { } /** - * Combined noun structure for transport/API boundaries (v4.8.0) + * Combined noun structure for transport/API boundaries * - * v4.8.0 BREAKING CHANGE: Standard fields moved to top-level + * Standard fields moved to top-level * - ALL standard fields (confidence, weight, timestamps, etc.) are now at top-level * - metadata contains ONLY custom user-defined fields * - Provides clean, predictable API: entity.confidence always works @@ -230,9 +230,9 @@ export interface HNSWNounWithMetadata { } /** - * Combined verb structure for transport/API boundaries (v4.8.0) + * Combined verb structure for transport/API boundaries * - * v4.8.0 BREAKING CHANGE: Standard fields moved to top-level + * Standard fields moved to top-level * - ALL standard fields (weight, confidence, timestamps, etc.) are now at top-level * - metadata contains ONLY custom user-defined fields * - Provides clean, predictable API: verb.weight always works @@ -308,7 +308,7 @@ export interface HNSWConfig { efSearch: number // Size of the dynamic candidate list during search ml: number // Maximum level useDiskBasedIndex?: boolean // Whether to use disk-based index - maxConcurrentNeighborWrites?: number // Maximum concurrent neighbor updates during insert (v4.10.0+). Default: unlimited (full concurrency) + maxConcurrentNeighborWrites?: number // Maximum concurrent neighbor updates during insert. Default: unlimited (full concurrency) } /** @@ -548,7 +548,7 @@ export interface StatisticsData { } /** - * Change record for getChangesSince (v4.0.0) + * Change record for getChangesSince * Replaces `any[]` with properly typed structure */ export interface Change { @@ -563,27 +563,27 @@ export interface StorageAdapter { init(): Promise /** - * Save noun - Pure HNSW vector data only (v4.0.0) + * Save noun - Pure HNSW vector data only * @param noun Pure HNSW vector data (no metadata) * Note: Use saveNounMetadata() to save metadata separately */ saveNoun(noun: HNSWNoun): Promise /** - * Save noun metadata separately (v4.0.0) + * Save noun metadata separately * @param id Noun ID * @param metadata Noun metadata */ saveNounMetadata(id: string, metadata: NounMetadata): Promise /** - * Delete noun metadata (v4.0.0) + * Delete noun metadata * @param id Noun ID */ deleteNounMetadata(id: string): Promise /** - * Get noun with metadata combined (v4.0.0) + * Get noun with metadata combined * @returns Combined HNSWNounWithMetadata or null */ getNoun(id: string): Promise @@ -622,14 +622,14 @@ export interface StorageAdapter { deleteNoun(id: string): Promise /** - * Save verb - Pure HNSW verb with core fields only (v4.0.0) + * Save verb - Pure HNSW verb with core fields only * @param verb Pure HNSW verb data (vector + core fields, no user metadata) * Note: Use saveVerbMetadata() to save metadata separately */ saveVerb(verb: HNSWVerb): Promise /** - * Get verb with metadata combined (v4.0.0) + * Get verb with metadata combined * @returns Combined HNSWVerbWithMetadata or null */ getVerb(id: string): Promise @@ -686,14 +686,14 @@ export interface StorageAdapter { deleteVerb(id: string): Promise /** - * Save metadata (v4.0.0: now typed) + * Save metadata * @param id Entity ID * @param metadata Typed noun metadata */ saveMetadata(id: string, metadata: NounMetadata): Promise /** - * Get metadata (v4.0.0: now typed) + * Get metadata * @param id Entity ID * @returns Typed noun metadata or null */ @@ -702,26 +702,26 @@ export interface StorageAdapter { /** * Get multiple metadata objects in batches (prevents socket exhaustion) * @param ids Array of IDs to get metadata for - * @returns Promise that resolves to a Map of id -> metadata (v4.0.0: typed) + * @returns Promise that resolves to a Map of id -> metadata */ getMetadataBatch?(ids: string[]): Promise> /** - * Get noun metadata from storage (v4.0.0: now typed) + * Get noun metadata from storage * @param id The ID of the noun * @returns Promise that resolves to the metadata or null if not found */ getNounMetadata(id: string): Promise /** - * Batch get multiple nouns with vectors (v6.2.0 - N+1 fix) + * Batch get multiple nouns with vectors * @param ids Array of noun IDs to fetch * @returns Map of id → HNSWNounWithMetadata (only successful reads included) */ getNounBatch?(ids: string[]): Promise> /** - * Save verb metadata to storage (v4.0.0: now typed) + * Save verb metadata to storage * @param id The ID of the verb * @param metadata The metadata to save * @returns Promise that resolves when the metadata is saved @@ -729,14 +729,14 @@ export interface StorageAdapter { saveVerbMetadata(id: string, metadata: VerbMetadata): Promise /** - * Get verb metadata from storage (v4.0.0: now typed) + * Get verb metadata from storage * @param id The ID of the verb * @returns Promise that resolves to the metadata or null if not found */ getVerbMetadata(id: string): Promise /** - * Batch get multiple verbs (v6.2.0 - N+1 fix) + * Batch get multiple verbs * @param ids Array of verb IDs to fetch * @returns Map of id → HNSWVerbWithMetadata (only successful reads included) */ @@ -745,7 +745,7 @@ export interface StorageAdapter { clear(): Promise /** - * Batch delete multiple objects from storage (v4.0.0) + * Batch delete multiple objects from storage * Efficient deletion of large numbers of entities using cloud provider batch APIs. * Significantly faster and cheaper than individual deletes (up to 1000x speedup). * @@ -853,7 +853,7 @@ export interface StorageAdapter { flushStatisticsToStorage(): Promise /** - * Track field names from a JSON document (v4.0.0: now typed) + * Track field names from a JSON document * @param jsonDocument The JSON document to extract field names from * @param service The service that inserted the data */ @@ -872,7 +872,7 @@ export interface StorageAdapter { getStandardFieldMappings(): Promise>> /** - * Get changes since a specific timestamp (v4.0.0: now typed) + * Get changes since a specific timestamp * @param timestamp The timestamp to get changes since * @param limit Optional limit on the number of changes to return * @returns Promise that resolves to an array of properly typed changes diff --git a/src/embeddings/candle-wasm/src/lib.rs b/src/embeddings/candle-wasm/src/lib.rs index df19f469..4fdb28e4 100644 --- a/src/embeddings/candle-wasm/src/lib.rs +++ b/src/embeddings/candle-wasm/src/lib.rs @@ -26,7 +26,7 @@ use js_sys::{Array, Float32Array}; use tokenizers::Tokenizer; use wasm_bindgen::prelude::*; -// v7.2.0: Model weights are NO LONGER embedded in WASM +// Model weights are NO LONGER embedded in WASM // // Previous design: 90MB WASM with model weights embedded via include_bytes!() // Problem: WASM parsing/compilation took 139+ seconds on throttled CPU (Cloud Run) @@ -78,7 +78,7 @@ impl EmbeddingEngine { /// Load the model and tokenizer from bytes /// - /// v7.2.0: This is now the ONLY way to initialize the engine. + /// This is now the ONLY way to initialize the engine. /// Model weights are no longer embedded in WASM for faster initialization. /// /// # Arguments diff --git a/src/embeddings/wasm/CandleEmbeddingEngine.ts b/src/embeddings/wasm/CandleEmbeddingEngine.ts index abe144fb..7c9ba6b0 100644 --- a/src/embeddings/wasm/CandleEmbeddingEngine.ts +++ b/src/embeddings/wasm/CandleEmbeddingEngine.ts @@ -5,7 +5,7 @@ * Pure Rust/WASM implementation for sentence embeddings. * Works with Bun, Node.js, Bun --compile, and browsers. * - * v7.2.0 Architecture (20x faster initialization): + * Architecture (20x faster initialization): * - WASM file: ~2.4MB (inference code only) * - Model files: ~88MB (loaded separately as raw bytes) * - Init time: ~5-7 seconds (vs 139 seconds with embedded model) @@ -34,7 +34,7 @@ interface CandleWasmModule { } interface CandleEngineInstance { - // v7.2.0: load() is the only initialization method (no more embedded model) + // load() is the only initialization method (no more embedded model) load(modelBytes: Uint8Array, tokenizerBytes: Uint8Array, configBytes: Uint8Array): void is_ready(): boolean embed(text: string): Float32Array @@ -54,7 +54,7 @@ let globalInitPromise: Promise | null = null * Uses the Candle ML framework (Rust/WASM) for inference. * Supports all-MiniLM-L6-v2 with 384-dimensional embeddings. * - * v7.2.0: Model weights are loaded separately from WASM for 20x faster init. + * Model weights are loaded separately from WASM for 20x faster init. * For bun --compile deployments, both WASM and model files are automatically * embedded in the binary - single file deployment still works. */ @@ -104,7 +104,7 @@ export class CandleEmbeddingEngine { /** * Perform actual initialization * - * v7.2.0: WASM and model files are loaded separately for 20x faster init. + * WASM and model files are loaded separately for 20x faster init. * - WASM (~2.4MB): Compiles in ~3-5 seconds * - Model (~88MB): Loads as raw bytes in ~1-2 seconds * - Total: ~5-7 seconds (vs 139 seconds with embedded model) @@ -153,7 +153,7 @@ export class CandleEmbeddingEngine { /** * Load the WASM module * - * v7.2.0: WASM is now only ~2.4MB (inference code only, no model weights). + * WASM is now only ~2.4MB (inference code only, no model weights). * Uses wasmLoader.ts for cross-environment compatibility. */ private async loadWasmModule(): Promise { diff --git a/src/embeddings/wasm/modelLoader.ts b/src/embeddings/wasm/modelLoader.ts index fb2f47d9..45ffc4d3 100644 --- a/src/embeddings/wasm/modelLoader.ts +++ b/src/embeddings/wasm/modelLoader.ts @@ -1,7 +1,7 @@ /** * Universal Model Loader for Candle Embeddings * - * v7.2.0: Model weights are now loaded separately from WASM for faster initialization. + * Model weights are now loaded separately from WASM for faster initialization. * This reduces WASM compilation time from 139 seconds to ~3-5 seconds. * * Loads model files from appropriate source based on environment: diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index b130c291..3c1a4986 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -45,7 +45,7 @@ export class GraphAdjacencyIndex { private lsmTreeVerbsBySource: LSMTree // sourceId -> verbIds private lsmTreeVerbsByTarget: LSMTree // targetId -> verbIds - // v5.7.0: ID-only tracking for billion-scale memory optimization + // ID-only tracking for billion-scale memory optimization // Previous: Map stored full objects (128GB @ 1B verbs) // Now: Set stores only IDs (~100KB @ 1B verbs) = 1,280,000x reduction private verbIdSet = new Set() @@ -117,7 +117,7 @@ export class GraphAdjacencyIndex { /** * Initialize the graph index (lazy initialization) - * v6.3.0: Added defensive auto-rebuild check for verbIdSet consistency + * Added defensive auto-rebuild check for verbIdSet consistency */ private async ensureInitialized(): Promise { if (this.initialized) { @@ -129,7 +129,7 @@ export class GraphAdjacencyIndex { await this.lsmTreeVerbsBySource.init() await this.lsmTreeVerbsByTarget.init() - // v6.3.0: Defensive check - if LSM-trees have data but verbIdSet is empty, + // Defensive check - if LSM-trees have data but verbIdSet is empty, // the index was created without proper rebuild (shouldn't happen with singleton // pattern but protects against edge cases and future refactoring) const lsmTreeSize = this.lsmTreeVerbsBySource.size() @@ -151,7 +151,7 @@ export class GraphAdjacencyIndex { } /** - * Populate verbIdSet from storage without full rebuild (v6.3.0) + * Populate verbIdSet from storage without full rebuild * Lighter weight than full rebuild - only loads verb IDs, not all verb data * @private */ @@ -192,7 +192,7 @@ export class GraphAdjacencyIndex { * Core API - Neighbor lookup with LSM-tree storage * * O(log n) with bloom filter optimization (90% of queries skip disk I/O) - * v5.8.0: Added pagination support for high-degree nodes + * Added pagination support for high-degree nodes * * @param id Entity ID to get neighbors for * @param optionsOrDirection Optional: direction string OR options object @@ -252,7 +252,7 @@ export class GraphAdjacencyIndex { // Convert to array for pagination let result = Array.from(neighbors) - // Apply pagination if requested (v5.8.0) + // Apply pagination if requested if (options?.limit !== undefined || options?.offset !== undefined) { const offset = options.offset || 0 const limit = options.limit !== undefined ? options.limit : result.length @@ -273,8 +273,8 @@ export class GraphAdjacencyIndex { * Get verb IDs by source - Billion-scale optimization for getVerbsBySource * * O(log n) LSM-tree lookup with bloom filter optimization - * v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround) - * v5.8.0: Added pagination support for entities with many relationships + * Filters out deleted verb IDs (tombstone deletion workaround) + * Added pagination support for entities with many relationships * * @param sourceId Source entity ID * @param options Optional configuration @@ -318,7 +318,7 @@ export class GraphAdjacencyIndex { const allIds = verbIds || [] let result = allIds.filter(id => this.verbIdSet.has(id)) - // Apply pagination if requested (v5.8.0) + // Apply pagination if requested if (options?.limit !== undefined || options?.offset !== undefined) { const offset = options.offset || 0 const limit = options.limit !== undefined ? options.limit : result.length @@ -332,8 +332,8 @@ export class GraphAdjacencyIndex { * Get verb IDs by target - Billion-scale optimization for getVerbsByTarget * * O(log n) LSM-tree lookup with bloom filter optimization - * v5.7.1: Filters out deleted verb IDs (tombstone deletion workaround) - * v5.8.0: Added pagination support for popular target entities + * Filters out deleted verb IDs (tombstone deletion workaround) + * Added pagination support for popular target entities * * @param targetId Target entity ID * @param options Optional configuration @@ -377,7 +377,7 @@ export class GraphAdjacencyIndex { const allIds = verbIds || [] let result = allIds.filter(id => this.verbIdSet.has(id)) - // Apply pagination if requested (v5.8.0) + // Apply pagination if requested if (options?.limit !== undefined || options?.offset !== undefined) { const offset = options.offset || 0 const limit = options.limit !== undefined ? options.limit : result.length @@ -414,7 +414,7 @@ export class GraphAdjacencyIndex { } /** - * Batch get multiple verbs with caching (v6.2.0 - N+1 fix) + * Batch get multiple verbs with caching * * **Performance**: Eliminates N+1 pattern for verb loading * - Current: N × getVerbCached() = N × 50ms on GCS = 250ms for 5 verbs @@ -433,7 +433,6 @@ export class GraphAdjacencyIndex { * @param verbIds Array of verb IDs to fetch * @returns Map of verbId → GraphVerb (only successful reads included) * - * @since v6.2.0 */ async getVerbsBatchCached(verbIds: string[]): Promise> { const results = new Map() @@ -514,7 +513,7 @@ export class GraphAdjacencyIndex { const targetStats = this.lsmTreeTarget.getStats() // Note: Exact unique node counts would require full LSM-tree scan - // v5.7.0: Using verbIdSet (ID-only tracking) for memory efficiency + // Using verbIdSet (ID-only tracking) for memory efficiency const uniqueSourceNodes = this.verbIdSet.size const uniqueTargetNodes = this.verbIdSet.size const totalNodes = this.verbIdSet.size @@ -622,13 +621,13 @@ export class GraphAdjacencyIndex { // Clear current index this.verbIdSet.clear() this.totalRelationshipsIndexed = 0 - // v6.2.4: CRITICAL FIX - Clear relationship counts to prevent accumulation + // CRITICAL FIX - Clear relationship counts to prevent accumulation this.relationshipCountsByType.clear() // Note: LSM-trees will be recreated from storage via their own initialization // Verb data will be loaded on-demand via UnifiedCache - // Adaptive loading strategy based on storage type (v4.2.4) + // Adaptive loading strategy based on storage type const storageType = this.storage?.constructor.name || '' const isLocalStorage = storageType === 'FileSystemStorage' || @@ -754,7 +753,7 @@ export class GraphAdjacencyIndex { bytes += targetStats.memTableMemory // Verb ID set (memory-efficient: IDs only, ~8 bytes per ID pointer) - // v5.7.0: Previous verbIndex Map stored full objects (128 bytes each = 128GB @ 1B verbs) + // Previous verbIndex Map stored full objects (128 bytes each = 128GB @ 1B verbs) // Now: verbIdSet stores only IDs (~8 bytes each = ~100KB @ 1B verbs) = 1,280,000x reduction bytes += this.verbIdSet.size * 8 @@ -792,7 +791,7 @@ export class GraphAdjacencyIndex { /** * Flush LSM-tree MemTables to disk - * CRITICAL FIX (v3.43.2): Now public so it can be called from brain.flush() + * CRITICAL FIX: Now public so it can be called from brain.flush() */ async flush(): Promise { if (!this.initialized) { diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index ee1d2ace..42a595d8 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -35,18 +35,18 @@ export class HNSWIndex { private distanceFunction: DistanceFunction private dimension: number | null = null private useParallelization: boolean = true // Whether to use parallelization for performance-critical operations - private storage: BaseStorage | null = null // Storage adapter for HNSW persistence (v3.35.0+) + private storage: BaseStorage | null = null // Storage adapter for HNSW persistence - // Universal memory management (v3.36.0+) + // Universal memory management private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes - // Always-adaptive caching (v3.36.0+) - no "mode" concept, system adapts automatically + // Always-adaptive caching - no "mode" concept, system adapts automatically - // COW (Copy-on-Write) support - v5.0.0 + // COW (Copy-on-Write) support private cowEnabled: boolean = false private cowModifiedNodes: Set = new Set() private cowParent: HNSWIndex | null = null - // v6.2.8: Deferred HNSW persistence for cloud storage performance + // Deferred HNSW persistence for cloud storage performance // In deferred mode, HNSW connections are only persisted on flush/close // This reduces GCS operations from 70 to 2-3 per add() (30-50× faster) private persistMode: 'immediate' | 'deferred' = 'immediate' @@ -86,7 +86,7 @@ export class HNSWIndex { } /** - * v6.2.8: Flush dirty HNSW data to storage + * Flush dirty HNSW data to storage * * In deferred persistence mode, HNSW connections are tracked as dirty but not * immediately persisted. Call flush() to persist all pending changes. @@ -361,6 +361,19 @@ export class HNSWIndex { this.entryPointId = id this.maxLevel = nounLevel this.nouns.set(id, noun) + + // Persist system data for first noun (previously skipped) + if (this.storage && this.persistMode === 'immediate') { + await this.storage.saveHNSWSystem({ + entryPointId: this.entryPointId, + maxLevel: this.maxLevel + }).catch(error => { + console.error('Failed to persist initial HNSW system data:', error) + }) + } else if (this.persistMode === 'deferred') { + this.dirtySystem = true + } + return id } @@ -438,7 +451,7 @@ export class HNSWIndex { ) // Add bidirectional connections - // PERFORMANCE OPTIMIZATION (v4.10.0): Collect all neighbor updates for concurrent execution + // PERFORMANCE OPTIMIZATION: Collect all neighbor updates for concurrent execution const neighborUpdates: Array<{ neighborId: string promise: Promise @@ -467,9 +480,9 @@ export class HNSWIndex { await this.pruneConnections(neighbor, level) } - // Persist updated neighbor HNSW data (v3.35.0+) + // Persist updated neighbor HNSW data // - // v6.2.8: Deferred persistence mode for cloud storage performance + // Deferred persistence mode for cloud storage performance // In deferred mode, we track dirty nodes instead of persisting immediately // This reduces GCS operations from 70 to 2-3 per add() (30-50× faster) if (this.storage && this.persistMode === 'immediate') { @@ -561,8 +574,8 @@ export class HNSWIndex { this.highLevelNodes.get(nounLevel)!.add(id) } - // Persist HNSW graph data to storage (v3.35.0+) - // v6.2.8: Respect persistMode setting + // Persist HNSW graph data to storage + // Respect persistMode setting if (this.storage && this.persistMode === 'immediate') { // IMMEDIATE MODE: Original behavior - persist new entity and system data const connectionsObj: Record = {} @@ -594,7 +607,7 @@ export class HNSWIndex { } /** - * O(1) entry point recovery using highLevelNodes index (v6.2.3). + * O(1) entry point recovery using highLevelNodes index. * At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist. * For tiny indexes with only level 0-1 nodes, any node works as entry point. */ @@ -640,7 +653,7 @@ export class HNSWIndex { } // Start from the entry point - // If entry point is null but nouns exist, attempt O(1) recovery (v6.2.3) + // If entry point is null but nouns exist, attempt O(1) recovery if (!this.entryPointId && this.nouns.size > 0) { const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1() if (recoveredId) { @@ -656,7 +669,7 @@ export class HNSWIndex { let entryPoint = this.nouns.get(this.entryPointId) if (!entryPoint) { - // Entry point ID exists but noun was deleted - O(1) recovery (v6.2.3) + // Entry point ID exists but noun was deleted - O(1) recovery if (this.nouns.size > 0) { const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1() if (recoveredId) { @@ -944,7 +957,7 @@ export class HNSWIndex { /** * Get vector safely (always uses adaptive caching via UnifiedCache) * - * Production-grade adaptive caching (v3.36.0+): + * Production-grade adaptive caching: * - Vector already loaded: Returns immediately (O(1)) * - Vector in cache: Loads from UnifiedCache (O(1) hash lookup) * - Vector on disk: Loads from storage → UnifiedCache (O(disk)) @@ -990,7 +1003,7 @@ export class HNSWIndex { } /** - * Get vector synchronously if available in memory (v3.36.0+) + * Get vector synchronously if available in memory * * Sync fast path optimization: * - Vector in memory: Returns immediately (zero overhead) @@ -1048,7 +1061,7 @@ export class HNSWIndex { } /** - * Calculate distance with sync fast path (v3.36.0+) + * Calculate distance with sync fast path * * Eliminates async overhead when vectors are in memory: * - Sync path: Vector in memory → returns number (zero overhead) @@ -1093,7 +1106,7 @@ export class HNSWIndex { } /** - * Rebuild HNSW index from persisted graph data (v3.35.0+) + * Rebuild HNSW index from persisted graph data * * This is a production-grade O(N) rebuild that restores the pre-computed graph structure * from storage. Much faster than re-building which is O(N log N). @@ -1157,7 +1170,7 @@ export class HNSWIndex { ) } - // Step 4: Adaptive loading strategy based on storage type (v4.2.4) + // Step 4: Adaptive loading strategy based on storage type // FileSystem/Memory/OPFS: Load all at once (avoids repeated getAllShardedFiles() calls) // Cloud (GCS/S3/R2): Use pagination (efficient native cloud APIs) const storageType = this.storage?.constructor.name || '' @@ -1238,7 +1251,7 @@ export class HNSWIndex { prodLog.info(`HNSW: Using cloud pagination strategy (${storageType})`) let hasMore = true - let offset = 0 // v5.7.11: Use offset-based pagination instead of cursor (bug fix for infinite loop) + let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop) while (hasMore) { // Fetch batch of nouns from storage (cast needed as method is not in base interface) @@ -1249,7 +1262,7 @@ export class HNSWIndex { nextCursor?: string } = await (this.storage as any).getNounsWithPagination({ limit: batchSize, - offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored) + offset // Pass offset for proper pagination (previously passed cursor which was ignored) }) // Set total count on first batch @@ -1307,11 +1320,11 @@ export class HNSWIndex { // Check for more data hasMore = result.hasMore - offset += batchSize // v5.7.11: Increment offset for next page + offset += batchSize // Increment offset for next page } } - // Step 5: CRITICAL - Recover entry point if missing (v6.2.3 - O(1)) + // Step 5: CRITICAL - Recover entry point if missing) // This ensures consistency even if getHNSWSystem() returned null if (this.nouns.size > 0 && this.entryPointId === null) { prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup') @@ -1426,7 +1439,7 @@ export class HNSWIndex { } /** - * Get cache performance statistics for monitoring and diagnostics (v3.36.0+) + * Get cache performance statistics for monitoring and diagnostics * * Production-grade monitoring: * - Adaptive caching strategy (preloading vs on-demand) diff --git a/src/hnsw/typeAwareHNSWIndex.ts b/src/hnsw/typeAwareHNSWIndex.ts index d5453449..4f57f448 100644 --- a/src/hnsw/typeAwareHNSWIndex.ts +++ b/src/hnsw/typeAwareHNSWIndex.ts @@ -118,7 +118,7 @@ export class TypeAwareHNSWIndex { } /** - * v6.2.8: Flush dirty HNSW data to storage for all type-specific indexes + * Flush dirty HNSW data to storage for all type-specific indexes * * In deferred persistence mode, HNSW connections are tracked as dirty but not * immediately persisted. Call flush() to persist all pending changes across @@ -502,7 +502,7 @@ export class TypeAwareHNSWIndex { // Load ALL nouns ONCE and route to correct type indexes // This is O(N) instead of O(42*N) from the previous parallel approach - let offset = 0 // v5.7.11: Use offset-based pagination instead of cursor (bug fix for infinite loop) + let offset = 0 // Use offset-based pagination instead of cursor (bug fix for infinite loop) let hasMore = true let totalLoaded = 0 const loadedByType = new Map() @@ -515,13 +515,13 @@ export class TypeAwareHNSWIndex { totalCount?: number } = await (this.storage as any).getNounsWithPagination({ limit: batchSize, - offset // v5.7.11: Pass offset for proper pagination (previously passed cursor which was ignored) + offset // Pass offset for proper pagination (previously passed cursor which was ignored) }) // Route each noun to its type index for (const nounData of result.items) { try { - // v7.3.0: Use 'type' property from HNSWNounWithMetadata (not 'nounType') + // Use 'type' property from HNSWNounWithMetadata (not 'nounType') // Previously accessed wrong property, causing N+1 metadata fetches // getNounsWithPagination already includes type in response const nounType = nounData.type @@ -578,7 +578,7 @@ export class TypeAwareHNSWIndex { } hasMore = result.hasMore - offset += batchSize // v5.7.11: Increment offset for next page + offset += batchSize // Increment offset for next page // Progress logging if (totalLoaded % 1000 === 0) { @@ -593,12 +593,26 @@ export class TypeAwareHNSWIndex { let entryPointId: string | null = null for (const [id, noun] of (index as any).nouns.entries()) { - if (noun.level > maxLevel) { + if (entryPointId === null || noun.level > maxLevel) { maxLevel = noun.level entryPointId = id } } + // Recovery: if still null after loop but nouns exist + if ((index as any).nouns.size > 0 && !entryPointId) { + const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1() + entryPointId = recoveredId + maxLevel = recoveredLevel + } + + // Validation: if entry point doesn't exist in loaded nouns + if (entryPointId && !(index as any).nouns.has(entryPointId)) { + const { id: recoveredId, level: recoveredLevel } = (index as any).recoverEntryPointO1() + entryPointId = recoveredId + maxLevel = recoveredLevel + } + ;(index as any).entryPointId = entryPointId ;(index as any).maxLevel = maxLevel diff --git a/src/import/FormatDetector.ts b/src/import/FormatDetector.ts index b4919420..25d99c2b 100644 --- a/src/import/FormatDetector.ts +++ b/src/import/FormatDetector.ts @@ -111,7 +111,7 @@ export class FormatDetector { return 'docx' } - // Images (v5.2.0: ImageHandler support) + // Images (ImageHandler support) if (mimeType.startsWith('image/')) { return 'image' } @@ -134,7 +134,7 @@ export class FormatDetector { } } - // YAML detection (v4.2.0) + // YAML detection if (this.looksLikeYAML(trimmed)) { return { format: 'yaml', @@ -206,7 +206,7 @@ export class FormatDetector { } } - // Image formats (v5.2.0) + // Image formats // JPEG: FF D8 FF if (buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) { return { @@ -384,7 +384,7 @@ export class FormatDetector { /** * Check if content looks like YAML - * v4.2.0: Added YAML detection + * Added YAML detection */ private looksLikeYAML(content: string): boolean { const lines = content.split('\n').filter(l => l.trim()).slice(0, 20) diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index 9ed7cad5..e80a2ff1 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -37,10 +37,10 @@ export interface ImportSource { /** Optional filename hint */ filename?: string - /** HTTP headers for URL imports (v4.2.0) */ + /** HTTP headers for URL imports */ headers?: Record - /** Basic authentication for URL imports (v4.2.0) */ + /** Basic authentication for URL imports */ auth?: { username: string password: string @@ -93,7 +93,7 @@ export interface ValidImportOptions { /** Create relationships in knowledge graph */ createRelationships?: boolean - /** Create provenance relationships (document → entity) [v4.9.0] */ + /** Create provenance relationships (document → entity) */ createProvenanceLinks?: boolean /** Preserve source file in VFS */ @@ -144,7 +144,7 @@ export interface ValidImportOptions { customMetadata?: Record /** - * Progress callback for tracking import progress (v4.2.0+) + * Progress callback for tracking import progress * * **Streaming Architecture** (always enabled): * - Indexes are flushed periodically during import (adaptive intervals) @@ -227,21 +227,21 @@ export type ImportOptions = ValidImportOptions & DeprecatedImportOptions export interface ImportProgress { stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete' - /** Phase of import - extraction or relationship building (v3.49.0) */ + /** Phase of import - extraction or relationship building */ phase?: 'extraction' | 'relationships' message: string processed?: number - /** Alias for processed, used in relationship phase (v3.49.0) */ + /** Alias for processed, used in relationship phase */ current?: number total?: number entities?: number relationships?: number - /** Rows per second (v3.38.0) */ + /** Rows per second */ throughput?: number - /** Estimated time remaining in ms (v3.38.0) */ + /** Estimated time remaining in ms */ eta?: number /** - * Whether data is queryable at this point (v4.2.0+) + * Whether data is queryable at this point * * When true, indexes have been flushed and queries will return up-to-date results. * When false, data exists in storage but indexes may not be current (queries may be slower/incomplete). @@ -357,7 +357,7 @@ export class ImportCoordinator { /** * Import from any source with auto-detection - * v4.2.0: Now supports URL imports with authentication + * Now supports URL imports with authentication */ async import( source: Buffer | string | object | ImportSource, @@ -365,10 +365,10 @@ export class ImportCoordinator { ): Promise { const startTime = Date.now() - // Validate options (v4.0.0+: Reject deprecated v3.x options) + // Validate options (Reject deprecated options) this.validateOptions(options) - // Normalize source (v4.2.0: handles URL fetching) + // Normalize source (handles URL fetching) const normalizedSource = await this.normalizeSource(source, options.format) // Report detection stage @@ -387,10 +387,10 @@ export class ImportCoordinator { } // Set defaults early (needed for tracking context) - // CRITICAL FIX (v4.3.2): Spread options FIRST, then apply defaults + // CRITICAL FIX: Spread options FIRST, then apply defaults // Previously: ...options at the end overwrote normalized defaults with undefined // Now: Defaults properly override undefined values - // v4.4.0: Enable AI features by default for smarter imports + // Enable AI features by default for smarter imports const opts = { ...options, // Spread first to get all options vfsPath: options.vfsPath || `/imports/${Date.now()}`, @@ -399,13 +399,13 @@ export class ImportCoordinator { createRelationships: options.createRelationships !== false, preserveSource: options.preserveSource !== false, enableDeduplication: options.enableDeduplication !== false, - enableNeuralExtraction: options.enableNeuralExtraction !== false, // v4.4.0: Default true - enableRelationshipInference: options.enableRelationshipInference !== false, // v4.4.0: Default true + enableNeuralExtraction: options.enableNeuralExtraction !== false, // Default true + enableRelationshipInference: options.enableRelationshipInference !== false, // Default true enableConceptExtraction: options.enableConceptExtraction !== false, // Already defaults to true deduplicationThreshold: options.deduplicationThreshold || 0.85 } - // Generate tracking context (v4.10.0+: Unified import/project tracking) + // Generate tracking context (Unified import/project tracking) const importId = options.importId || uuidv4() const projectId = options.projectId || this.deriveProjectId(opts.vfsPath) const trackingContext: TrackingContext = { @@ -441,13 +441,13 @@ export class ImportCoordinator { groupBy: opts.groupBy, customGrouping: opts.customGrouping, preserveSource: opts.preserveSource, - // v5.1.2: Fix sourceBuffer for file paths - type is 'path' not 'buffer' from normalizeSource() + // Fix sourceBuffer for file paths - type is 'path' not 'buffer' from normalizeSource() sourceBuffer: Buffer.isBuffer(normalizedSource.data) ? normalizedSource.data as Buffer : undefined, sourceFilename: normalizedSource.filename || `import.${detection.format}`, createRelationshipFile: true, createMetadataFile: true, - trackingContext, // v4.10.0: Pass tracking metadata to VFS - // v4.11.1: Pass progress callback for VFS creation updates + trackingContext, // Pass tracking metadata to VFS + // Pass progress callback for VFS creation updates onProgress: (vfsProgress) => { options.onProgress?.({ stage: 'storing-vfs', @@ -473,7 +473,7 @@ export class ImportCoordinator { sourceFilename: normalizedSource.filename || `import.${detection.format}`, format: detection.format }, - trackingContext // v4.10.0: Pass tracking metadata to graph creation + trackingContext // Pass tracking metadata to graph creation ) // Report complete @@ -520,7 +520,7 @@ export class ImportCoordinator { ) } - // CRITICAL FIX (v3.43.2): Auto-flush all indexes before returning + // CRITICAL FIX: Auto-flush all indexes before returning // Ensures imported data survives server restarts // Bug #5: Import data was only in memory, lost on restart options.onProgress?.({ @@ -535,7 +535,7 @@ export class ImportCoordinator { /** * Normalize source to ImportSource - * v4.2.0: Now async to support URL fetching + * Now async to support URL fetching */ private async normalizeSource( source: Buffer | string | object | ImportSource, @@ -616,7 +616,7 @@ export class ImportCoordinator { /** * Fetch content from URL - * v4.2.0: Supports authentication and custom headers + * Supports authentication and custom headers */ private async fetchUrl(source: ImportSource): Promise { const url = typeof source.data === 'string' ? source.data : String(source.data) @@ -722,7 +722,7 @@ export class ImportCoordinator { format: SupportedFormat, options: ImportOptions ): Promise { - // v5.2.0: Check if IntelligentImportAugmentation already extracted data + // Check if IntelligentImportAugmentation already extracted data if ((options as any)._intelligentImport && (options as any)._extractedData) { const extractedData = (options as any)._extractedData // Convert extracted data to ExtractedRow format @@ -760,7 +760,7 @@ export class ImportCoordinator { enableConceptExtraction: options.enableConceptExtraction !== false, confidenceThreshold: options.confidenceThreshold || 0.6, onProgress: (stats: any) => { - // Enhanced progress reporting (v3.38.0) with throughput and ETA + // Enhanced progress reporting with throughput and ETA const message = stats.throughput ? `Extracting entities from ${format} (${stats.throughput} rows/sec, ETA: ${Math.round(stats.eta / 1000)}s)...` : `Extracting entities from ${format}...` @@ -825,7 +825,7 @@ export class ImportCoordinator { return await this.docxImporter.extract(docxBuffer, extractOptions) case 'image': - // v5.2.0: Images are handled by IntelligentImportAugmentation + // Images are handled by IntelligentImportAugmentation // If we reach here, augmentation didn't process it - return minimal result const imageName = source.filename || 'image' const imageId = `image-${Date.now()}` @@ -867,7 +867,7 @@ export class ImportCoordinator { /** * Create entities and relationships in knowledge graph - * v4.9.0: Added sourceInfo parameter for document entity creation + * Added sourceInfo parameter for document entity creation */ private async createGraphEntities( extractionResult: any, @@ -877,7 +877,7 @@ export class ImportCoordinator { sourceFilename: string format: string }, - trackingContext?: TrackingContext // v4.10.0: Import/project tracking + trackingContext?: TrackingContext // Import/project tracking ): Promise<{ entities: Array<{ id: string; name: string; type: NounType; vfsPath?: string; metadata?: Record }> relationships: Array<{ id: string; from: string; to: string; type: VerbType }> @@ -891,7 +891,7 @@ export class ImportCoordinator { let mergedCount = 0 let newCount = 0 - // CRITICAL FIX (v4.3.2): Default to true when undefined + // CRITICAL FIX: Default to true when undefined // Previously: if (!options.createEntities) treated undefined as false // Now: Only skip when explicitly set to false if (options.createEntities === false) { @@ -908,7 +908,7 @@ export class ImportCoordinator { // Extract rows/sections/entities from result (unified across formats) const rows = extractionResult.rows || extractionResult.sections || extractionResult.entities || [] - // Progressive flush interval - adjusts based on current count (v4.2.0+) + // Progressive flush interval - adjusts based on current count // Starts at 100, increases to 1000 at 1K entities, then 5000 at 10K // This works for both known totals (files) and unknown totals (streaming APIs) let currentFlushInterval = 100 // Start with frequent updates for better UX @@ -938,7 +938,7 @@ export class ImportCoordinator { } // ============================================ - // v4.9.0: Create document entity for import source + // Create document entity for import source // ============================================ let documentEntityId: string | null = null let provenanceCount = 0 @@ -957,7 +957,7 @@ export class ImportCoordinator { vfsPath: vfsResult.rootPath, totalRows: rows.length, byType: this.countByType(rows), - // v4.10.0: Import tracking metadata + // Import tracking metadata ...(trackingContext && { importIds: [trackingContext.importId], projectId: trackingContext.projectId, @@ -973,7 +973,7 @@ export class ImportCoordinator { } // ============================================ - // v4.11.0: Batch entity creation using addMany() + // Batch entity creation using addMany() // Replaces entity-by-entity loop for 10-100x performance improvement on cloud storage // ============================================ @@ -1038,7 +1038,7 @@ export class ImportCoordinator { name: entity.name, type: entity.type, vfsPath: vfsFile?.path, - metadata: entity.metadata // v5.2.0: Include metadata in return (for ImageHandler, etc) + metadata: entity.metadata // Include metadata in return (for ImageHandler, etc) }) newCount++ } @@ -1090,7 +1090,7 @@ export class ImportCoordinator { const importSource = vfsResult.rootPath let entityId: string - // v5.7.0: No deduplication during import (12-24x speedup) + // No deduplication during import (12-24x speedup) // Background deduplication runs 5 minutes after import completes entityId = await this.brain.add({ data: entity.description || entity.name, @@ -1101,7 +1101,7 @@ export class ImportCoordinator { confidence: entity.confidence, vfsPath: vfsFile?.path, importedFrom: 'import-coordinator', - // v4.10.0: Import tracking metadata + // Import tracking metadata ...(trackingContext && { importId: trackingContext.importId, // Used for background dedup importIds: [trackingContext.importId], @@ -1126,11 +1126,11 @@ export class ImportCoordinator { name: entity.name, type: entity.type, vfsPath: vfsFile?.path, - metadata: entity.metadata // v5.2.0: Include metadata in return (for ImageHandler, etc) + metadata: entity.metadata // Include metadata in return (for ImageHandler, etc) }) // ============================================ - // v4.9.0: Create provenance relationship (document → entity) + // Create provenance relationship (document → entity) // ============================================ if (documentEntityId && options.createProvenanceLinks !== false) { await this.brain.relate({ @@ -1144,7 +1144,7 @@ export class ImportCoordinator { rowNumber: row.rowNumber, extractedAt: Date.now(), format: sourceInfo?.format, - // v4.10.0: Import tracking metadata + // Import tracking metadata ...(trackingContext && { importIds: [trackingContext.importId], projectId: trackingContext.projectId, @@ -1161,7 +1161,7 @@ export class ImportCoordinator { if (options.createRelationships && row.relationships) { for (const rel of row.relationships) { try { - // CRITICAL FIX (v3.43.2): Prevent infinite placeholder creation loop + // CRITICAL FIX: Prevent infinite placeholder creation loop // Find or create target entity using EXACT matching only let targetEntityId: string | undefined @@ -1195,7 +1195,7 @@ export class ImportCoordinator { name: rel.to, placeholder: true, inferredFrom: entity.name, - // v4.10.0: Import tracking metadata + // Import tracking metadata ...(trackingContext && { importIds: [trackingContext.importId], projectId: trackingContext.projectId, @@ -1221,11 +1221,11 @@ export class ImportCoordinator { from: entityId, to: targetEntityId, type: rel.type, - confidence: rel.confidence, // v4.2.0: Top-level field - weight: rel.weight || 1.0, // v4.2.0: Top-level field + confidence: rel.confidence, // Top-level field + weight: rel.weight || 1.0, // Top-level field metadata: { evidence: rel.evidence, - // v4.10.0: Import tracking metadata (will be merged in batch creation) + // Import tracking metadata (will be merged in batch creation) ...(trackingContext && { importIds: [trackingContext.importId], projectId: trackingContext.projectId, @@ -1242,7 +1242,7 @@ export class ImportCoordinator { } } - // Streaming import: Progressive flush with dynamic interval adjustment (v4.2.0+) + // Streaming import: Progressive flush with dynamic interval adjustment entitiesSinceFlush++ if (entitiesSinceFlush >= currentFlushInterval) { @@ -1307,7 +1307,7 @@ export class ImportCoordinator { } // Batch create all relationships using brain.relateMany() for performance - // v4.9.0: Enhanced with type-based inference and semantic metadata + // Enhanced with type-based inference and semantic metadata if (options.createRelationships && relationships.length > 0) { try { const relationshipParams = relationships.map(rel => { @@ -1331,7 +1331,7 @@ export class ImportCoordinator { type: verbType, // Enhanced type metadata: { ...((rel as any).metadata || {}), - relationshipType: 'semantic', // v4.9.0: Distinguish from VFS/provenance + relationshipType: 'semantic', // Distinguish from VFS/provenance inferredType: verbType !== rel.type, // Track if type was enhanced originalType: rel.type } @@ -1369,7 +1369,7 @@ export class ImportCoordinator { } } - // v5.7.0: Schedule background deduplication (debounced 5 minutes) + // Schedule background deduplication (debounced 5 minutes) if (trackingContext && trackingContext.importId) { this.backgroundDedup.scheduleDedup(trackingContext.importId) } @@ -1457,7 +1457,7 @@ export class ImportCoordinator { } } - // YAML: entities -> rows (v4.2.0) + // YAML: entities -> rows if (format === 'yaml') { const rows = result.entities.map((entity: any) => ({ entity, @@ -1477,7 +1477,7 @@ export class ImportCoordinator { } } - // DOCX: entities -> rows (v4.2.0) + // DOCX: entities -> rows if (format === 'docx') { const rows = result.entities.map((entity: any) => ({ entity, @@ -1502,7 +1502,7 @@ export class ImportCoordinator { } /** - * Validate options and reject deprecated v3.x options (v4.0.0+) + * Validate options and reject deprecated v3.x options * Throws clear errors with migration guidance */ private validateOptions(options: any): void { @@ -1657,7 +1657,7 @@ ${optionDetails} } /** - * Get progressive flush interval based on CURRENT entity count (v4.2.0+) + * Get progressive flush interval based on CURRENT entity count * * Unlike adaptive intervals (which require knowing total count upfront), * progressive intervals adjust dynamically as import proceeds. @@ -1699,7 +1699,7 @@ ${optionDetails} /** * Infer relationship type based on entity types and context - * v4.9.0: Semantic relationship enhancement + * Semantic relationship enhancement * * @param sourceType - Type of source entity * @param targetType - Type of target entity @@ -1769,7 +1769,7 @@ ${optionDetails} /** * Count entities by type for document metadata - * v4.9.0: Used for document entity statistics + * Used for document entity statistics * * @param rows - Extracted rows from import * @returns Record of entity type counts diff --git a/src/importers/SmartCSVImporter.ts b/src/importers/SmartCSVImporter.ts index 576bb979..f11832e6 100644 --- a/src/importers/SmartCSVImporter.ts +++ b/src/importers/SmartCSVImporter.ts @@ -42,17 +42,17 @@ export interface SmartCSVOptions extends FormatHandlerOptions { csvDelimiter?: string csvHeaders?: boolean - /** Progress callback (v3.39.0: Enhanced with performance metrics) */ + /** Progress callback (Enhanced with performance metrics) */ onProgress?: (stats: { processed: number total: number entities: number relationships: number - /** Rows per second (v3.39.0) */ + /** Rows per second */ throughput?: number - /** Estimated time remaining in ms (v3.39.0) */ + /** Estimated time remaining in ms */ eta?: number - /** Current phase (v3.39.0) */ + /** Current phase */ phase?: string }) => void } @@ -169,7 +169,7 @@ export class SmartCSVImporter { } // Parse CSV using existing handler - // v4.5.0: Pass progress hooks to handler for file parsing progress + // Pass progress hooks to handler for file parsing progress const processedData = await this.csvHandler.process(buffer, { ...options, csvDelimiter: opts.csvDelimiter, @@ -217,7 +217,7 @@ export class SmartCSVImporter { // Detect column names const columns = this.detectColumns(rows[0], opts) - // Process each row with BATCHED PARALLEL PROCESSING (v3.39.0) + // Process each row with BATCHED PARALLEL PROCESSING const extractedRows: ExtractedRow[] = [] const entityMap = new Map() const stats = { @@ -375,7 +375,7 @@ export class SmartCSVImporter { total: rows.length, entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0), relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0), - // Additional performance metrics (v3.39.0) + // Additional performance metrics throughput: Math.round(rowsPerSecond * 10) / 10, eta: Math.round(estimatedTimeRemaining), phase: 'extracting' diff --git a/src/importers/SmartDOCXImporter.ts b/src/importers/SmartDOCXImporter.ts index 02a017c9..59521274 100644 --- a/src/importers/SmartDOCXImporter.ts +++ b/src/importers/SmartDOCXImporter.ts @@ -9,7 +9,7 @@ * - NaturalLanguageProcessor for relationship inference * - Hierarchical relationship creation based on heading hierarchy * - * v4.2.0: New format handler + * New format handler * NO MOCKS - Production-ready implementation */ @@ -187,7 +187,7 @@ export class SmartDOCXImporter { await this.init() } - // v4.5.0: Report parsing start + // Report parsing start options.onProgress?.({ processed: 0, entities: 0, @@ -200,7 +200,7 @@ export class SmartDOCXImporter { // Extract HTML for structure analysis (headings, tables) const htmlResult = await mammoth.convertToHtml({ buffer }) - // v4.5.0: Report parsing complete + // Report parsing complete options.onProgress?.({ processed: 0, entities: 0, diff --git a/src/importers/SmartExcelImporter.ts b/src/importers/SmartExcelImporter.ts index e5d4781c..479a6b75 100644 --- a/src/importers/SmartExcelImporter.ts +++ b/src/importers/SmartExcelImporter.ts @@ -36,17 +36,17 @@ export interface SmartExcelOptions extends FormatHandlerOptions { typeColumn?: string // e.g., "Type", "Category" relatedColumn?: string // e.g., "Related Terms", "See Also" - /** Progress callback (v3.38.0: Enhanced with performance metrics) */ + /** Progress callback (Enhanced with performance metrics) */ onProgress?: (stats: { processed: number total: number entities: number relationships: number - /** Rows per second (v3.38.0) */ + /** Rows per second */ throughput?: number - /** Estimated time remaining in ms (v3.38.0) */ + /** Estimated time remaining in ms */ eta?: number - /** Current phase (v3.38.0) */ + /** Current phase */ phase?: string }) => void } @@ -111,7 +111,7 @@ export interface SmartExcelResult { } } - /** Sheet-specific data for VFS extraction (v4.2.0) */ + /** Sheet-specific data for VFS extraction */ sheets?: Array<{ name: string rows: ExtractedRow[] @@ -161,7 +161,7 @@ export class SmartExcelImporter { const opts = { enableNeuralExtraction: true, enableRelationshipInference: true, - // CONCEPT EXTRACTION PRODUCTION-READY (v3.33.0+): + // CONCEPT EXTRACTION PRODUCTION-READY: // Type embeddings are now pre-computed at build time - zero runtime cost! // All 42 noun types + 127 verb types instantly available // @@ -184,7 +184,7 @@ export class SmartExcelImporter { } // Parse Excel using existing handler - // v4.5.0: Pass progress hooks to handler for file parsing progress + // Pass progress hooks to handler for file parsing progress const processedData = await this.excelHandler.process(buffer, { ...options, totalBytes: buffer.length, @@ -227,7 +227,7 @@ export class SmartExcelImporter { return this.emptyResult(startTime) } - // CRITICAL FIX (v4.8.6): Detect columns per-sheet, not globally + // CRITICAL FIX: Detect columns per-sheet, not globally // Different sheets may have different column structures (Term vs Name, etc.) // Group rows by sheet and detect columns for each sheet separately const rowsBySheet = new Map() @@ -247,7 +247,7 @@ export class SmartExcelImporter { } } - // Process each row with BATCHED PARALLEL PROCESSING (v3.38.0) + // Process each row with BATCHED PARALLEL PROCESSING const extractedRows: ExtractedRow[] = [] const entityMap = new Map() const stats = { @@ -269,7 +269,7 @@ export class SmartExcelImporter { chunk.map(async (row, chunkIndex) => { const i = chunkStart + chunkIndex - // CRITICAL FIX (v4.8.6): Use sheet-specific column mapping + // CRITICAL FIX: Use sheet-specific column mapping const sheet = row._sheet || 'default' const columns = columnsBySheet.get(sheet) || this.detectColumns(row, opts) @@ -353,7 +353,7 @@ export class SmartExcelImporter { } // ============================================ - // v4.9.0: Enhanced column-based relationship detection + // Enhanced column-based relationship detection // ============================================ // Parse explicit "Related Terms" column if (relatedTerms) { @@ -379,7 +379,7 @@ export class SmartExcelImporter { } } - // v4.9.0: Check for additional relationship-indicating columns + // Check for additional relationship-indicating columns // Expanded patterns for various relationship types const relationshipColumnPatterns = [ { pattern: /^(location|home|lives in|resides|dwelling|place)$/i, defaultType: VerbType.LocatedAt }, @@ -485,14 +485,14 @@ export class SmartExcelImporter { total: rows.length, entities: extractedRows.reduce((sum, row) => sum + 1 + row.relatedEntities.length, 0), relationships: extractedRows.reduce((sum, row) => sum + row.relationships.length, 0), - // Additional performance metrics (v3.38.0) + // Additional performance metrics throughput: Math.round(rowsPerSecond * 10) / 10, eta: Math.round(estimatedTimeRemaining), phase: 'extracting' } as any) } - // Group rows by sheet for VFS extraction (v4.2.0) + // Group rows by sheet for VFS extraction const sheetGroups = new Map() extractedRows.forEach((extractedRow, index) => { const originalRow = rows[index] diff --git a/src/importers/SmartJSONImporter.ts b/src/importers/SmartJSONImporter.ts index fd5a79e8..10e3518a 100644 --- a/src/importers/SmartJSONImporter.ts +++ b/src/importers/SmartJSONImporter.ts @@ -167,7 +167,7 @@ export class SmartJSONImporter { ...options } - // v4.5.0: Report parsing start + // Report parsing start opts.onProgress({ processed: 0, entities: 0, @@ -186,7 +186,7 @@ export class SmartJSONImporter { jsonData = data } - // v4.5.0: Report parsing complete, starting traversal + // Report parsing complete, starting traversal opts.onProgress({ processed: 0, entities: 0, @@ -228,7 +228,7 @@ export class SmartJSONImporter { } ) - // v4.5.0: Report completion + // Report completion opts.onProgress({ processed: nodesProcessed, entities: entities.length, diff --git a/src/importers/SmartMarkdownImporter.ts b/src/importers/SmartMarkdownImporter.ts index 44471f65..27ea20a9 100644 --- a/src/importers/SmartMarkdownImporter.ts +++ b/src/importers/SmartMarkdownImporter.ts @@ -174,7 +174,7 @@ export class SmartMarkdownImporter { ...options } - // v4.5.0: Report parsing start + // Report parsing start opts.onProgress({ processed: 0, total: 0, @@ -185,7 +185,7 @@ export class SmartMarkdownImporter { // Parse markdown into sections const parsedSections = this.parseMarkdown(markdown, opts) - // v4.5.0: Report parsing complete + // Report parsing complete opts.onProgress({ processed: 0, total: parsedSections.length, @@ -218,7 +218,7 @@ export class SmartMarkdownImporter { }) } - // v4.5.0: Report completion + // Report completion const totalEntities = sections.reduce((sum, s) => sum + s.entities.length, 0) const totalRelationships = sections.reduce((sum, s) => sum + s.relationships.length, 0) opts.onProgress({ diff --git a/src/importers/SmartPDFImporter.ts b/src/importers/SmartPDFImporter.ts index 18dc716c..3cec79fb 100644 --- a/src/importers/SmartPDFImporter.ts +++ b/src/importers/SmartPDFImporter.ts @@ -40,17 +40,17 @@ export interface SmartPDFOptions extends FormatHandlerOptions { /** Group by page or full document */ groupBy?: 'page' | 'document' - /** Progress callback (v3.39.0: Enhanced with performance metrics) */ + /** Progress callback (Enhanced with performance metrics) */ onProgress?: (stats: { processed: number total: number entities: number relationships: number - /** Sections per second (v3.39.0) */ + /** Sections per second */ throughput?: number - /** Estimated time remaining in ms (v3.39.0) */ + /** Estimated time remaining in ms */ eta?: number - /** Current phase (v3.39.0) */ + /** Current phase */ phase?: string }) => void } @@ -181,7 +181,7 @@ export class SmartPDFImporter { } // Parse PDF using existing handler - // v4.5.0: Pass progress hooks to handler for file parsing progress + // Pass progress hooks to handler for file parsing progress const processedData = await this.pdfHandler.process(buffer, { ...options, totalBytes: buffer.length, @@ -228,7 +228,7 @@ export class SmartPDFImporter { // Group data by page or combine into single document const grouped = this.groupData(data, opts) - // Process each group with BATCHED PARALLEL PROCESSING (v3.39.0) + // Process each group with BATCHED PARALLEL PROCESSING const sections: ExtractedSection[] = [] const entityMap = new Map() const stats = { @@ -270,7 +270,7 @@ export class SmartPDFImporter { total: totalGroups, entities: sections.reduce((sum, s) => sum + s.entities.length, 0), relationships: sections.reduce((sum, s) => sum + s.relationships.length, 0), - // Additional performance metrics (v3.39.0) + // Additional performance metrics throughput: Math.round(sectionsPerSecond * 10) / 10, eta: Math.round(estimatedTimeRemaining), phase: 'extracting' @@ -367,7 +367,7 @@ export class SmartPDFImporter { const combinedText = texts.join('\n\n') - // Parallel extraction: entities AND concepts at the same time (v3.39.0) + // Parallel extraction: entities AND concepts at the same time const [extractedEntities, concepts] = await Promise.all([ // Extract entities if enabled options.enableNeuralExtraction && combinedText.length > 0 diff --git a/src/importers/SmartYAMLImporter.ts b/src/importers/SmartYAMLImporter.ts index 73d6fc7d..72d45e9a 100644 --- a/src/importers/SmartYAMLImporter.ts +++ b/src/importers/SmartYAMLImporter.ts @@ -8,7 +8,7 @@ * - NaturalLanguageProcessor for relationship inference * - Hierarchical relationship creation (parent-child, contains, etc.) * - * v4.2.0: New format handler + * New format handler * NO MOCKS - Production-ready implementation */ @@ -159,7 +159,7 @@ export class SmartYAMLImporter { ): Promise { const startTime = Date.now() - // v4.5.0: Report parsing start + // Report parsing start options.onProgress?.({ processed: 0, entities: 0, @@ -178,7 +178,7 @@ export class SmartYAMLImporter { throw new Error(`Failed to parse YAML: ${error.message}`) } - // v4.5.0: Report parsing complete + // Report parsing complete options.onProgress?.({ processed: 0, entities: 0, diff --git a/src/importers/VFSStructureGenerator.ts b/src/importers/VFSStructureGenerator.ts index 06c7acf5..46b74cf5 100644 --- a/src/importers/VFSStructureGenerator.ts +++ b/src/importers/VFSStructureGenerator.ts @@ -40,10 +40,10 @@ export interface VFSStructureOptions { /** Create metadata file */ createMetadataFile?: boolean - /** Import tracking context (v4.10.0) */ + /** Import tracking context */ trackingContext?: TrackingContext - /** Progress callback (v4.11.1) - Reports VFS creation progress */ + /** Progress callback - Reports VFS creation progress */ onProgress?: (progress: { stage: 'directories' | 'entities' | 'metadata' message: string @@ -98,7 +98,7 @@ export class VFSStructureGenerator { // Get brain's cached VFS instance (creates if doesn't exist) this.vfs = this.brain.vfs - // CRITICAL FIX (v4.10.2): Always call vfs.init() explicitly + // CRITICAL FIX: Always call vfs.init() explicitly // The previous code tried to check if initialized via stat('/') but this was unreliable // vfs.init() is idempotent, so calling it multiple times is safe await this.vfs.init() @@ -123,7 +123,7 @@ export class VFSStructureGenerator { // Ensure VFS is initialized await this.init() - // v4.11.1: Calculate total operations for progress tracking + // Calculate total operations for progress tracking const groups = this.groupEntities(importResult, options) const totalEntities = Array.from(groups.values()).reduce((sum, entities) => sum + entities.length, 0) const totalOperations = @@ -161,7 +161,7 @@ export class VFSStructureGenerator { try { await this.vfs.mkdir(options.rootPath, { recursive: true, - metadata: trackingMetadata // v4.10.0: Add tracking metadata + metadata: trackingMetadata // Add tracking metadata }) result.directories.push(options.rootPath) result.operations++ @@ -179,7 +179,7 @@ export class VFSStructureGenerator { if (options.preserveSource && options.sourceBuffer && options.sourceFilename) { const sourcePath = `${options.rootPath}/_source${this.getExtension(options.sourceFilename)}` await this.vfs.writeFile(sourcePath, options.sourceBuffer, { - metadata: trackingMetadata // v4.10.0: Add tracking metadata + metadata: trackingMetadata // Add tracking metadata }) result.files.push({ path: sourcePath, @@ -200,7 +200,7 @@ export class VFSStructureGenerator { try { await this.vfs.mkdir(groupPath, { recursive: true, - metadata: trackingMetadata // v4.10.0: Add tracking metadata + metadata: trackingMetadata // Add tracking metadata }) result.directories.push(groupPath) result.operations++ @@ -242,7 +242,7 @@ export class VFSStructureGenerator { await this.vfs.writeFile(entityPath, JSON.stringify(entityJson, null, 2), { metadata: { - ...trackingMetadata, // v4.10.0: Add tracking metadata + ...trackingMetadata, // Add tracking metadata entityId: extracted.entity.id } }) @@ -253,7 +253,7 @@ export class VFSStructureGenerator { }) result.operations++ - // v4.11.1: Report progress every 10 entities (or on last entity) + // Report progress every 10 entities (or on last entity) if (entityCount % 10 === 0 || entityCount === entities.length) { reportProgress('entities', `Created ${entityCount}/${entities.length} ${groupName} files`) } @@ -280,7 +280,7 @@ export class VFSStructureGenerator { } await this.vfs.writeFile(relationshipsPath, JSON.stringify(relationshipsJson, null, 2), { - metadata: trackingMetadata // v4.10.0: Add tracking metadata + metadata: trackingMetadata // Add tracking metadata }) result.files.push({ path: relationshipsPath, @@ -322,7 +322,7 @@ export class VFSStructureGenerator { } await this.vfs.writeFile(metadataPath, JSON.stringify(metadataJson, null, 2), { - metadata: trackingMetadata // v4.10.0: Add tracking metadata + metadata: trackingMetadata // Add tracking metadata }) result.files.push({ path: metadataPath, @@ -332,7 +332,7 @@ export class VFSStructureGenerator { reportProgress('metadata', 'Created metadata file') } - // v4.11.1: Final progress update + // Final progress update if (options.onProgress) { options.onProgress({ stage: 'metadata', @@ -355,7 +355,7 @@ export class VFSStructureGenerator { ): Map { const groups = new Map() - // Handle sheet-based grouping (v4.2.0) + // Handle sheet-based grouping if (options.groupBy === 'sheet' && importResult.sheets && importResult.sheets.length > 0) { for (const sheet of importResult.sheets) { groups.set(sheet.name, sheet.rows) diff --git a/src/index.ts b/src/index.ts index 5fd6e269..28d2953d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -70,7 +70,7 @@ export type { NeuralImportOptions } from './cortex/neuralImport.js' -// Export Neural Entity Extraction (v5.7.6 - Workshop request) +// Export Neural Entity Extraction export { NeuralEntityExtractor } from './neural/entityExtractor.js' export { SmartExtractor } from './neural/SmartExtractor.js' export { SmartRelationshipExtractor } from './neural/SmartRelationshipExtractor.js' @@ -220,7 +220,7 @@ export { // FileSystemStorage is exported separately to avoid browser build issues export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js' -// Export COW (Copy-on-Write) infrastructure for v5.0.0 +// Export COW (Copy-on-Write) infrastructure // Enables premium augmentations to implement temporal features import { CommitLog } from './storage/cow/CommitLog.js' import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js' @@ -543,7 +543,7 @@ export type { MCPTool } -// ============= Integration Hub (v7.4.0) ============= +// ============= Integration Hub ============= // Connect Brainy to Excel, Power BI, Google Sheets, and more // Enable with: new Brainy({ integrations: true }) diff --git a/src/integrations/index.ts b/src/integrations/index.ts index 804d272c..757a9fe5 100644 --- a/src/integrations/index.ts +++ b/src/integrations/index.ts @@ -7,7 +7,7 @@ * - Real-time dashboards (SSE streaming) * - External webhooks (push notifications) * - * @example Enable integrations (v7.4.0 - recommended) + * @example Enable integrations (recommended) * ```typescript * import { Brainy } from '@soulcraft/brainy' * diff --git a/src/interfaces/IIndex.ts b/src/interfaces/IIndex.ts index eb16357b..b88ca8dc 100644 --- a/src/interfaces/IIndex.ts +++ b/src/interfaces/IIndex.ts @@ -1,5 +1,5 @@ /** - * Unified Index Interface (v3.35.0+) + * Unified Index Interface * * Standardizes index lifecycle across all index types in Brainy. * All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface @@ -57,7 +57,7 @@ export interface RebuildOptions { * - On-demand: Large datasets loaded adaptively via UnifiedCache * * This option is kept for backwards compatibility but is ignored. - * The system always uses adaptive caching (v3.36.0+). + * The system always uses adaptive caching. */ lazy?: boolean @@ -104,7 +104,7 @@ export interface IIndex { * - Provide progress reporting for large datasets * - Recover gracefully from partial failures * - * Adaptive Caching (v3.36.0+): + * Adaptive Caching: * System automatically chooses optimal strategy: * - Small datasets: Preload all data at init for zero-latency access * - Large datasets: Load on-demand via UnifiedCache for memory efficiency diff --git a/src/neural/entityExtractor.ts b/src/neural/entityExtractor.ts index 3d8a85e4..736eaa05 100644 --- a/src/neural/entityExtractor.ts +++ b/src/neural/entityExtractor.ts @@ -2,7 +2,7 @@ * Neural Entity Extractor using Brainy's NounTypes * Uses embeddings and similarity matching for accurate type detection * - * v4.2.0: Now powered by SmartExtractor for ultra-neural classification + * Now powered by SmartExtractor for ultra-neural classification * PRODUCTION-READY with caching support */ @@ -24,7 +24,7 @@ export interface ExtractedEntity { type: NounType position: { start: number; end: number } confidence: number - weight?: number // v4.2.0: Entity importance/salience + weight?: number // Entity importance/salience vector?: Vector metadata?: any } @@ -39,7 +39,7 @@ export class NeuralEntityExtractor { // Entity extraction cache private cache: EntityExtractionCache - // Runtime embedding cache for performance (v3.38.0) + // Runtime embedding cache for performance // Caches candidate embeddings during an extraction session to avoid redundant model calls private embeddingCache: Map = new Map() private embeddingCacheStats = { @@ -48,7 +48,7 @@ export class NeuralEntityExtractor { size: 0 } - // v4.2.0: SmartExtractor for ultra-neural classification + // SmartExtractor for ultra-neural classification private smartExtractor: SmartExtractor constructor(brain: Brainy | Brainy, cacheOptions?: EntityCacheOptions) { @@ -63,7 +63,7 @@ export class NeuralEntityExtractor { /** * Initialize type embeddings for neural matching - * PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed embeddings from build time + * PRODUCTION OPTIMIZATION: Uses pre-computed embeddings from build time * Zero runtime cost - embeddings are loaded instantly from embedded data */ private async initializeTypeEmbeddings(requestedTypes?: NounType[]): Promise { @@ -109,7 +109,7 @@ export class NeuralEntityExtractor { } } ): Promise { - // PRODUCTION OPTIMIZATION (v3.33.0): Load pre-computed type embeddings + // PRODUCTION OPTIMIZATION: Load pre-computed type embeddings // Zero runtime cost - embeddings were computed at build time await this.initializeTypeEmbeddings(options?.types) @@ -138,7 +138,7 @@ export class NeuralEntityExtractor { // Step 1: Extract potential entities using patterns const candidates = await this.extractCandidates(text) - // Step 2: Classify each candidate using SmartExtractor (v4.2.0) + // Step 2: Classify each candidate using SmartExtractor for (const candidate of candidates) { // Use SmartExtractor for unified neural + rule-based classification const classification = await this.smartExtractor.extract(candidate.text, { @@ -357,7 +357,7 @@ export class NeuralEntityExtractor { } /** - * Get embedding for text with caching (v3.38.0) + * Get embedding for text with caching * * PERFORMANCE OPTIMIZATION: Caches embeddings during extraction session * to avoid redundant model calls for repeated text (common in large imports) @@ -509,7 +509,7 @@ export class NeuralEntityExtractor { } /** - * Clear embedding cache (v3.38.0) + * Clear embedding cache * * Clears the runtime embedding cache. Useful for: * - Freeing memory after large imports @@ -525,7 +525,7 @@ export class NeuralEntityExtractor { } /** - * Get embedding cache statistics (v3.38.0) + * Get embedding cache statistics * * Returns performance metrics for the embedding cache: * - hits: Number of cache hits (avoided model calls) diff --git a/src/neural/naturalLanguageProcessor.ts b/src/neural/naturalLanguageProcessor.ts index cb7a5b7f..cf37b74c 100644 --- a/src/neural/naturalLanguageProcessor.ts +++ b/src/neural/naturalLanguageProcessor.ts @@ -99,7 +99,7 @@ export class NaturalLanguageProcessor { /** * Initialize embeddings for all NounTypes and VerbTypes - * PRODUCTION OPTIMIZATION (v3.33.0): Uses pre-computed type embeddings + * PRODUCTION OPTIMIZATION: Uses pre-computed type embeddings * Zero runtime cost - embeddings are loaded instantly from embedded data */ private async initializeTypeEmbeddings(): Promise { diff --git a/src/neural/signals/PatternSignal.ts b/src/neural/signals/PatternSignal.ts index 6a16c3c6..f2ab1ee2 100644 --- a/src/neural/signals/PatternSignal.ts +++ b/src/neural/signals/PatternSignal.ts @@ -122,7 +122,7 @@ export class PatternSignal { ]) // Location patterns - MEDIUM PRIORITY (city/country format - requires more context) - // v4.11.2: Lower priority to avoid matching person names with commas + // Lower priority to avoid matching person names with commas this.addPatterns(NounType.Location, 0.75, [ /\b[A-Z][a-z]+,\s*(?:Japan|China|France|Germany|Italy|Spain|Canada|Mexico|Brazil|India|Australia|Russia|UK|USA)\b/ ]) @@ -170,7 +170,7 @@ export class PatternSignal { // Technology patterns (Thing type) this.addPatterns(NounType.Thing, 0.82, [ /\b(?:JavaScript|TypeScript|Python|Java|Go|Rust|Swift|Kotlin)\b/, - /\bC\+\+(?!\w)/, // v4.11.2: Special handling for C++ (word boundary doesn't work with +) + /\bC\+\+(?!\w)/, // Special handling for C++ (word boundary doesn't work with +) /\b(?:React|Vue|Angular|Node|Express|Django|Flask|Rails)\b/, /\b(?:AWS|Azure|GCP|Docker|Kubernetes|Git|GitHub|GitLab)\b/, /\b(?:API|SDK|CLI|IDE|framework|library|package|module)\b/i, diff --git a/src/setup.ts b/src/setup.ts index cb7d030e..97e4a220 100644 --- a/src/setup.ts +++ b/src/setup.ts @@ -1,7 +1,7 @@ /** * Brainy Setup - Minimal Polyfills * - * ARCHITECTURE (v7.0.0): + * ARCHITECTURE: * Brainy uses Candle WASM (Rust-based) for embeddings. * No transformers.js or ONNX Runtime dependency, no hacks required. * diff --git a/src/storage/adapters/azureBlobStorage.ts b/src/storage/adapters/azureBlobStorage.ts index fa5e405a..ae6fdd7b 100644 --- a/src/storage/adapters/azureBlobStorage.ts +++ b/src/storage/adapters/azureBlobStorage.ts @@ -9,7 +9,7 @@ * 4. SAS Token * 5. Azure AD (OAuth2) via DefaultAzureCredential * - * v4.0.0: Fully compatible with metadata/vector separation architecture + * Fully compatible with metadata/vector separation architecture */ import { @@ -65,7 +65,7 @@ const MAX_AZURE_PAGE_SIZE = 5000 * 3. Storage Account Key - if accountName + accountKey provided * 4. SAS Token - if accountName + sasToken provided * - * v5.4.0: Type-aware storage now built into BaseStorage + * Type-aware storage now built into BaseStorage * - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation) * - Removed pagination overrides * - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths) @@ -105,7 +105,7 @@ export class AzureBlobStorage extends BaseStorage { // Request coalescer for deduplication private requestCoalescer: RequestCoalescer | null = null - // v6.2.7: Write buffering always enabled for consistent performance + // Write buffering always enabled for consistent performance // Removes dynamic mode switching complexity - cloud storage always benefits from batching // Multi-level cache manager for efficient data access @@ -115,7 +115,7 @@ export class AzureBlobStorage extends BaseStorage { // Module logger private logger = createModuleLogger('AzureBlobStorage') - // v5.4.0: HNSW mutex locks to prevent read-modify-write races + // HNSW mutex locks to prevent read-modify-write races private hnswLocks = new Map>() /** @@ -162,7 +162,7 @@ export class AzureBlobStorage extends BaseStorage { } /** - * Initialization mode for fast cold starts (v7.3.0+) + * Initialization mode for fast cold starts * * - `'auto'` (default): Progressive in cloud environments (Azure Functions), * strict locally. Zero-config optimization. @@ -171,7 +171,6 @@ export class AzureBlobStorage extends BaseStorage { * - `'strict'`: Traditional blocking init. Validates container and loads counts * before init() returns. * - * @since v7.3.0 */ initMode?: InitMode @@ -185,7 +184,7 @@ export class AzureBlobStorage extends BaseStorage { this.sasToken = options.sasToken this.readOnly = options.readOnly || false - // v7.3.0: Handle initMode + // Handle initMode if (options.initMode) { this.initMode = options.initMode } @@ -201,7 +200,7 @@ export class AzureBlobStorage extends BaseStorage { this.nounCacheManager = new CacheManager(options.cacheConfig) this.verbCacheManager = new CacheManager(options.cacheConfig) - // v6.2.7: Write buffering always enabled - no env var check needed + // Write buffering always enabled - no env var check needed } /** @@ -216,7 +215,7 @@ export class AzureBlobStorage extends BaseStorage { * Recent Azure improvements make parallel downloads very efficient * * @returns Azure Blob-optimized batch configuration - * @since v5.12.0 - Updated for native batch API + * Updated for native batch API */ public getBatchConfig(): StorageBatchConfig { return { @@ -241,7 +240,6 @@ export class AzureBlobStorage extends BaseStorage { * * @param paths - Array of Azure blob paths to read * @returns Map of path -> parsed JSON data (only successful reads) - * @since v5.12.0 */ public async readBatch(paths: string[]): Promise> { await this.ensureInitialized() @@ -297,7 +295,7 @@ export class AzureBlobStorage extends BaseStorage { /** * Initialize the storage adapter * - * v7.3.0: Supports progressive initialization for fast cold starts + * Supports progressive initialization for fast cold starts * * | Mode | Init Time | When | * |------|-----------|------| @@ -404,7 +402,7 @@ export class AzureBlobStorage extends BaseStorage { this.verbCacheManager.clear() prodLog.info('✅ Cache cleared - starting fresh') - // v7.3.0: Progressive vs Strict initialization + // Progressive vs Strict initialization if (effectiveMode === 'progressive') { // PROGRESSIVE MODE: Fast init, background validation // Mark as initialized immediately - ready to accept operations @@ -413,7 +411,7 @@ export class AzureBlobStorage extends BaseStorage { prodLog.info(`✅ Azure progressive init complete: ${this.containerName} (validation deferred)`) - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() // Schedule background tasks (non-blocking) @@ -434,7 +432,7 @@ export class AzureBlobStorage extends BaseStorage { await this.initializeCounts() this.countsLoaded = true - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() // Mark background tasks as complete (nothing to do in background) @@ -447,7 +445,7 @@ export class AzureBlobStorage extends BaseStorage { } // ============================================= - // Progressive Initialization (v7.3.0+) + // Progressive Initialization // ============================================= /** @@ -461,7 +459,6 @@ export class AzureBlobStorage extends BaseStorage { * * @protected * @override - * @since v7.3.0 */ protected async runBackgroundInit(): Promise { const startTime = Date.now() @@ -485,7 +482,6 @@ export class AzureBlobStorage extends BaseStorage { * bucketValidated/bucketValidationError for lazy use. * * @private - * @since v7.3.0 */ private async validateContainerInBackground(): Promise { try { @@ -517,7 +513,6 @@ export class AzureBlobStorage extends BaseStorage { * Load counts from storage in background. * * @private - * @since v7.3.0 */ private async loadCountsInBackground(): Promise { try { @@ -541,7 +536,6 @@ export class AzureBlobStorage extends BaseStorage { * @throws Error if container validation fails * @protected * @override - * @since v7.3.0 */ protected async ensureValidatedForWrite(): Promise { // If already validated, nothing to do @@ -665,7 +659,7 @@ export class AzureBlobStorage extends BaseStorage { } } - // v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage + // Removed checkVolumeMode() - write buffering always enabled for cloud storage /** * Flush noun buffer to Azure @@ -697,20 +691,20 @@ export class AzureBlobStorage extends BaseStorage { await Promise.all(writes) } - // v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation + // Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation /** * Save a node to storage - * v6.2.7: Always uses write buffer for consistent performance + * Always uses write buffer for consistent performance */ protected async saveNode(node: HNSWNode): Promise { await this.ensureInitialized() - // v6.2.7: Always use write buffer - cloud storage benefits from batching + // Always use write buffer - cloud storage benefits from batching if (this.nounWriteBuffer) { this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`) - // v6.2.6: Populate cache BEFORE buffering for read-after-write consistency + // Populate cache BEFORE buffering for read-after-write consistency if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) { this.nounCacheManager.set(node.id, node) } @@ -791,7 +785,7 @@ export class AzureBlobStorage extends BaseStorage { } } - // v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation + // Removed getNoun_internal - now inherit from BaseStorage's type-first implementation /** * Get a node from storage @@ -889,18 +883,18 @@ export class AzureBlobStorage extends BaseStorage { } } - // v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation + // Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation /** * Write an object to a specific path in Azure * Primitive operation required by base class * - * v7.3.0: Performs lazy container validation on first write in progressive mode. + * Performs lazy container validation on first write in progressive mode. * @protected */ protected async writeObjectToPath(path: string, data: any): Promise { await this.ensureInitialized() - // v7.3.0: Lazy container validation for progressive init + // Lazy container validation for progressive init await this.ensureValidatedForWrite() try { @@ -954,12 +948,12 @@ export class AzureBlobStorage extends BaseStorage { * Delete an object from a specific path in Azure * Primitive operation required by base class * - * v7.3.0: Performs lazy container validation on first delete in progressive mode. + * Performs lazy container validation on first delete in progressive mode. * @protected */ protected async deleteObjectFromPath(path: string): Promise { await this.ensureInitialized() - // v7.3.0: Lazy container validation for progressive init + // Lazy container validation for progressive init await this.ensureValidatedForWrite() try { @@ -1209,20 +1203,20 @@ export class AzureBlobStorage extends BaseStorage { }) } - // v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation + // Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation /** * Save an edge to storage - * v6.2.7: Always uses write buffer for consistent performance + * Always uses write buffer for consistent performance */ protected async saveEdge(edge: Edge): Promise { await this.ensureInitialized() - // v6.2.7: Always use write buffer - cloud storage benefits from batching + // Always use write buffer - cloud storage benefits from batching if (this.verbWriteBuffer) { this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`) - // v6.2.6: Populate cache BEFORE buffering for read-after-write consistency + // Populate cache BEFORE buffering for read-after-write consistency this.verbCacheManager.set(edge.id, edge) await this.verbWriteBuffer.add(edge.id, edge) @@ -1255,7 +1249,7 @@ export class AzureBlobStorage extends BaseStorage { ]) ), - // CORE RELATIONAL DATA (v4.0.0) + // CORE RELATIONAL DATA verb: edge.verb, sourceId: edge.sourceId, targetId: edge.targetId, @@ -1280,7 +1274,7 @@ export class AzureBlobStorage extends BaseStorage { // Update cache this.verbCacheManager.set(edge.id, edge) - // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // Count tracking happens in baseStorage.saveVerbMetadata_internal // This fixes the race condition where metadata didn't exist yet this.logger.trace(`Edge ${edge.id} saved successfully`) @@ -1298,7 +1292,7 @@ export class AzureBlobStorage extends BaseStorage { } } - // v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation + // Removed getVerb_internal - now inherit from BaseStorage's type-first implementation /** * Get an edge from storage @@ -1335,7 +1329,7 @@ export class AzureBlobStorage extends BaseStorage { connections.set(Number(level), new Set(verbIds as string[])) } - // v4.0.0: Return HNSWVerb with core relational fields (NO metadata field) + // Return HNSWVerb with core relational fields (NO metadata field) const edge: Edge = { id: data.id, vector: data.vector, @@ -1346,7 +1340,7 @@ export class AzureBlobStorage extends BaseStorage { sourceId: data.sourceId, targetId: data.targetId - // ✅ NO metadata field in v4.0.0 + // ✅ NO metadata field // User metadata retrieved separately via getVerbMetadata() } @@ -1375,13 +1369,13 @@ export class AzureBlobStorage extends BaseStorage { } } - // v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation + // Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation - // v5.4.0: Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation + // Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation - // v5.4.0: Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation + // Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation - // v5.4.0: Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation + // Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation /** * Clear all data from storage @@ -1393,7 +1387,7 @@ export class AzureBlobStorage extends BaseStorage { this.logger.info('🧹 Clearing all data from Azure container...') // Delete all blobs in container - // v5.6.1: listBlobsFlat() returns ALL blobs including _cow/ prefix + // listBlobsFlat() returns ALL blobs including _cow/ prefix // This correctly deletes COW version control data (commits, trees, blobs, refs) for await (const blob of this.containerClient!.listBlobsFlat()) { if (blob.name) { @@ -1402,7 +1396,7 @@ export class AzureBlobStorage extends BaseStorage { } } - // v5.11.0: Reset COW managers (but don't disable COW - it's always enabled) + // Reset COW managers (but don't disable COW - it's always enabled) // COW will re-initialize automatically on next use this.refManager = undefined this.blobStorage = undefined @@ -1461,12 +1455,12 @@ export class AzureBlobStorage extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts + * Fixes bug where clear() doesn't persist across instance restarts * @returns true if marker blob exists, false otherwise * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ @@ -1643,7 +1637,7 @@ export class AzureBlobStorage extends BaseStorage { /** * Get a noun's vector for HNSW rebuild - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getNounVector(id: string): Promise { const noun = await this.getNoun(id) @@ -1653,7 +1647,7 @@ export class AzureBlobStorage extends BaseStorage { /** * Save HNSW graph data for a noun * - * v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths) + * Uses BaseStorage's getNoun/saveNoun (type-first paths) * CRITICAL: Uses mutex locking to prevent read-modify-write races */ public async saveHNSWData(nounId: string, hnswData: { @@ -1662,7 +1656,7 @@ export class AzureBlobStorage extends BaseStorage { }): Promise { const lockKey = `hnsw/${nounId}` - // CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races + // CRITICAL FIX: Mutex lock to prevent read-modify-write races // Problem: Without mutex, concurrent operations can: // 1. Thread A reads noun (connections: [1,2,3]) // 2. Thread B reads noun (connections: [1,2,3]) @@ -1682,7 +1676,7 @@ export class AzureBlobStorage extends BaseStorage { this.hnswLocks.set(lockKey, lockPromise) try { - // v5.4.0: Use BaseStorage's getNoun (type-first paths) + // Use BaseStorage's getNoun (type-first paths) // Read existing noun data (if exists) const existingNoun = await this.getNoun(nounId) @@ -1704,7 +1698,7 @@ export class AzureBlobStorage extends BaseStorage { connections: connectionsMap } - // v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch) + // Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch) await this.saveNoun(updatedNoun) } finally { // Release lock (ALWAYS runs, even if error thrown) @@ -1715,7 +1709,7 @@ export class AzureBlobStorage extends BaseStorage { /** * Get HNSW graph data for a noun - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getHNSWData(nounId: string): Promise<{ level: number @@ -1744,7 +1738,7 @@ export class AzureBlobStorage extends BaseStorage { /** * Save HNSW system data (entry point, max level) * - * CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions + * CRITICAL FIX: Optimistic locking with ETags to prevent race conditions */ public async saveHNSWSystem(systemData: { entryPointId: string | null @@ -1840,7 +1834,7 @@ export class AzureBlobStorage extends BaseStorage { } /** - * Set the access tier for a specific blob (v4.0.0 cost optimization) + * Set the access tier for a specific blob (cost optimization) * Azure Blob Storage tiers: * - Hot: $0.0184/GB/month - Frequently accessed data * - Cool: $0.01/GB/month - Infrequently accessed data (45% cheaper) @@ -1906,7 +1900,7 @@ export class AzureBlobStorage extends BaseStorage { } /** - * Set access tier for multiple blobs in batch (v4.0.0 cost optimization) + * Set access tier for multiple blobs in batch (cost optimization) * Efficiently move large numbers of blobs between tiers for cost optimization * * @param blobs - Array of blob names and their target tiers @@ -2128,7 +2122,7 @@ export class AzureBlobStorage extends BaseStorage { } /** - * Set lifecycle management policy for automatic tier transitions and deletions (v4.0.0) + * Set lifecycle management policy for automatic tier transitions and deletions * Automates cost optimization by moving old data to cheaper tiers or deleting it * * Azure Lifecycle Management rules run once per day and apply to the entire container. diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index f6d37ad5..66acfd87 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -19,7 +19,7 @@ import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/field import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js' // ============================================================================= -// Progressive Initialization Types (v7.3.0+) +// Progressive Initialization Types // ============================================================================= /** @@ -29,7 +29,6 @@ import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js' * fast cold starts in serverless environments while maintaining strict * validation for local development. * - * @since v7.3.0 * * | Mode | Description | Use Case | * |------|-------------|----------| @@ -96,7 +95,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { abstract getVerbMetadata(id: string): Promise - // HNSW Index Persistence (v3.35.0+) + // HNSW Index Persistence // These methods enable HNSW index rebuilding after container restarts abstract getNounVector(id: string): Promise @@ -139,7 +138,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * relateMany(), and import operations to automatically adapt to storage capabilities. * * @returns Batch configuration optimized for this storage type - * @since v4.11.0 */ public getBatchConfig(): StorageBatchConfig { // Conservative defaults that work safely across all storage types @@ -295,7 +293,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { }> = new Map() // ============================================= - // Progressive Initialization State (v7.3.0+) + // Progressive Initialization State // ============================================= // These properties enable fast cold starts in cloud environments // by deferring validation and count loading to background tasks. @@ -308,7 +306,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * - `'strict'`: Always validate during init (traditional behavior) * * @protected - * @since v7.3.0 */ protected initMode: InitMode = 'auto' @@ -319,7 +316,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * after the first successful write operation or background validation. * * @protected - * @since v7.3.0 */ protected bucketValidated = false @@ -329,7 +325,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * Stored here so subsequent operations can fail fast with the same error. * * @protected - * @since v7.3.0 */ protected bucketValidationError: Error | null = null @@ -340,7 +335,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * Operations work immediately; counts become accurate after background load. * * @protected - * @since v7.3.0 */ protected countsLoaded = false @@ -350,7 +344,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * Useful for tests and diagnostics to ensure full initialization. * * @protected - * @since v7.3.0 */ protected backgroundTasksComplete = false @@ -361,7 +354,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * before proceeding (e.g., in tests). * * @protected - * @since v7.3.0 */ protected backgroundInitPromise: Promise | null = null @@ -1057,7 +1049,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { protected readonly COUNT_CACHE_TTL = 60000 // 1 minute cache TTL // ============================================= - // Smart Count Batching (v3.32.3+) + // Smart Count Batching // ============================================= // Count batching state - mirrors statistics batching pattern @@ -1112,7 +1104,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { const mutex = getGlobalMutex() await mutex.runExclusive(`count-entity-${type}`, async () => { this.incrementEntityCount(type) - // Smart batching (v3.32.3+): Adapts to storage type + // Smart batching: Adapts to storage type // - Cloud storage (GCS, S3): Batches 10 ops OR 5 seconds // - Local storage (File, Memory): Persists immediately await this.scheduleCountPersist() @@ -1147,13 +1139,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter { const mutex = getGlobalMutex() await mutex.runExclusive(`count-entity-${type}`, async () => { this.decrementEntityCount(type) - // Smart batching (v3.32.3+): Adapts to storage type + // Smart batching: Adapts to storage type await this.scheduleCountPersist() }) } /** - * Increment verb count - O(1) operation (v4.1.2: now synchronous) + * Increment verb count - O(1) operation (now synchronous) * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.) * @param type The verb type */ @@ -1168,7 +1160,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } /** - * Thread-safe increment for verb counts (v4.1.2) + * Thread-safe increment for verb counts * Uses mutex for single-node, distributed consensus for multi-node * @param type The verb type */ @@ -1176,13 +1168,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter { const mutex = getGlobalMutex() await mutex.runExclusive(`count-verb-${type}`, async () => { this.incrementVerbCount(type) - // Smart batching (v3.32.3+): Adapts to storage type + // Smart batching: Adapts to storage type await this.scheduleCountPersist() }) } /** - * Decrement verb count - O(1) operation (v4.1.2: now synchronous) + * Decrement verb count - O(1) operation (now synchronous) * @param type The verb type */ protected decrementVerbCount(type: string): void { @@ -1203,20 +1195,20 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } /** - * Thread-safe decrement for verb counts (v4.1.2) + * Thread-safe decrement for verb counts * @param type The verb type */ protected async decrementVerbCountSafe(type: string): Promise { const mutex = getGlobalMutex() await mutex.runExclusive(`count-verb-${type}`, async () => { this.decrementVerbCount(type) - // Smart batching (v3.32.3+): Adapts to storage type + // Smart batching: Adapts to storage type await this.scheduleCountPersist() }) } // ============================================= - // Smart Batching Methods (v3.32.3+) + // Smart Batching Methods // ============================================= /** @@ -1235,7 +1227,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } // ============================================= - // Progressive Initialization Methods (v7.3.0+) + // Progressive Initialization Methods // ============================================= /** @@ -1254,7 +1246,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * * @returns `true` if running in a detected cloud environment * @protected - * @since v7.3.0 */ protected detectCloudEnvironment(): boolean { return !!( @@ -1274,7 +1265,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * * @returns The resolved init mode (`'progressive'` or `'strict'`) * @protected - * @since v7.3.0 */ protected resolveInitMode(): 'progressive' | 'strict' { if (this.initMode === 'auto') { @@ -1295,7 +1285,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * Always call `super.scheduleBackgroundInit()` first. * * @protected - * @since v7.3.0 */ protected scheduleBackgroundInit(): void { // Create a promise that tracks all background work @@ -1320,7 +1309,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * The default implementation does nothing (for local storage adapters). * * @protected - * @since v7.3.0 */ protected async runBackgroundInit(): Promise { // Default implementation: nothing to do for local storage @@ -1348,7 +1336,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * ``` * * @public - * @since v7.3.0 */ public async awaitBackgroundInit(): Promise { if (this.backgroundInitPromise) { @@ -1361,7 +1348,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * * @returns `true` if all background tasks are complete * @public - * @since v7.3.0 */ public isBackgroundInitComplete(): boolean { return this.backgroundTasksComplete @@ -1379,7 +1365,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * * @throws Error if bucket validation fails * @protected - * @since v7.3.0 */ protected async ensureValidatedForWrite(): Promise { // If already validated, nothing to do diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index a1bc9c46..1bc97436 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -59,7 +59,7 @@ try { * File system storage adapter for Node.js environments * Uses the file system to store data in the specified directory structure * - * v5.4.0: Type-aware storage now built into BaseStorage + * Type-aware storage now built into BaseStorage * - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation) * - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination) * - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths) @@ -91,12 +91,12 @@ export class FileSystemStorage extends BaseStorage { private lockTimers: Map = new Map() // Track timers for cleanup private allTimers: Set = new Set() // Track all timers for cleanup - // CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control + // CRITICAL FIX: Mutex locks for HNSW concurrency control // Prevents read-modify-write races during concurrent neighbor updates at scale (1000+ ops) // Matches MemoryStorage and OPFSStorage behavior (tested in production) private hnswLocks = new Map>() - // Compression configuration (v4.0.0) + // Compression configuration private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced) @@ -136,7 +136,6 @@ export class FileSystemStorage extends BaseStorage { * - Parallel processing supported * * @returns FileSystem-optimized batch configuration - * @since v4.11.0 */ public getBatchConfig(): StorageBatchConfig { return { @@ -179,7 +178,7 @@ export class FileSystemStorage extends BaseStorage { try { // Initialize directory paths now that path module is loaded - // Clean directory structure (v4.7.2+) + // Clean directory structure this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw') this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw') this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata') // Legacy reference @@ -245,7 +244,7 @@ export class FileSystemStorage extends BaseStorage { // Always use fixed depth after migration/detection this.cachedShardingDepth = this.SHARDING_DEPTH - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() } catch (error) { console.error('Error initializing FileSystemStorage:', error) @@ -281,7 +280,7 @@ export class FileSystemStorage extends BaseStorage { /** * Save a node to storage - * CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports + * CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports */ protected async saveNode(node: HNSWNode): Promise { await this.ensureInitialized() @@ -303,7 +302,7 @@ export class FileSystemStorage extends BaseStorage { const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` try { - // ATOMIC WRITE SEQUENCE (v4.10.3): + // ATOMIC WRITE SEQUENCE: // 1. Write to temp file await this.ensureDirectoryExists(path.dirname(tempPath)) await fs.promises.writeFile(tempPath, JSON.stringify(serializableNode, null, 2)) @@ -320,7 +319,7 @@ export class FileSystemStorage extends BaseStorage { throw error } - // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2) + // Count tracking happens in baseStorage.saveNounMetadata_internal // This fixes the race condition where metadata didn't exist yet } @@ -361,7 +360,7 @@ export class FileSystemStorage extends BaseStorage { /** * Get all nodes from storage - * CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1) + * CRITICAL FIX: Now scans sharded subdirectories (depth=1) * Previously only scanned flat directory, causing rebuild to find 0 entities */ protected async getAllNodes(): Promise { @@ -406,7 +405,7 @@ export class FileSystemStorage extends BaseStorage { /** * Get nodes by noun type - * CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1) + * CRITICAL FIX: Now scans sharded subdirectories (depth=1) * @param nounType The noun type to filter by * @returns Promise that resolves to an array of nodes of the specified noun type */ @@ -462,7 +461,7 @@ export class FileSystemStorage extends BaseStorage { const filePath = this.getNodePath(id) - // Load metadata to get type for count update (v4.0.0: separate storage) + // Load metadata to get type for count update (separate storage) try { const metadata = await this.getNounMetadata(id) if (metadata) { @@ -490,13 +489,13 @@ export class FileSystemStorage extends BaseStorage { /** * Save an edge to storage - * CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports + * CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports */ protected async saveEdge(edge: Edge): Promise { await this.ensureInitialized() // Convert connections Map to a serializable format - // ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file + // ARCHITECTURAL FIX: Include core relational fields in verb vector file // These fields are essential for 90% of operations - no metadata lookup needed const serializableEdge = { id: edge.id, @@ -505,7 +504,7 @@ export class FileSystemStorage extends BaseStorage { Array.from(set as Set) ), - // CORE RELATIONAL DATA (v3.50.1+) + // CORE RELATIONAL DATA verb: edge.verb, sourceId: edge.sourceId, targetId: edge.targetId, @@ -518,7 +517,7 @@ export class FileSystemStorage extends BaseStorage { const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` try { - // ATOMIC WRITE SEQUENCE (v4.10.3): + // ATOMIC WRITE SEQUENCE: // 1. Write to temp file await this.ensureDirectoryExists(path.dirname(tempPath)) await fs.promises.writeFile(tempPath, JSON.stringify(serializableEdge, null, 2)) @@ -535,7 +534,7 @@ export class FileSystemStorage extends BaseStorage { throw error } - // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // Count tracking happens in baseStorage.saveVerbMetadata_internal // This fixes the race condition where metadata didn't exist yet } @@ -556,7 +555,7 @@ export class FileSystemStorage extends BaseStorage { connections.set(Number(level), new Set(nodeIds as string[])) } - // v4.0.0: Return HNSWVerb with core relational fields (NO metadata field) + // Return HNSWVerb with core relational fields (NO metadata field) return { id: parsedEdge.id, vector: parsedEdge.vector, @@ -567,7 +566,7 @@ export class FileSystemStorage extends BaseStorage { sourceId: parsedEdge.sourceId, targetId: parsedEdge.targetId - // ✅ NO metadata field in v4.0.0 + // ✅ NO metadata field // User metadata retrieved separately via getVerbMetadata() } } catch (error: any) { @@ -580,7 +579,7 @@ export class FileSystemStorage extends BaseStorage { /** * Get all edges from storage - * CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1) + * CRITICAL FIX: Now scans sharded subdirectories (depth=1) * Previously only scanned flat directory, causing rebuild to find 0 relationships */ protected async getAllEdges(): Promise { @@ -608,7 +607,7 @@ export class FileSystemStorage extends BaseStorage { connections.set(Number(level), new Set(nodeIds as string[])) } - // v4.0.0: Include core relational fields (NO metadata field) + // Include core relational fields (NO metadata field) allEdges.push({ id: parsedEdge.id, vector: parsedEdge.vector, @@ -619,7 +618,7 @@ export class FileSystemStorage extends BaseStorage { sourceId: parsedEdge.sourceId, targetId: parsedEdge.targetId - // ✅ NO metadata field in v4.0.0 + // ✅ NO metadata field // User metadata retrieved separately via getVerbMetadata() }) } @@ -696,8 +695,8 @@ export class FileSystemStorage extends BaseStorage { /** * Primitive operation: Write object to path * All metadata operations use this internally via base class routing - * v4.0.0: Supports gzip compression for 60-80% disk savings - * CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports + * Supports gzip compression for 60-80% disk savings + * CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports */ protected async writeObjectToPath(pathStr: string, data: any): Promise { await this.ensureInitialized() @@ -711,7 +710,7 @@ export class FileSystemStorage extends BaseStorage { const tempPath = `${compressedPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` try { - // ATOMIC WRITE SEQUENCE (v4.10.3): + // ATOMIC WRITE SEQUENCE: // 1. Compress and write to temp file const jsonString = JSON.stringify(data, null, 2) const compressed = await new Promise((resolve, reject) => { @@ -748,7 +747,7 @@ export class FileSystemStorage extends BaseStorage { const tempPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}` try { - // ATOMIC WRITE SEQUENCE (v4.10.3): + // ATOMIC WRITE SEQUENCE: // 1. Write to temp file await fs.promises.writeFile(tempPath, JSON.stringify(data, null, 2)) @@ -770,7 +769,7 @@ export class FileSystemStorage extends BaseStorage { * Primitive operation: Read object from path * All metadata operations use this internally via base class routing * Enhanced error handling for corrupted metadata files (Bug #3 mitigation) - * v4.0.0: Supports reading both compressed (.gz) and uncompressed files for backward compatibility + * Supports reading both compressed (.gz) and uncompressed files for backward compatibility */ protected async readObjectFromPath(pathStr: string): Promise { await this.ensureInitialized() @@ -822,7 +821,7 @@ export class FileSystemStorage extends BaseStorage { /** * Primitive operation: Delete object from path * All metadata operations use this internally via base class routing - * v4.0.0: Deletes both compressed and uncompressed versions (for cleanup) + * Deletes both compressed and uncompressed versions (for cleanup) */ protected async deleteObjectFromPath(pathStr: string): Promise { await this.ensureInitialized() @@ -863,7 +862,7 @@ export class FileSystemStorage extends BaseStorage { /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing - * v4.0.0: Handles both .json and .json.gz files, normalizes paths + * Handles both .json and .json.gz files, normalizes paths */ protected async listObjectsUnderPath(prefix: string): Promise { await this.ensureInitialized() @@ -877,7 +876,7 @@ export class FileSystemStorage extends BaseStorage { for (const entry of entries) { if (entry.isFile()) { - // v5.3.5: Handle multiple compression formats for broad compatibility + // Handle multiple compression formats for broad compatibility // - .json.gz: Standard entity/metadata files (JSON compressed) // - .gz: COW files (refs, blobs, commits - raw compressed) // - .json: Uncompressed JSON files @@ -890,7 +889,7 @@ export class FileSystemStorage extends BaseStorage { seen.add(normalizedPath) } } else if (entry.name.endsWith('.gz')) { - // v5.3.5 fix: COW files stored as .gz (not .json.gz) + // COW files stored as .gz (not .json.gz) // Strip .gz extension and return path const normalizedName = entry.name.slice(0, -3) // Remove .gz const normalizedPath = path.join(prefix, normalizedName) @@ -966,7 +965,7 @@ export class FileSystemStorage extends BaseStorage { * Get nouns with pagination support * @param options Pagination options */ - // v5.4.0: Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation + // Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation /** * Clear all data from storage @@ -1002,7 +1001,7 @@ export class FileSystemStorage extends BaseStorage { } } - // v5.10.4: Clear the entire branches/ directory (branch-based storage) + // Clear the entire branches/ directory (branch-based storage) // Bug fix: Data is stored in branches/main/entities/, not just entities/ // The branch-based structure was introduced for COW support const branchesDir = path.join(this.rootDir, 'branches') @@ -1016,7 +1015,7 @@ export class FileSystemStorage extends BaseStorage { await removeDirectoryContents(this.indexDir) } - // v5.6.1: Remove COW (copy-on-write) version control data + // Remove COW (copy-on-write) version control data // This directory stores all git-like versioning data (commits, trees, blobs, refs) // Must be deleted to fully clear all data including version history const cowDir = path.join(this.rootDir, '_cow') @@ -1024,7 +1023,7 @@ export class FileSystemStorage extends BaseStorage { // Delete the entire _cow/ directory (not just contents) await fs.promises.rm(cowDir, { recursive: true, force: true }) - // v5.11.0: Reset COW managers (but don't disable COW - it's always enabled) + // Reset COW managers (but don't disable COW - it's always enabled) // COW will re-initialize automatically on next use this.refManager = undefined this.blobStorage = undefined @@ -1035,12 +1034,12 @@ export class FileSystemStorage extends BaseStorage { this.statisticsCache = null this.statisticsModified = false - // v5.6.1: Reset entity counters (inherited from BaseStorageAdapter) + // Reset entity counters (inherited from BaseStorageAdapter) // These in-memory counters must be reset to 0 after clearing all data ;(this as any).totalNounCount = 0 ;(this as any).totalVerbCount = 0 - // v7.3.1: Clear write-through cache (inherited from BaseStorage) + // Clear write-through cache (inherited from BaseStorage) // Without this, readWithInheritance() would return stale cached data // after clear(), causing "ghost" entities to appear this.clearWriteCache() @@ -1074,12 +1073,12 @@ export class FileSystemStorage extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts + * Fixes bug where clear() doesn't persist across instance restarts * @returns true if marker file exists, false otherwise * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ @@ -1152,7 +1151,7 @@ export class FileSystemStorage extends BaseStorage { totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize - // CRITICAL FIX (v3.43.2): Use persisted counts instead of directory reads + // CRITICAL FIX: Use persisted counts instead of directory reads // This is O(1) instead of O(n), and handles sharded structure correctly const nounsCount = this.totalNounCount const verbsCount = this.totalVerbCount @@ -1210,8 +1209,8 @@ export class FileSystemStorage extends BaseStorage { } } - // v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation - // v5.4.0: Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation + // Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation + // Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation /** * Acquire a file-based lock for coordinating operations across multiple processes @@ -1503,14 +1502,14 @@ export class FileSystemStorage extends BaseStorage { this.totalVerbCount = validVerbFiles.length // Sample some files to get type distribution (don't read all) - // v4.0.0: Load metadata separately for type information + // Load metadata separately for type information const sampleSize = Math.min(100, validNounFiles.length) for (let i = 0; i < sampleSize; i++) { try { const file = validNounFiles[i] const id = file.replace('.json', '') - // v4.0.0: Load metadata from separate storage for type info + // Load metadata from separate storage for type info const metadata = await this.getNounMetadata(id) if (metadata) { const type = metadata.noun || 'default' @@ -2148,7 +2147,7 @@ export class FileSystemStorage extends BaseStorage { const edge = JSON.parse(data) const metadata = await this.getVerbMetadata(id) - // v4.8.1: Don't skip verbs without metadata - metadata is optional + // Don't skip verbs without metadata - metadata is optional // FIX: This was the root cause of the VFS bug (11 versions) // Verbs can exist without metadata files (e.g., from imports/migrations) @@ -2162,7 +2161,7 @@ export class FileSystemStorage extends BaseStorage { connections = connectionsMap } - // v4.8.0: Extract standard fields from metadata to top-level + // Extract standard fields from metadata to top-level const metadataObj = (metadata || {}) as VerbMetadata const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj @@ -2309,12 +2308,12 @@ export class FileSystemStorage extends BaseStorage { } // ============================================= - // HNSW Index Persistence (v3.35.0+) + // HNSW Index Persistence // ============================================= /** * Get vector for a noun - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getNounVector(id: string): Promise { const noun = await this.getNoun(id) @@ -2324,7 +2323,7 @@ export class FileSystemStorage extends BaseStorage { /** * Save HNSW graph data for a noun * - * v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths) + * Uses BaseStorage's getNoun/saveNoun (type-first paths) * CRITICAL: Preserves mutex locking to prevent read-modify-write races */ public async saveHNSWData(nounId: string, hnswData: { @@ -2333,7 +2332,7 @@ export class FileSystemStorage extends BaseStorage { }): Promise { const lockKey = `hnsw/${nounId}` - // CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races + // CRITICAL FIX: Mutex lock to prevent read-modify-write races // Problem: Without mutex, concurrent operations can: // 1. Thread A reads noun (connections: [1,2,3]) // 2. Thread B reads noun (connections: [1,2,3]) @@ -2353,7 +2352,7 @@ export class FileSystemStorage extends BaseStorage { this.hnswLocks.set(lockKey, lockPromise) try { - // v5.4.0: Use BaseStorage's getNoun (type-first paths) + // Use BaseStorage's getNoun (type-first paths) // Read existing noun data (if exists) const existingNoun = await this.getNoun(nounId) @@ -2375,7 +2374,7 @@ export class FileSystemStorage extends BaseStorage { connections: connectionsMap } - // v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write) + // Use BaseStorage's saveNoun (type-first paths, atomic write) await this.saveNoun(updatedNoun) } finally { // Release lock (ALWAYS runs, even if error thrown) @@ -2386,7 +2385,7 @@ export class FileSystemStorage extends BaseStorage { /** * Get HNSW graph data for a noun - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getHNSWData(nounId: string): Promise<{ level: number @@ -2415,7 +2414,7 @@ export class FileSystemStorage extends BaseStorage { /** * Save HNSW system data (entry point, max level) * - * CRITICAL FIX (v4.10.1): Mutex lock + atomic write to prevent race conditions + * CRITICAL FIX: Mutex lock + atomic write to prevent race conditions */ public async saveHNSWSystem(systemData: { entryPointId: string | null @@ -2425,7 +2424,7 @@ export class FileSystemStorage extends BaseStorage { const lockKey = 'hnsw/system' - // CRITICAL FIX (v4.10.1): Mutex lock to serialize system updates + // CRITICAL FIX: Mutex lock to serialize system updates // System data (entry point, max level) updated frequently during HNSW construction // Without mutex, concurrent updates can lose data (same as entity-level problem) diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts index 79d2ccb0..b65c8ed7 100644 --- a/src/storage/adapters/gcsStorage.ts +++ b/src/storage/adapters/gcsStorage.ts @@ -65,7 +65,7 @@ const MAX_GCS_PAGE_SIZE = 5000 * 3. Service Account Credentials Object (if credentials provided) * 4. HMAC Keys (if accessKeyId/secretAccessKey provided) * - * v5.4.0: Type-aware storage now built into BaseStorage + * Type-aware storage now built into BaseStorage * - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation) * - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination) * - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths) @@ -110,7 +110,7 @@ export class GcsStorage extends BaseStorage { // Request coalescer for deduplication private requestCoalescer: RequestCoalescer | null = null - // v6.2.7: Write buffering always enabled for consistent performance + // Write buffering always enabled for consistent performance // Removes dynamic mode switching complexity - cloud storage always benefits from batching // Multi-level cache manager for efficient data access @@ -120,7 +120,7 @@ export class GcsStorage extends BaseStorage { // Module logger private logger = createModuleLogger('GcsStorage') - // v5.4.0: HNSW mutex locks to prevent read-modify-write races + // HNSW mutex locks to prevent read-modify-write races private hnswLocks = new Map>() // Configuration options @@ -166,7 +166,7 @@ export class GcsStorage extends BaseStorage { secretAccessKey?: string /** - * Initialization mode for fast cold starts (v7.3.0+) + * Initialization mode for fast cold starts * * - `'auto'` (default): Progressive in cloud environments (Cloud Run, Lambda), * strict locally. Zero-config optimization. @@ -175,19 +175,18 @@ export class GcsStorage extends BaseStorage { * - `'strict'`: Traditional blocking init. Validates bucket and loads counts * before init() returns. * - * @since v7.3.0 */ initMode?: InitMode /** * @deprecated Use `initMode: 'progressive'` instead. - * Will be removed in v8.0.0. + * Will be removed in a future version. */ skipInitialScan?: boolean /** * @deprecated Use `initMode: 'progressive'` instead. - * Will be removed in v8.0.0. + * Will be removed in a future version. */ skipCountsFile?: boolean @@ -206,7 +205,7 @@ export class GcsStorage extends BaseStorage { this.accessKeyId = options.accessKeyId this.secretAccessKey = options.secretAccessKey - // v7.3.0: Handle initMode and deprecated skip* flags + // Handle initMode and deprecated skip* flags if (options.initMode) { this.initMode = options.initMode } @@ -215,14 +214,14 @@ export class GcsStorage extends BaseStorage { if (options.skipInitialScan) { console.warn( '[GcsStorage] DEPRECATION WARNING: skipInitialScan is deprecated. ' + - 'Use initMode: "progressive" instead. Will be removed in v8.0.0.' + 'Use initMode: "progressive" instead. Will be removed in a future version.' ) this.skipInitialScan = true } if (options.skipCountsFile) { console.warn( '[GcsStorage] DEPRECATION WARNING: skipCountsFile is deprecated. ' + - 'Use initMode: "progressive" instead. Will be removed in v8.0.0.' + 'Use initMode: "progressive" instead. Will be removed in a future version.' ) this.skipCountsFile = true } @@ -240,13 +239,13 @@ export class GcsStorage extends BaseStorage { this.nounCacheManager = new CacheManager(options.cacheConfig) this.verbCacheManager = new CacheManager(options.cacheConfig) - // v6.2.7: Write buffering always enabled - no env var check needed + // Write buffering always enabled - no env var check needed } /** * Initialize the storage adapter * - * v7.3.0: Supports progressive initialization for fast cold starts + * Supports progressive initialization for fast cold starts * * | Mode | Init Time | When | * |------|-----------|------| @@ -337,14 +336,14 @@ export class GcsStorage extends BaseStorage { } ) - // CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs + // CRITICAL FIX: Clear any stale cache entries from previous runs // This prevents cache poisoning from causing silent failures on container restart prodLog.info('🧹 Clearing cache from previous run to prevent cache poisoning') this.nounCacheManager.clear() this.verbCacheManager.clear() prodLog.info('✅ Cache cleared - starting fresh') - // v7.3.0: Progressive vs Strict initialization + // Progressive vs Strict initialization if (effectiveMode === 'progressive') { // PROGRESSIVE MODE: Fast init, background validation // Mark as initialized immediately - ready to accept operations @@ -353,7 +352,7 @@ export class GcsStorage extends BaseStorage { prodLog.info(`✅ GCS progressive init complete: ${this.bucketName} (validation deferred)`) - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() // Schedule background tasks (non-blocking) @@ -373,7 +372,7 @@ export class GcsStorage extends BaseStorage { await this.initializeCounts() this.countsLoaded = true - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() // Mark background tasks as complete (nothing to do in background) @@ -386,7 +385,7 @@ export class GcsStorage extends BaseStorage { } // ============================================= - // Progressive Initialization (v7.3.0+) + // Progressive Initialization // ============================================= /** @@ -400,7 +399,6 @@ export class GcsStorage extends BaseStorage { * * @protected * @override - * @since v7.3.0 */ protected async runBackgroundInit(): Promise { const startTime = Date.now() @@ -423,7 +421,6 @@ export class GcsStorage extends BaseStorage { * Stores result in bucketValidated/bucketValidationError for lazy use. * * @private - * @since v7.3.0 */ private async validateBucketInBackground(): Promise { try { @@ -451,7 +448,6 @@ export class GcsStorage extends BaseStorage { * Uses the existing initializeCounts() logic but in background. * * @private - * @since v7.3.0 */ private async loadCountsInBackground(): Promise { try { @@ -475,7 +471,6 @@ export class GcsStorage extends BaseStorage { * @throws Error if bucket does not exist or is not accessible * @protected * @override - * @since v7.3.0 */ protected async ensureValidatedForWrite(): Promise { // If already validated, nothing to do @@ -562,7 +557,7 @@ export class GcsStorage extends BaseStorage { } /** - * Override base class to enable smart batching for cloud storage (v3.32.3+) + * Override base class to enable smart batching for cloud storage * * GCS is cloud storage with network latency (~50ms per write). * Smart batching reduces writes from 1000 ops → 100 batches. @@ -596,7 +591,7 @@ export class GcsStorage extends BaseStorage { } } - // v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage + // Removed checkVolumeMode() - write buffering always enabled for cloud storage /** * Flush noun buffer to GCS @@ -628,20 +623,20 @@ export class GcsStorage extends BaseStorage { await Promise.all(writes) } - // v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation + // Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation /** * Save a node to storage - * v6.2.7: Always uses write buffer for consistent performance + * Always uses write buffer for consistent performance */ protected async saveNode(node: HNSWNode): Promise { await this.ensureInitialized() - // v6.2.7: Always use write buffer - cloud storage benefits from batching + // Always use write buffer - cloud storage benefits from batching if (this.nounWriteBuffer) { this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`) - // v6.2.6: Populate cache BEFORE buffering for read-after-write consistency + // Populate cache BEFORE buffering for read-after-write consistency if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) { this.nounCacheManager.set(node.id, node) } @@ -657,10 +652,10 @@ export class GcsStorage extends BaseStorage { /** * Save a node directly to GCS (bypass buffer) * - * v7.3.0: Performs lazy bucket validation on first write in progressive mode. + * Performs lazy bucket validation on first write in progressive mode. */ private async saveNodeDirect(node: HNSWNode): Promise { - // v7.3.0: Lazy bucket validation for progressive init + // Lazy bucket validation for progressive init await this.ensureValidatedForWrite() // Apply backpressure before starting operation @@ -695,14 +690,14 @@ export class GcsStorage extends BaseStorage { resumable: false // For small objects, non-resumable is faster }) - // CRITICAL FIX (v3.37.8): Only cache nodes with non-empty vectors + // CRITICAL FIX: Only cache nodes with non-empty vectors // This prevents cache pollution from HNSW's lazy-loading nodes (vector: []) if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) { this.nounCacheManager.set(node.id, node) } // Note: Empty vectors are intentional during HNSW lazy mode - not logged - // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2) + // Count tracking happens in baseStorage.saveNounMetadata_internal // This fixes the race condition where metadata didn't exist yet this.logger.trace(`Node ${node.id} saved successfully`) @@ -721,7 +716,7 @@ export class GcsStorage extends BaseStorage { } } - // v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation + // Removed getNoun_internal - now inherit from BaseStorage's type-first implementation /** * Get a node from storage @@ -732,7 +727,7 @@ export class GcsStorage extends BaseStorage { // Check cache first const cached: HNSWNode | null = await this.nounCacheManager.get(id) - // Validate cached object before returning (v3.37.8+) + // Validate cached object before returning if (cached !== undefined && cached !== null) { // Validate cached object has required fields (including non-empty vector!) if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) { @@ -831,18 +826,18 @@ export class GcsStorage extends BaseStorage { } } - // v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation + // Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation /** * Write an object to a specific path in GCS * Primitive operation required by base class * - * v7.3.0: Performs lazy bucket validation on first write in progressive mode. + * Performs lazy bucket validation on first write in progressive mode. * @protected */ protected async writeObjectToPath(path: string, data: any): Promise { await this.ensureInitialized() - // v7.3.0: Lazy bucket validation for progressive init + // Lazy bucket validation for progressive init await this.ensureValidatedForWrite() try { @@ -892,7 +887,7 @@ export class GcsStorage extends BaseStorage { } /** - * Batch read multiple objects from GCS (v5.12.0 - Cloud Storage Optimization) + * Batch read multiple objects from GCS * * **Performance**: GCS-optimized parallel downloads * - Uses Promise.all() for concurrent requests @@ -908,7 +903,6 @@ export class GcsStorage extends BaseStorage { * @returns Map of path → data (only successful reads included) * * @public - Called by baseStorage.readBatchFromAdapter() - * @since v5.12.0 */ public async readBatch(paths: string[]): Promise> { await this.ensureInitialized() @@ -960,12 +954,11 @@ export class GcsStorage extends BaseStorage { } /** - * Get GCS-specific batch configuration (v5.12.0) + * Get GCS-specific batch configuration * * GCS performs well with high concurrency due to HTTP/2 multiplexing * * @public - Overrides BaseStorage.getBatchConfig() - * @since v5.12.0 */ public getBatchConfig(): StorageBatchConfig { return { @@ -984,12 +977,12 @@ export class GcsStorage extends BaseStorage { * Delete an object from a specific path in GCS * Primitive operation required by base class * - * v7.3.0: Performs lazy bucket validation on first delete in progressive mode. + * Performs lazy bucket validation on first delete in progressive mode. * @protected */ protected async deleteObjectFromPath(path: string): Promise { await this.ensureInitialized() - // v7.3.0: Lazy bucket validation for progressive init + // Lazy bucket validation for progressive init await this.ensureValidatedForWrite() try { @@ -1034,20 +1027,20 @@ export class GcsStorage extends BaseStorage { } } - // v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation + // Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation /** * Save an edge to storage - * v6.2.7: Always uses write buffer for consistent performance + * Always uses write buffer for consistent performance */ protected async saveEdge(edge: Edge): Promise { await this.ensureInitialized() - // v6.2.7: Always use write buffer - cloud storage benefits from batching + // Always use write buffer - cloud storage benefits from batching if (this.verbWriteBuffer) { this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`) - // v6.2.6: Populate cache BEFORE buffering for read-after-write consistency + // Populate cache BEFORE buffering for read-after-write consistency this.verbCacheManager.set(edge.id, edge) await this.verbWriteBuffer.add(edge.id, edge) @@ -1061,10 +1054,10 @@ export class GcsStorage extends BaseStorage { /** * Save an edge directly to GCS (bypass buffer) * - * v7.3.0: Performs lazy bucket validation on first write in progressive mode. + * Performs lazy bucket validation on first write in progressive mode. */ private async saveEdgeDirect(edge: Edge): Promise { - // v7.3.0: Lazy bucket validation for progressive init + // Lazy bucket validation for progressive init await this.ensureValidatedForWrite() const requestId = await this.applyBackpressure() @@ -1073,7 +1066,7 @@ export class GcsStorage extends BaseStorage { this.logger.trace(`Saving edge ${edge.id}`) // Convert connections Map to serializable format - // ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file + // ARCHITECTURAL FIX: Include core relational fields in verb vector file // These fields are essential for 90% of operations - no metadata lookup needed const serializableEdge = { id: edge.id, @@ -1085,7 +1078,7 @@ export class GcsStorage extends BaseStorage { ]) ), - // CORE RELATIONAL DATA (v3.50.1+) + // CORE RELATIONAL DATA verb: edge.verb, sourceId: edge.sourceId, targetId: edge.targetId, @@ -1107,7 +1100,7 @@ export class GcsStorage extends BaseStorage { // Update cache this.verbCacheManager.set(edge.id, edge) - // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // Count tracking happens in baseStorage.saveVerbMetadata_internal // This fixes the race condition where metadata didn't exist yet this.logger.trace(`Edge ${edge.id} saved successfully`) @@ -1125,7 +1118,7 @@ export class GcsStorage extends BaseStorage { } } - // v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation + // Removed getVerb_internal - now inherit from BaseStorage's type-first implementation /** * Get an edge from storage @@ -1161,7 +1154,7 @@ export class GcsStorage extends BaseStorage { connections.set(Number(level), new Set(verbIds as string[])) } - // v4.0.0: Return HNSWVerb with core relational fields (NO metadata field) + // Return HNSWVerb with core relational fields (NO metadata field) const edge: Edge = { id: data.id, vector: data.vector, @@ -1172,7 +1165,7 @@ export class GcsStorage extends BaseStorage { sourceId: data.sourceId, targetId: data.targetId - // ✅ NO metadata field in v4.0.0 + // ✅ NO metadata field // User metadata retrieved separately via getVerbMetadata() } @@ -1201,13 +1194,13 @@ export class GcsStorage extends BaseStorage { } } - // v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation + // Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation - // v5.4.0: Removed pagination overrides - use BaseStorage's type-first implementation + // Removed pagination overrides - use BaseStorage's type-first implementation // - getNounsWithPagination, getNodesWithPagination, getVerbsWithPagination // - getNouns, getVerbs (public wrappers) - // v5.4.0: Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation + // Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation // (getNounsByNounType_internal, getVerbsBySource_internal, getVerbsByTarget_internal, getVerbsByType_internal) /** @@ -1280,7 +1273,7 @@ export class GcsStorage extends BaseStorage { } } - // v5.11.0: Clear ALL data using correct paths + // Clear ALL data using correct paths // Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks) await deleteObjectsWithPrefix('branches/') @@ -1290,7 +1283,7 @@ export class GcsStorage extends BaseStorage { // Delete system metadata await deleteObjectsWithPrefix('_system/') - // v5.11.0: Reset COW managers (but don't disable COW - it's always enabled) + // Reset COW managers (but don't disable COW - it's always enabled) // COW will re-initialize automatically on next use this.refManager = undefined this.blobStorage = undefined @@ -1352,12 +1345,12 @@ export class GcsStorage extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts + * Fixes bug where clear() doesn't persist across instance restarts * @returns true if marker object exists, false otherwise * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ @@ -1413,7 +1406,7 @@ export class GcsStorage extends BaseStorage { } } catch (error: any) { if (error.code === 404) { - // CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart) + // CRITICAL FIX: Statistics file doesn't exist yet (first restart) // Return minimal stats with counts instead of null // This prevents HNSW from seeing entityCount=0 during index rebuild this.logger.trace('Statistics file not found - returning minimal stats with counts') @@ -1607,11 +1600,11 @@ export class GcsStorage extends BaseStorage { } } - // HNSW Index Persistence (v3.35.0+) + // HNSW Index Persistence /** * Get a noun's vector for HNSW rebuild - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getNounVector(id: string): Promise { const noun = await this.getNoun(id) @@ -1621,7 +1614,7 @@ export class GcsStorage extends BaseStorage { /** * Save HNSW graph data for a noun * - * v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths) + * Uses BaseStorage's getNoun/saveNoun (type-first paths) * CRITICAL: Uses mutex locking to prevent read-modify-write races */ public async saveHNSWData(nounId: string, hnswData: { @@ -1630,7 +1623,7 @@ export class GcsStorage extends BaseStorage { }): Promise { const lockKey = `hnsw/${nounId}` - // CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races + // CRITICAL FIX: Mutex lock to prevent read-modify-write races // Problem: Without mutex, concurrent operations can: // 1. Thread A reads noun (connections: [1,2,3]) // 2. Thread B reads noun (connections: [1,2,3]) @@ -1650,7 +1643,7 @@ export class GcsStorage extends BaseStorage { this.hnswLocks.set(lockKey, lockPromise) try { - // v5.4.0: Use BaseStorage's getNoun (type-first paths) + // Use BaseStorage's getNoun (type-first paths) // Read existing noun data (if exists) const existingNoun = await this.getNoun(nounId) @@ -1672,7 +1665,7 @@ export class GcsStorage extends BaseStorage { connections: connectionsMap } - // v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch) + // Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch) await this.saveNoun(updatedNoun) } finally { // Release lock (ALWAYS runs, even if error thrown) @@ -1683,7 +1676,7 @@ export class GcsStorage extends BaseStorage { /** * Get HNSW graph data for a noun - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getHNSWData(nounId: string): Promise<{ level: number @@ -1713,7 +1706,7 @@ export class GcsStorage extends BaseStorage { * Save HNSW system data (entry point, max level) * Storage path: system/hnsw-system.json * - * CRITICAL FIX (v4.10.1): Optimistic locking with generation numbers to prevent race conditions + * CRITICAL FIX: Optimistic locking with generation numbers to prevent race conditions */ public async saveHNSWSystem(systemData: { entryPointId: string | null @@ -1807,7 +1800,7 @@ export class GcsStorage extends BaseStorage { } // ============================================================================ - // GCS Lifecycle Management & Autoclass (v4.0.0) + // GCS Lifecycle Management & Autoclass // Cost optimization through automatic tier transitions and Autoclass // ============================================================================ diff --git a/src/storage/adapters/historicalStorageAdapter.ts b/src/storage/adapters/historicalStorageAdapter.ts index 150c7c5c..0363738a 100644 --- a/src/storage/adapters/historicalStorageAdapter.ts +++ b/src/storage/adapters/historicalStorageAdapter.ts @@ -25,7 +25,7 @@ * - Lazy loading: only loads entities when accessed * - No eager-loading of entire commit state * - * v5.4.0: Production-ready, billion-scale historical queries + * Production-ready, billion-scale historical queries */ import { BaseStorage } from '../baseStorage.js' @@ -144,7 +144,7 @@ export class HistoricalStorageAdapter extends BaseStorage { */ public async init(): Promise { // Get COW components from underlying storage - // v6.2.4: Fixed property names - use public properties without underscore prefix + // Fixed property names - use public properties without underscore prefix this.commitLog = this.underlyingStorage.commitLog this.blobStorage = this.underlyingStorage.blobStorage @@ -161,7 +161,7 @@ export class HistoricalStorageAdapter extends BaseStorage { throw new Error(`Commit not found: ${this.commitId}`) } - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() } @@ -310,12 +310,12 @@ export class HistoricalStorageAdapter extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: No-op for HistoricalStorageAdapter (read-only, doesn't manage COW) + * No-op for HistoricalStorageAdapter (read-only, doesn't manage COW) * @returns Always false (read-only adapter doesn't manage COW state) * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index c0ca0daa..03798bea 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -24,7 +24,7 @@ import { PaginatedResult } from '../../types/paginationTypes.js' * Uses Maps to store data in memory */ export class MemoryStorage extends BaseStorage { - // v5.4.0: Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths + // Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths private statistics: StatisticsData | null = null // Unified object store for primitive operations (replaces metadata, nounMetadata, verbMetadata) @@ -55,7 +55,6 @@ export class MemoryStorage extends BaseStorage { * - Parallel processing maximizes throughput * * @returns Memory-optimized batch configuration - * @since v4.11.0 */ public getBatchConfig(): StorageBatchConfig { return { @@ -72,27 +71,27 @@ export class MemoryStorage extends BaseStorage { /** * Initialize the storage adapter - * v6.0.0: Calls super.init() to initialize GraphAdjacencyIndex and type statistics + * Calls super.init() to initialize GraphAdjacencyIndex and type statistics */ public async init(): Promise { await super.init() } - // v5.4.0: Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation + // Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation /** * Get nouns with pagination and filtering - * v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field) + * Returns HNSWNounWithMetadata[] (includes metadata field) * @param options Pagination and filtering options * @returns Promise that resolves to a paginated result of nouns with metadata */ - // v5.4.0: Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation + // Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation - // v5.4.0: Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation + // Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation - // v5.4.0: Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation + // Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation - // v5.4.0: Removed verb *_internal method overrides - using BaseStorage's type-first implementation + // Removed verb *_internal method overrides - using BaseStorage's type-first implementation /** * Primitive operation: Write object to path @@ -159,8 +158,8 @@ export class MemoryStorage extends BaseStorage { /** * Clear all data from storage - * v5.4.0: Clears objectStore (type-first paths) - * v7.3.1: Also clears writeCache to prevent stale data after clear + * Clears objectStore (type-first paths) + * Also clears writeCache to prevent stale data after clear */ public async clear(): Promise { this.objectStore.clear() @@ -174,7 +173,7 @@ export class MemoryStorage extends BaseStorage { this.statisticsCache = null this.statisticsModified = false - // v7.3.1: Clear write-through cache (inherited from BaseStorage) + // Clear write-through cache (inherited from BaseStorage) // Without this, readWithInheritance() would return stale cached data // after clear(), causing "ghost" entities to appear this.clearWriteCache() @@ -182,7 +181,7 @@ export class MemoryStorage extends BaseStorage { /** * Get information about storage usage and capacity - * v5.4.0: Uses BaseStorage counts + * Uses BaseStorage counts */ public async getStorageStatus(): Promise<{ type: string @@ -204,12 +203,12 @@ export class MemoryStorage extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: No-op for MemoryStorage (doesn't persist) + * No-op for MemoryStorage (doesn't persist) * @returns Always false (marker doesn't persist in memory) * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ @@ -252,7 +251,7 @@ export class MemoryStorage extends BaseStorage { */ protected async getStatisticsData(): Promise { if (!this.statistics) { - // CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init) + // CRITICAL FIX: Statistics don't exist yet (first init) // Return minimal stats with counts instead of null // This prevents HNSW from seeing entityCount=0 during index rebuild return { @@ -299,10 +298,10 @@ export class MemoryStorage extends BaseStorage { } /** - * Initialize counts from in-memory storage - O(1) operation (v4.0.0) + * Initialize counts from in-memory storage - O(1) operation */ protected async initializeCounts(): Promise { - // v6.0.0: Scan objectStore paths (ID-first structure) to count entities + // Scan objectStore paths (ID-first structure) to count entities this.entityCounts.clear() this.verbCounts.clear() @@ -314,14 +313,14 @@ export class MemoryStorage extends BaseStorage { // Count nouns (entities/nouns/{shard}/{id}/vectors.json) const nounMatch = path.match(/^entities\/nouns\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/) if (nounMatch) { - // v6.0.0: Type is in metadata, not path - just count total + // Type is in metadata, not path - just count total totalNouns++ } // Count verbs (entities/verbs/{shard}/{id}/vectors.json) const verbMatch = path.match(/^entities\/verbs\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/) if (verbMatch) { - // v6.0.0: Type is in metadata, not path - just count total + // Type is in metadata, not path - just count total totalVerbs++ } } @@ -339,26 +338,26 @@ export class MemoryStorage extends BaseStorage { } // ============================================= - // HNSW Index Persistence (v3.35.0+) + // HNSW Index Persistence // ============================================= /** * Get vector for a noun - * v5.4.0: Uses BaseStorage's type-first implementation + * Uses BaseStorage's type-first implementation */ public async getNounVector(id: string): Promise { const noun = await this.getNoun(id) return noun ? [...noun.vector] : null } - // CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control + // CRITICAL FIX: Mutex locks for HNSW concurrency control // Even in-memory operations need serialization to prevent async race conditions private hnswLocks = new Map>() /** * Save HNSW graph data for a noun * - * CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions during concurrent HNSW updates + * CRITICAL FIX: Mutex locking to prevent race conditions during concurrent HNSW updates * Even in-memory operations can race due to async/await interleaving * Prevents data corruption when multiple entities connect to same neighbor simultaneously */ @@ -417,7 +416,7 @@ export class MemoryStorage extends BaseStorage { /** * Save HNSW system data (entry point, max level) * - * CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions + * CRITICAL FIX: Mutex locking to prevent race conditions */ public async saveHNSWSystem(systemData: { entryPointId: string | null diff --git a/src/storage/adapters/opfsStorage.ts b/src/storage/adapters/opfsStorage.ts index 3ced13ed..7139d82a 100644 --- a/src/storage/adapters/opfsStorage.ts +++ b/src/storage/adapters/opfsStorage.ts @@ -57,7 +57,7 @@ const ROOT_DIR = 'opfs-vector-db' * OPFS storage adapter for browser environments * Uses the Origin Private File System API to store data persistently * - * v5.4.0: Type-aware storage now built into BaseStorage + * Type-aware storage now built into BaseStorage * - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation) * - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination) * - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths) @@ -97,7 +97,6 @@ export class OPFSStorage extends BaseStorage { * - Sequential processing preferred for stability * * @returns OPFS-optimized batch configuration - * @since v4.11.0 */ public getBatchConfig(): StorageBatchConfig { return { @@ -172,7 +171,7 @@ export class OPFSStorage extends BaseStorage { // Initialize counts from storage await this.initializeCounts() - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() } catch (error) { console.error('Failed to initialize OPFS storage:', error) @@ -232,7 +231,7 @@ export class OPFSStorage extends BaseStorage { } } - // v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation + // Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation /** * Delete an edge from storage @@ -482,14 +481,14 @@ export class OPFSStorage extends BaseStorage { // Remove all files in the index directory await removeDirectoryContents(this.indexDir!) - // v5.6.1: Remove COW (copy-on-write) version control data + // Remove COW (copy-on-write) version control data // This directory stores all git-like versioning data (commits, trees, blobs, refs) // Must be deleted to fully clear all data including version history try { // Delete the entire _cow/ directory (not just contents) await this.rootDir!.removeEntry('_cow', { recursive: true }) - // v5.11.0: Reset COW managers (but don't disable COW - it's always enabled) + // Reset COW managers (but don't disable COW - it's always enabled) // COW will re-initialize automatically on next use this.refManager = undefined this.blobStorage = undefined @@ -505,7 +504,7 @@ export class OPFSStorage extends BaseStorage { this.statisticsCache = null this.statisticsModified = false - // v5.6.1: Reset entity counters (inherited from BaseStorageAdapter) + // Reset entity counters (inherited from BaseStorageAdapter) // These in-memory counters must be reset to 0 after clearing all data ;(this as any).totalNounCount = 0 ;(this as any).totalVerbCount = 0 @@ -517,16 +516,16 @@ export class OPFSStorage extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts + * Fixes bug where clear() doesn't persist across instance restarts * @returns true if marker file exists, false otherwise * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ - // Quota monitoring configuration (v4.0.0) + // Quota monitoring configuration private quotaWarningThreshold = 0.8 // Warn at 80% usage private quotaCriticalThreshold = 0.95 // Critical at 95% usage private lastQuotaCheck: number = 0 @@ -675,7 +674,7 @@ export class OPFSStorage extends BaseStorage { } /** - * Get detailed quota status with warnings (v4.0.0) + * Get detailed quota status with warnings * Monitors storage usage and warns when approaching quota limits * * @returns Promise that resolves to quota status with warning levels @@ -769,7 +768,7 @@ export class OPFSStorage extends BaseStorage { } /** - * Monitor quota during operations (v4.0.0) + * Monitor quota during operations * Automatically checks quota at regular intervals and warns if approaching limits * Call this before write operations to ensure quota is available * @@ -1185,7 +1184,7 @@ export class OPFSStorage extends BaseStorage { } } } catch (error) { - // CRITICAL FIX (v3.37.4): No statistics files exist (first init) + // CRITICAL FIX: No statistics files exist (first init) // Return minimal stats with counts instead of null // This prevents HNSW from seeing entityCount=0 during index rebuild return { @@ -1228,7 +1227,7 @@ export class OPFSStorage extends BaseStorage { * @param options Pagination and filter options * @returns Promise that resolves to a paginated result of nouns */ - // v5.4.0: Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation + // Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation /** * Initialize counts from OPFS storage @@ -1313,28 +1312,28 @@ export class OPFSStorage extends BaseStorage { } } - // HNSW Index Persistence (v3.35.0+) + // HNSW Index Persistence /** * Get a noun's vector for HNSW rebuild */ /** * Get vector for a noun - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getNounVector(id: string): Promise { const noun = await this.getNoun(id) return noun ? noun.vector : null } - // CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control + // CRITICAL FIX: Mutex locks for HNSW concurrency control // Browser environments are single-threaded but async operations can still interleave private hnswLocks = new Map>() /** * Save HNSW graph data for a noun * - * v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths) + * Uses BaseStorage's getNoun/saveNoun (type-first paths) * CRITICAL: Preserves mutex locking to prevent read-modify-write races */ public async saveHNSWData(nounId: string, hnswData: { @@ -1354,7 +1353,7 @@ export class OPFSStorage extends BaseStorage { this.hnswLocks.set(lockKey, lockPromise) try { - // v5.4.0: Use BaseStorage's getNoun (type-first paths) + // Use BaseStorage's getNoun (type-first paths) const existingNoun = await this.getNoun(nounId) if (!existingNoun) { @@ -1374,7 +1373,7 @@ export class OPFSStorage extends BaseStorage { connections: connectionsMap } - // v5.4.0: Use BaseStorage's saveNoun (type-first paths) + // Use BaseStorage's saveNoun (type-first paths) await this.saveNoun(updatedNoun) } finally { // Release lock @@ -1385,7 +1384,7 @@ export class OPFSStorage extends BaseStorage { /** * Get HNSW graph data for a noun - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getHNSWData(nounId: string): Promise<{ level: number @@ -1415,7 +1414,7 @@ export class OPFSStorage extends BaseStorage { * Save HNSW system data (entry point, max level) * Storage path: index/hnsw-system.json * - * CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions + * CRITICAL FIX: Mutex locking to prevent race conditions */ public async saveHNSWSystem(systemData: { entryPointId: string | null diff --git a/src/storage/adapters/optimizedS3Search.ts b/src/storage/adapters/optimizedS3Search.ts index 6b1f0dc4..0d88b966 100644 --- a/src/storage/adapters/optimizedS3Search.ts +++ b/src/storage/adapters/optimizedS3Search.ts @@ -107,7 +107,7 @@ export class OptimizedS3Search { } // Determine if there are more items - const hasMore = listResult.hasMore || nouns.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop) + const hasMore = listResult.hasMore || nouns.length > limit // Fixed >= to > (was causing infinite loop) // Set next cursor let nextCursor: string | undefined @@ -190,7 +190,7 @@ export class OptimizedS3Search { } // Determine if there are more items - const hasMore = listResult.hasMore || verbs.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop) + const hasMore = listResult.hasMore || verbs.length > limit // Fixed >= to > (was causing infinite loop) // Set next cursor let nextCursor: string | undefined diff --git a/src/storage/adapters/r2Storage.ts b/src/storage/adapters/r2Storage.ts index 49b6f8c0..73ac4500 100644 --- a/src/storage/adapters/r2Storage.ts +++ b/src/storage/adapters/r2Storage.ts @@ -58,7 +58,7 @@ const MAX_R2_PAGE_SIZE = 1000 * Dedicated Cloudflare R2 storage adapter * Optimized for R2's unique characteristics and global edge network * - * v5.4.0: Type-aware storage now built into BaseStorage + * Type-aware storage now built into BaseStorage * - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation) * - Removed getNounsWithPagination override * - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths) @@ -112,7 +112,7 @@ export class R2Storage extends BaseStorage { // Request coalescer for deduplication private requestCoalescer: RequestCoalescer | null = null - // v6.2.7: Write buffering always enabled for consistent performance + // Write buffering always enabled for consistent performance // Removes dynamic mode switching complexity - cloud storage always benefits from batching // Multi-level cache manager for efficient data access @@ -122,7 +122,7 @@ export class R2Storage extends BaseStorage { // Module logger private logger = createModuleLogger('R2Storage') - // v5.4.0: HNSW mutex locks to prevent read-modify-write races + // HNSW mutex locks to prevent read-modify-write races private hnswLocks = new Map>() /** @@ -168,7 +168,7 @@ export class R2Storage extends BaseStorage { }) this.verbCacheManager = new CacheManager(options.cacheConfig) - // v6.2.7: Write buffering always enabled - no env var check needed + // Write buffering always enabled - no env var check needed } /** @@ -183,7 +183,7 @@ export class R2Storage extends BaseStorage { * Zero egress fees enable aggressive caching and parallel downloads * * @returns R2-optimized batch configuration - * @since v5.12.0 - Updated for native batch API + * Updated for native batch API */ public getBatchConfig(): StorageBatchConfig { return { @@ -208,7 +208,6 @@ export class R2Storage extends BaseStorage { * * @param paths - Array of R2 object keys to read * @returns Map of path -> parsed JSON data (only successful reads) - * @since v5.12.0 */ public async readBatch(paths: string[]): Promise> { await this.ensureInitialized() @@ -336,7 +335,7 @@ export class R2Storage extends BaseStorage { this.nounCacheManager.clear() this.verbCacheManager.clear() - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() } catch (error) { this.logger.error('Failed to initialize R2 storage:', error) @@ -409,7 +408,7 @@ export class R2Storage extends BaseStorage { } } - // v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage + // Removed checkVolumeMode() - write buffering always enabled for cloud storage /** * Flush noun buffer to R2 @@ -444,16 +443,16 @@ export class R2Storage extends BaseStorage { /** * Save a node to storage - * v6.2.7: Always uses write buffer for consistent performance + * Always uses write buffer for consistent performance */ protected async saveNode(node: HNSWNode): Promise { await this.ensureInitialized() - // v6.2.7: Always use write buffer - cloud storage benefits from batching + // Always use write buffer - cloud storage benefits from batching if (this.nounWriteBuffer) { this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`) - // v6.2.6: Populate cache BEFORE buffering for read-after-write consistency + // Populate cache BEFORE buffering for read-after-write consistency if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) { this.nounCacheManager.set(node.id, node) } @@ -728,14 +727,14 @@ export class R2Storage extends BaseStorage { /** * Save an edge to storage - * v6.2.7: Always uses write buffer for consistent performance + * Always uses write buffer for consistent performance */ protected async saveEdge(edge: Edge): Promise { await this.ensureInitialized() - // v6.2.7: Always use write buffer - cloud storage benefits from batching + // Always use write buffer - cloud storage benefits from batching if (this.verbWriteBuffer) { - // v6.2.6: Populate cache BEFORE buffering for read-after-write consistency + // Populate cache BEFORE buffering for read-after-write consistency this.verbCacheManager.set(edge.id, edge) await this.verbWriteBuffer.add(edge.id, edge) @@ -750,7 +749,7 @@ export class R2Storage extends BaseStorage { const requestId = await this.applyBackpressure() try { - // ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file + // ARCHITECTURAL FIX: Include core relational fields in verb vector file // These fields are essential for 90% of operations - no metadata lookup needed const serializableEdge = { id: edge.id, @@ -762,7 +761,7 @@ export class R2Storage extends BaseStorage { ]) ), - // CORE RELATIONAL DATA (v3.50.1+) + // CORE RELATIONAL DATA verb: edge.verb, sourceId: edge.sourceId, targetId: edge.targetId, @@ -785,7 +784,7 @@ export class R2Storage extends BaseStorage { this.verbCacheManager.set(edge.id, edge) - // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // Count tracking happens in baseStorage.saveVerbMetadata_internal // This fixes the race condition where metadata didn't exist yet this.releaseBackpressure(true, requestId) @@ -831,7 +830,7 @@ export class R2Storage extends BaseStorage { connections.set(Number(level), new Set(verbIds as string[])) } - // v4.0.0: Return HNSWVerb with core relational fields (NO metadata field) + // Return HNSWVerb with core relational fields (NO metadata field) const edge: Edge = { id: data.id, vector: data.vector, @@ -842,7 +841,7 @@ export class R2Storage extends BaseStorage { sourceId: data.sourceId, targetId: data.targetId - // ✅ NO metadata field in v4.0.0 + // ✅ NO metadata field // User metadata retrieved separately via getVerbMetadata() } @@ -1081,7 +1080,7 @@ export class R2Storage extends BaseStorage { prodLog.info('🧹 R2: Clearing all data from bucket...') - // v5.11.0: Clear ALL data using correct paths + // Clear ALL data using correct paths // Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks) const branchObjects = await this.listObjectsUnderPath('branches/') for (const key of branchObjects) { @@ -1100,7 +1099,7 @@ export class R2Storage extends BaseStorage { await this.deleteObjectFromPath(key) } - // v5.11.0: Reset COW managers (but don't disable COW - it's always enabled) + // Reset COW managers (but don't disable COW - it's always enabled) // COW will re-initialize automatically on next use this.refManager = undefined this.blobStorage = undefined @@ -1143,17 +1142,17 @@ export class R2Storage extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts + * Fixes bug where clear() doesn't persist across instance restarts * @returns true if marker object exists, false otherwise * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ - // v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation + // Removed getNounsWithPagination override - use BaseStorage's type-first implementation - // v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation + // Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation } diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index 13112cf4..92c531b7 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -85,7 +85,7 @@ type S3Command = any * - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com') * - bucketName: GCS bucket name * - * v5.4.0: Type-aware storage now built into BaseStorage + * Type-aware storage now built into BaseStorage * - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation) * - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination) * - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths) @@ -152,7 +152,7 @@ export class S3CompatibleStorage extends BaseStorage { // Request coalescer for deduplication private requestCoalescer: RequestCoalescer | null = null - // v6.2.7: Write buffering always enabled for consistent performance + // Write buffering always enabled for consistent performance // Removes dynamic mode switching complexity - cloud storage always benefits from batching // Operation executors for timeout and retry handling @@ -165,7 +165,7 @@ export class S3CompatibleStorage extends BaseStorage { // Module logger private logger = createModuleLogger('S3Storage') - // v5.4.0: HNSW mutex locks to prevent read-modify-write races + // HNSW mutex locks to prevent read-modify-write races private hnswLocks = new Map>() /** @@ -210,7 +210,7 @@ export class S3CompatibleStorage extends BaseStorage { } /** - * Initialization mode for fast cold starts (v7.3.0+) + * Initialization mode for fast cold starts * * - `'auto'` (default): Progressive in cloud environments (Lambda, Cloud Run), * strict locally. Zero-config optimization. @@ -219,7 +219,6 @@ export class S3CompatibleStorage extends BaseStorage { * - `'strict'`: Traditional blocking init. Validates bucket and loads counts * before init() returns. * - * @since v7.3.0 */ initMode?: InitMode @@ -236,7 +235,7 @@ export class S3CompatibleStorage extends BaseStorage { this.serviceType = options.serviceType || 's3' this.readOnly = options.readOnly || false - // v7.3.0: Handle initMode + // Handle initMode if (options.initMode) { this.initMode = options.initMode } @@ -270,7 +269,7 @@ export class S3CompatibleStorage extends BaseStorage { * S3 supports ~5000 operations/second with burst capacity up to 10,000 * * @returns S3-optimized batch configuration - * @since v5.12.0 - Updated for native batch API + * Updated for native batch API */ public getBatchConfig(): StorageBatchConfig { return { @@ -295,7 +294,6 @@ export class S3CompatibleStorage extends BaseStorage { * * @param paths - Array of S3 object keys to read * @returns Map of path -> parsed JSON data (only successful reads) - * @since v5.12.0 */ public async readBatch(paths: string[]): Promise> { await this.ensureInitialized() @@ -358,7 +356,7 @@ export class S3CompatibleStorage extends BaseStorage { /** * Initialize the storage adapter * - * v7.3.0: Supports progressive initialization for fast cold starts + * Supports progressive initialization for fast cold starts * * | Mode | Init Time | When | * |------|-----------|------| @@ -514,7 +512,7 @@ export class S3CompatibleStorage extends BaseStorage { // Initialize request coalescer this.initializeCoalescer() - // CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs + // CRITICAL FIX: Clear any stale cache entries from previous runs // This prevents cache poisoning from causing silent failures on container restart const nodeCacheSize = this.nodeCache?.size || 0 if (nodeCacheSize > 0) { @@ -524,7 +522,7 @@ export class S3CompatibleStorage extends BaseStorage { prodLog.info('🧹 Node cache is empty - starting fresh') } - // v7.3.0: Progressive vs Strict initialization + // Progressive vs Strict initialization if (effectiveMode === 'progressive') { // PROGRESSIVE MODE: Fast init, background validation // Mark as initialized immediately - ready to accept operations @@ -533,7 +531,7 @@ export class S3CompatibleStorage extends BaseStorage { prodLog.info(`✅ S3 progressive init complete: ${this.bucketName} (validation deferred)`) - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() // Schedule background tasks (non-blocking) @@ -556,7 +554,7 @@ export class S3CompatibleStorage extends BaseStorage { await this.initializeCounts() this.countsLoaded = true - // v6.0.0: Initialize GraphAdjacencyIndex and type statistics + // Initialize GraphAdjacencyIndex and type statistics await super.init() // Mark background tasks as complete (nothing to do in background) @@ -573,7 +571,7 @@ export class S3CompatibleStorage extends BaseStorage { } // ============================================= - // Progressive Initialization (v7.3.0+) + // Progressive Initialization // ============================================= /** @@ -588,7 +586,6 @@ export class S3CompatibleStorage extends BaseStorage { * * @protected * @override - * @since v7.3.0 */ protected async runBackgroundInit(): Promise { const startTime = Date.now() @@ -614,7 +611,6 @@ export class S3CompatibleStorage extends BaseStorage { * Stores result in bucketValidated/bucketValidationError for lazy use. * * @private - * @since v7.3.0 */ private async validateBucketInBackground(): Promise { try { @@ -638,7 +634,6 @@ export class S3CompatibleStorage extends BaseStorage { * Load counts from storage in background. * * @private - * @since v7.3.0 */ private async loadCountsInBackground(): Promise { try { @@ -662,7 +657,6 @@ export class S3CompatibleStorage extends BaseStorage { * @throws Error if bucket does not exist or is not accessible * @protected * @override - * @since v7.3.0 */ protected async ensureValidatedForWrite(): Promise { // If already validated, nothing to do @@ -918,7 +912,7 @@ export class S3CompatibleStorage extends BaseStorage { ) } - // v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage + // Removed checkVolumeMode() - write buffering always enabled for cloud storage /** * Bulk write nouns to S3 @@ -1179,20 +1173,20 @@ export class S3CompatibleStorage extends BaseStorage { return this.socketManager.getBatchSize() } - // v5.4.0: Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation + // Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation /** * Save a node to storage - * v6.2.7: Always uses write buffer for consistent performance + * Always uses write buffer for consistent performance */ protected async saveNode(node: HNSWNode): Promise { await this.ensureInitialized() - // v6.2.7: Always use write buffer - cloud storage benefits from batching + // Always use write buffer - cloud storage benefits from batching if (this.nounWriteBuffer) { this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`) - // v6.2.6: Populate cache BEFORE buffering for read-after-write consistency + // Populate cache BEFORE buffering for read-after-write consistency if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) { this.nounCacheManager.set(node.id, node) } @@ -1242,7 +1236,7 @@ export class S3CompatibleStorage extends BaseStorage { this.logger.debug(`Node ${node.id} saved successfully`) - // Log the change for efficient synchronization (v4.0.0: no metadata on node) + // Log the change for efficient synchronization (no metadata on node) await this.appendToChangeLog({ timestamp: Date.now(), operation: 'add', // Could be 'update' if we track existing nodes @@ -1250,7 +1244,7 @@ export class S3CompatibleStorage extends BaseStorage { entityId: node.id, data: { vector: node.vector - // ✅ NO metadata field in v4.0.0 - stored separately + // ✅ NO metadata field - stored separately } }) @@ -1296,7 +1290,7 @@ export class S3CompatibleStorage extends BaseStorage { } } - // v5.4.0: Removed getNoun_internal override - uses BaseStorage type-first implementation + // Removed getNoun_internal override - uses BaseStorage type-first implementation /** * Get a node from storage @@ -1307,7 +1301,7 @@ export class S3CompatibleStorage extends BaseStorage { // Check cache first const cached = this.nodeCache.get(id) - // Validate cached object before returning (v3.37.8+) + // Validate cached object before returning if (cached !== undefined && cached !== null) { // Validate cached object has required fields (including non-empty vector!) if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) { @@ -1607,7 +1601,7 @@ export class S3CompatibleStorage extends BaseStorage { return nodes } - // v5.4.0: Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal) + // Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal) // Now inherit from BaseStorage's type-first implementation @@ -1788,23 +1782,23 @@ export class S3CompatibleStorage extends BaseStorage { return true // Return all edges since filtering requires metadata } - // v5.4.0: Removed getVerbsWithPagination override - use BaseStorage's type-first implementation + // Removed getVerbsWithPagination override - use BaseStorage's type-first implementation - // v5.4.0: Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb) + // Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb) // Total: 8 *_internal methods removed - all now inherit from BaseStorage's type-first implementation /** * Primitive operation: Write object to path * All metadata operations use this internally via base class routing * - * v7.3.0: Performs lazy bucket validation on first write in progressive mode. + * Performs lazy bucket validation on first write in progressive mode. */ protected async writeObjectToPath(path: string, data: any): Promise { await this.ensureInitialized() - // v7.3.0: Lazy bucket validation for progressive init + // Lazy bucket validation for progressive init await this.ensureValidatedForWrite() // Apply backpressure before starting operation @@ -1897,11 +1891,11 @@ export class S3CompatibleStorage extends BaseStorage { * Primitive operation: Delete object from path * All metadata operations use this internally via base class routing * - * v7.3.0: Performs lazy bucket validation on first delete in progressive mode. + * Performs lazy bucket validation on first delete in progressive mode. */ protected async deleteObjectFromPath(path: string): Promise { await this.ensureInitialized() - // v7.3.0: Lazy bucket validation for progressive init + // Lazy bucket validation for progressive init await this.ensureValidatedForWrite() try { @@ -2315,7 +2309,7 @@ export class S3CompatibleStorage extends BaseStorage { } } - // v5.11.0: Clear ALL data using correct paths + // Clear ALL data using correct paths // Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks) await deleteObjectsWithPrefix('branches/') @@ -2325,7 +2319,7 @@ export class S3CompatibleStorage extends BaseStorage { // Delete system metadata await deleteObjectsWithPrefix('_system/') - // v5.11.0: Reset COW managers (but don't disable COW - it's always enabled) + // Reset COW managers (but don't disable COW - it's always enabled) // COW will re-initialize automatically on next use this.refManager = undefined this.blobStorage = undefined @@ -2335,7 +2329,7 @@ export class S3CompatibleStorage extends BaseStorage { this.statisticsCache = null this.statisticsModified = false - // v5.6.1: Reset entity counters (inherited from BaseStorageAdapter) + // Reset entity counters (inherited from BaseStorageAdapter) // These in-memory counters must be reset to 0 after clearing all data ;(this as any).totalNounCount = 0 ;(this as any).totalVerbCount = 0 @@ -2457,12 +2451,12 @@ export class S3CompatibleStorage extends BaseStorage { /** * Check if COW has been explicitly disabled via clear() - * v5.10.4: Fixes bug where clear() doesn't persist across instance restarts + * Fixes bug where clear() doesn't persist across instance restarts * @returns true if marker object exists, false otherwise * @protected */ /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() methods + * Removed checkClearMarker() and createClearMarker() methods * COW is now always enabled - marker files are no longer used */ @@ -2851,7 +2845,7 @@ export class S3CompatibleStorage extends BaseStorage { totalEdges: this.totalVerbCount } } - // CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart) + // CRITICAL FIX: Statistics file doesn't exist yet (first restart) // Return minimal stats with counts instead of null // This prevents HNSW from seeing entityCount=0 during index rebuild return { @@ -3415,7 +3409,7 @@ export class S3CompatibleStorage extends BaseStorage { } } - // v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation + // Removed getNounsWithPagination override - use BaseStorage's type-first implementation /** * Estimate total noun count by listing objects across all shards @@ -3549,7 +3543,7 @@ export class S3CompatibleStorage extends BaseStorage { } /** - * Override base class to enable smart batching for cloud storage (v3.32.3+) + * Override base class to enable smart batching for cloud storage * * S3 is cloud storage with network latency (~50ms per write). * Smart batching reduces writes from 1000 ops → 100 batches. @@ -3560,11 +3554,11 @@ export class S3CompatibleStorage extends BaseStorage { return true // S3 benefits from batching } - // HNSW Index Persistence (v3.35.0+) + // HNSW Index Persistence /** * Get a noun's vector for HNSW rebuild - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getNounVector(id: string): Promise { const noun = await this.getNoun(id) @@ -3574,7 +3568,7 @@ export class S3CompatibleStorage extends BaseStorage { /** * Save HNSW graph data for a noun * - * v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths) + * Uses BaseStorage's getNoun/saveNoun (type-first paths) * CRITICAL: Uses mutex locking to prevent read-modify-write races */ public async saveHNSWData(nounId: string, hnswData: { @@ -3583,7 +3577,7 @@ export class S3CompatibleStorage extends BaseStorage { }): Promise { const lockKey = `hnsw/${nounId}` - // CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races + // CRITICAL FIX: Mutex lock to prevent read-modify-write races // Problem: Without mutex, concurrent operations can: // 1. Thread A reads noun (connections: [1,2,3]) // 2. Thread B reads noun (connections: [1,2,3]) @@ -3603,7 +3597,7 @@ export class S3CompatibleStorage extends BaseStorage { this.hnswLocks.set(lockKey, lockPromise) try { - // v5.4.0: Use BaseStorage's getNoun (type-first paths) + // Use BaseStorage's getNoun (type-first paths) // Read existing noun data (if exists) const existingNoun = await this.getNoun(nounId) @@ -3625,7 +3619,7 @@ export class S3CompatibleStorage extends BaseStorage { connections: connectionsMap } - // v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch) + // Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch) await this.saveNoun(updatedNoun) } finally { // Release lock (ALWAYS runs, even if error thrown) @@ -3636,7 +3630,7 @@ export class S3CompatibleStorage extends BaseStorage { /** * Get HNSW graph data for a noun - * v5.4.0: Uses BaseStorage's getNoun (type-first paths) + * Uses BaseStorage's getNoun (type-first paths) */ public async getHNSWData(nounId: string): Promise<{ level: number @@ -3666,7 +3660,7 @@ export class S3CompatibleStorage extends BaseStorage { * Save HNSW system data (entry point, max level) * Storage path: system/hnsw-system.json * - * CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions + * CRITICAL FIX: Optimistic locking with ETags to prevent race conditions */ public async saveHNSWSystem(systemData: { entryPointId: string | null @@ -3781,7 +3775,7 @@ export class S3CompatibleStorage extends BaseStorage { } /** - * Set S3 lifecycle policy for automatic tier transitions and deletions (v4.0.0) + * Set S3 lifecycle policy for automatic tier transitions and deletions * Automates cost optimization by moving old data to cheaper storage classes * * S3 Storage Classes: @@ -3976,7 +3970,7 @@ export class S3CompatibleStorage extends BaseStorage { } /** - * Enable S3 Intelligent-Tiering for automatic cost optimization (v4.0.0) + * Enable S3 Intelligent-Tiering for automatic cost optimization * Automatically moves objects between access tiers based on usage patterns * * Intelligent-Tiering automatically saves up to 95% on storage costs: diff --git a/src/storage/backwardCompatibility.ts b/src/storage/backwardCompatibility.ts index c840fe3a..6acfae84 100644 --- a/src/storage/backwardCompatibility.ts +++ b/src/storage/backwardCompatibility.ts @@ -1,6 +1,6 @@ /** - * DEPRECATED (v4.7.2): Backward compatibility stubs - * TODO: Remove in v4.7.3 after migrating s3CompatibleStorage + * DEPRECATED: Backward compatibility stubs + * TODO: Remove after migrating s3CompatibleStorage */ export class StorageCompatibilityLayer { diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 7245ddd2..f2bc3c22 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -48,7 +48,6 @@ interface StorageKeyInfo { * Each storage adapter declares its optimal batch behavior for rate limiting * and performance optimization * - * @since v4.11.0 */ export interface StorageBatchConfig { /** Maximum items per batch */ @@ -73,15 +72,15 @@ export interface StorageBatchConfig { } } -// Clean directory structure (v4.7.2+) +// Clean directory structure // All storage adapters use this consistent structure export const NOUNS_METADATA_DIR = 'entities/nouns/metadata' export const VERBS_METADATA_DIR = 'entities/verbs/metadata' export const SYSTEM_DIR = '_system' export const STATISTICS_KEY = 'statistics' -// DEPRECATED (v4.7.2): Temporary stubs for adapters not yet migrated -// TODO: Remove in v4.7.3 after migrating remaining adapters +// DEPRECATED: Temporary stubs for adapters not yet migrated +// TODO: Remove after migrating remaining adapters export const NOUNS_DIR = 'entities/nouns/hnsw' export const VERBS_DIR = 'entities/verbs/hnsw' export const METADATA_DIR = 'entities/nouns/metadata' @@ -97,12 +96,12 @@ export function getDirectoryPath(entityType: 'noun' | 'verb', dataType: 'vector' } /** - * Type-first path generators (v5.4.0) + * Type-first path generators * Built-in type-aware organization for all storage adapters */ /** - * Get ID-first path for noun vectors (v6.0.0) + * Get ID-first path for noun vectors * No type parameter needed - direct O(1) lookup by ID */ function getNounVectorPath(id: string): string { @@ -111,7 +110,7 @@ function getNounVectorPath(id: string): string { } /** - * Get ID-first path for noun metadata (v6.0.0) + * Get ID-first path for noun metadata * No type parameter needed - direct O(1) lookup by ID */ function getNounMetadataPath(id: string): string { @@ -120,7 +119,7 @@ function getNounMetadataPath(id: string): string { } /** - * Get ID-first path for verb vectors (v6.0.0) + * Get ID-first path for verb vectors * No type parameter needed - direct O(1) lookup by ID */ function getVerbVectorPath(id: string): string { @@ -129,7 +128,7 @@ function getVerbVectorPath(id: string): string { } /** - * Get ID-first path for verb metadata (v6.0.0) + * Get ID-first path for verb metadata * No type parameter needed - direct O(1) lookup by ID */ function getVerbMetadataPath(id: string): string { @@ -147,8 +146,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected graphIndexPromise?: Promise protected readOnly = false - // v5.7.2: Write-through cache for read-after-write consistency - // v5.7.3: Extended lifetime - persists until explicit flush() call + // Write-through cache for read-after-write consistency + // Extended lifetime - persists until explicit flush() call // Guarantees that immediately after writeObjectToBranch(), readWithInheritance() returns the data // Cache key: resolved branchPath (includes branch scope for COW isolation) // Cache lifetime: write start → flush() call (provides safety net for batch operations) @@ -156,7 +155,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { private writeCache = new Map() /** - * v7.3.1: Clear the write-through cache + * Clear the write-through cache * MUST be called by all storage adapter clear() implementations to ensure * read-after-write consistency cache doesn't return stale data after clear. * @protected - Available to subclasses for clear() implementation @@ -165,24 +164,24 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.writeCache.clear() } - // COW (Copy-on-Write) support - v5.0.0 + // COW (Copy-on-Write) support public refManager?: RefManager public blobStorage?: BlobStorage public commitLog?: CommitLog public currentBranch: string = 'main' - // v5.11.0: Removed cowEnabled flag - COW is ALWAYS enabled (mandatory, cannot be disabled) + // Removed cowEnabled flag - COW is ALWAYS enabled (mandatory, cannot be disabled) - // Type-first indexing support (v5.4.0) + // Type-first indexing support // Built into all storage adapters for billion-scale efficiency protected nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT) // 168 bytes (Stage 3: 42 types) protected verbCountsByType = new Uint32Array(VERB_TYPE_COUNT) // 508 bytes (Stage 3: 127 types) // Total: 676 bytes (99.2% reduction vs Map-based tracking) - // v6.0.0: Type caches REMOVED - ID-first paths eliminate need for type lookups! + // Type caches REMOVED - ID-first paths eliminate need for type lookups! // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json // Type is just a field in the metadata, indexed by MetadataIndexManager for queries - // v5.5.0: Track if type counts have been rebuilt (prevent repeated rebuilds) + // Track if type counts have been rebuilt (prevent repeated rebuilds) private typeCountsRebuilt = false /** @@ -193,7 +192,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * @private */ private analyzeKey(id: string, context: 'noun-metadata' | 'verb-metadata' | 'system'): StorageKeyInfo { - // v4.8.0: Guard against undefined/null IDs + // Guard against undefined/null IDs if (!id || typeof id !== 'string') { throw new Error(`Invalid storage key: ${id} (must be a non-empty string)`) } @@ -263,13 +262,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Initialize the storage adapter (v5.4.0) + * Initialize the storage adapter * Loads type statistics for built-in type-aware indexing * * IMPORTANT: If your adapter overrides init(), call await super.init() first! */ public async init(): Promise { - // v6.0.1: CRITICAL FIX - Set flag FIRST to prevent infinite recursion + // CRITICAL FIX - Set flag FIRST to prevent infinite recursion // If any code path during initialization calls ensureInitialized(), it would // trigger init() again. Setting the flag immediately breaks the recursion cycle. this.isInitialized = true @@ -278,7 +277,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Load type statistics from storage (if they exist) await this.loadTypeStatistics() - // v6.3.0: GraphAdjacencyIndex is now SINGLETON via getGraphIndex() + // GraphAdjacencyIndex is now SINGLETON via getGraphIndex() // - Removed direct creation here to fix dual-ownership bug // - GraphAdjacencyIndex will be created lazily on first getGraphIndex() call // - This ensures there's only ONE instance per storage adapter @@ -292,7 +291,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Rebuild GraphAdjacencyIndex from existing verbs (v6.0.0) + * Rebuild GraphAdjacencyIndex from existing verbs * Call this manually if you have existing verb data that needs to be indexed * @public */ @@ -304,7 +303,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Invalidate GraphAdjacencyIndex (v6.3.0) + * Invalidate GraphAdjacencyIndex * Call this when switching branches or clearing data to force re-creation * The next getGraphIndex() call will create a fresh instance and rebuild * @public @@ -336,7 +335,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * RefManager/BlobStorage/CommitLog are lazy-initialized on first fork() * @param branch - Branch name to use (default: 'main') * - * v5.11.0: COW is always enabled - this method now just sets the branch name (idempotent) + * COW is always enabled - this method now just sets the branch name (idempotent) */ public enableCOWLightweight(branch: string = 'main'): void { this.currentBranch = branch @@ -347,7 +346,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Initialize COW (Copy-on-Write) support * Creates RefManager and BlobStorage for instant fork() capability * - * v5.0.1: Now called automatically by storageFactory (zero-config) + * Now called automatically by storageFactory (zero-config) * * @param options - COW initialization options * @param options.branch - Initial branch name (default: 'main') @@ -358,7 +357,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { branch?: string enableCompression?: boolean }): Promise { - // v5.11.0: COW is ALWAYS enabled - idempotent initialization only + // COW is ALWAYS enabled - idempotent initialization only // Removed marker file check (cowEnabled flag removed, COW is mandatory) // Check if RefManager already initialized (idempotent) @@ -380,7 +379,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (data === null) { return undefined } - // v5.7.5/v5.10.1: Use shared binaryDataCodec utility (single source of truth) + // Use shared binaryDataCodec utility (single source of truth) // Unwraps binary data stored as {_binary: true, data: "base64..."} // Fixes "Blob integrity check failed" - hash must be calculated on original content return unwrapBinaryData(data) @@ -390,7 +389,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { }, put: async (key: string, data: Buffer): Promise => { - // v6.2.0 PERMANENT FIX: Use key naming convention (explicit type contract) + // PERMANENT FIX: Use key naming convention (explicit type contract) // NO GUESSING - key format explicitly declares data type: // // JSON keys (metadata and refs): @@ -423,7 +422,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { list: async (prefix: string): Promise => { try { - // v5.3.5 fix: Handle file prefixes, not just directory paths + // Handle file prefixes, not just directory paths // Refs are stored as files like: _cow/ref:refs/heads/main // So list('ref:') should find all files starting with '_cow/ref:' @@ -459,7 +458,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const mainRef = await this.refManager.getRef('main') if (!mainRef) { // Create initial commit with empty tree - // v5.3.4: Use NULL_HASH constant instead of hardcoded string + // Use NULL_HASH constant instead of hardcoded string const { NULL_HASH } = await import('./cow/constants.js') const emptyTreeHash = NULL_HASH @@ -498,7 +497,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // v5.11.0: COW is always enabled - no flag to set + // COW is always enabled - no flag to set } /** @@ -506,7 +505,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * @protected - Available to subclasses for COW implementation */ protected resolveBranchPath(basePath: string, branch?: string): string { - // CRITICAL FIX (v5.3.6): COW metadata (_cow/*) must NEVER be branch-scoped + // CRITICAL FIX: COW metadata (_cow/*) must NEVER be branch-scoped // Refs, commits, and blobs are global metadata with their own internal branching. // Branch-scoping COW paths causes fork() to write refs to wrong locations, // leading to "Branch does not exist" errors on checkout (see Workshop bug report). @@ -514,7 +513,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { return basePath // COW metadata is global across all branches } - // v5.11.0: COW is always enabled - always use branch-scoped paths + // COW is always enabled - always use branch-scoped paths const targetBranch = branch || this.currentBranch || 'main' // Branch-scoped path: branches// @@ -528,15 +527,15 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected async writeObjectToBranch(path: string, data: any, branch?: string): Promise { const branchPath = this.resolveBranchPath(path, branch) - // v5.7.2: Add to write cache BEFORE async write (guarantees read-after-write consistency) - // v5.7.3: Cache persists until flush() is called (extended lifetime for batch operations) + // Add to write cache BEFORE async write (guarantees read-after-write consistency) + // Cache persists until flush() is called (extended lifetime for batch operations) // This ensures readWithInheritance() returns data immediately, fixing "Source entity not found" bug this.writeCache.set(branchPath, data) // Write to storage (async) await this.writeObjectToPath(branchPath, data) - // v5.7.3: Cache is NOT cleared here anymore - persists until flush() + // Cache is NOT cleared here anymore - persists until flush() // This provides a safety net for immediate queries after batch writes } @@ -545,13 +544,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Tries current branch first, then walks commit history * @protected - Available to subclasses for COW implementation * - * v5.11.0: COW is always enabled - always use branch-scoped paths with inheritance + * COW is always enabled - always use branch-scoped paths with inheritance */ protected async readWithInheritance(path: string, branch?: string): Promise { const targetBranch = branch || this.currentBranch || 'main' const branchPath = this.resolveBranchPath(path, targetBranch) - // v5.7.2: Check write cache FIRST (synchronous, instant) + // Check write cache FIRST (synchronous, instant) // This guarantees read-after-write consistency within the same process // Fixes bug: brain.add() → brain.relate() → "Source entity not found" const cachedData = this.writeCache.get(branchPath) @@ -608,7 +607,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected async deleteObjectFromBranch(path: string, branch?: string): Promise { const branchPath = this.resolveBranchPath(path, branch) - // v5.7.2: Remove from write cache immediately (before async delete) + // Remove from write cache immediately (before async delete) // Ensures subsequent reads don't return stale cached data this.writeCache.delete(branchPath) @@ -631,13 +630,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * List objects with inheritance (v5.0.1) + * List objects with inheritance * Lists objects from current branch AND main branch, returns unique paths * This enables fork to see parent's data in pagination operations * * Simplified approach: All branches inherit from main * - * v5.11.0: COW is always enabled - always use inheritance + * COW is always enabled - always use inheritance */ protected async listObjectsWithInheritance(prefix: string, branch?: string): Promise { const targetBranch = branch || this.currentBranch || 'main' @@ -657,7 +656,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Save a noun to storage (v4.0.0: vector only, metadata saved separately) + * Save a noun to storage (vector only, metadata saved separately) * @param noun Pure HNSW vector data (no metadata) */ public async saveNoun(noun: HNSWNoun): Promise { @@ -669,7 +668,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get a noun from storage (v4.0.0: returns combined HNSWNounWithMetadata) + * Get a noun from storage (returns combined HNSWNounWithMetadata) * @param id Entity ID * @returns Combined vector + metadata or null */ @@ -685,11 +684,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Load metadata const metadata = await this.getNounMetadata(id) if (!metadata) { - prodLog.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen in v4.0.0`) + prodLog.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen`) return null } - // Combine into HNSWNounWithMetadata - v4.8.0: Extract standard fields to top-level + // Combine into HNSWNounWithMetadata - Extract standard fields to top-level const { noun, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata return { @@ -697,7 +696,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { vector: vector.vector, connections: vector.connections, level: vector.level, - // v4.8.0: Standard fields at top-level + // Standard fields at top-level type: (noun as NounType) || NounType.Thing, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), @@ -722,7 +721,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Internal method returns HNSWNoun[], need to combine with metadata const nouns = await this.getNounsByNounType_internal(nounType) - // Combine each noun with its metadata - v4.8.0: Extract standard fields to top-level + // Combine each noun with its metadata - Extract standard fields to top-level const nounsWithMetadata: HNSWNounWithMetadata[] = [] for (const noun of nouns) { const metadata = await this.getNounMetadata(noun.id) @@ -731,7 +730,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { nounsWithMetadata.push({ ...noun, - // v4.8.0: Standard fields at top-level + // Standard fields at top-level type: (nounType as NounType) || NounType.Thing, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), @@ -768,7 +767,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Save a verb to storage (v4.0.0: verb only, metadata saved separately) + * Save a verb to storage (verb only, metadata saved separately) * * @param verb Pure HNSW verb with core relational fields (verb, sourceId, targetId) */ @@ -784,7 +783,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get a verb from storage (v4.0.0: returns combined HNSWVerbWithMetadata) + * Get a verb from storage (returns combined HNSWVerbWithMetadata) * @param id Entity ID * @returns Combined verb + metadata or null */ @@ -800,11 +799,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Load metadata const metadata = await this.getVerbMetadata(id) if (!metadata) { - prodLog.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen in v4.0.0`) + prodLog.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen`) return null } - // Combine into HNSWVerbWithMetadata - v4.8.0: Extract standard fields to top-level + // Combine into HNSWVerbWithMetadata - Extract standard fields to top-level const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata return { @@ -814,7 +813,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { verb: verb.verb, sourceId: verb.sourceId, targetId: verb.targetId, - // v4.8.0: Standard fields at top-level + // Standard fields at top-level createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -828,7 +827,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Batch get multiple verbs (v6.2.0 - N+1 fix) + * Batch get multiple verbs * * **Performance**: Eliminates N+1 pattern for verb loading * - Current: N × getVerb() = N × 50ms on GCS = 250ms for 5 verbs @@ -842,7 +841,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * @param ids Array of verb IDs to fetch * @returns Map of id → HNSWVerbWithMetadata (only successful reads included) * - * @since v6.2.0 */ public async getVerbsBatch(ids: string[]): Promise> { await this.ensureInitialized() @@ -850,7 +848,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const results = new Map() if (ids.length === 0) return results - // v6.2.0: Batch-fetch vectors and metadata in parallel + // Batch-fetch vectors and metadata in parallel // Build paths for vectors const vectorPaths: Array<{ path: string; id: string }> = ids.map(id => ({ path: getVerbVectorPath(id), @@ -879,7 +877,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Deserialize verb const verb = this.deserializeVerb(vectorData) - // Extract standard fields to top-level (v4.8.0 pattern) + // Extract standard fields to top-level const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataData results.set(id, { @@ -889,7 +887,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { verb: verb.verb, sourceId: verb.sourceId, targetId: verb.targetId, - // v4.8.0: Standard fields at top-level + // Standard fields at top-level createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -982,7 +980,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { pagination: { limit: Number.MAX_SAFE_INTEGER } }) - // v4.0.0: Convert HNSWVerbWithMetadata to HNSWVerb (strip metadata) + // Convert HNSWVerbWithMetadata to HNSWVerb (strip metadata) const hnswVerbs: HNSWVerb[] = result.items.map(verbWithMetadata => ({ id: verbWithMetadata.id, vector: verbWithMetadata.vector, @@ -1193,7 +1191,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get nouns with pagination (v5.4.0: Type-first implementation) + * Get nouns with pagination (Type-first implementation) * * CRITICAL: This method is required for brain.find() to work! * Iterates through noun types with billion-scale optimizations. @@ -1201,7 +1199,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * ARCHITECTURE: Reads storage directly (not indexes) to avoid circular dependencies. * Storage → Indexes (one direction only). GraphAdjacencyIndex built FROM storage. * - * OPTIMIZATIONS (v5.5.0): + * OPTIMIZATIONS: * - Skip empty types using nounCountsByType[] tracking (O(1) check) * - Early termination when offset + limit entities collected * - Memory efficient: Never loads full dataset @@ -1209,7 +1207,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async getNounsWithPagination(options: { limit: number offset: number - cursor?: string // v5.7.11: Currently ignored (offset-based pagination). Cursor support planned for v5.8.0 + cursor?: string // Currently ignored (offset-based pagination). Cursor support planned filter?: { nounType?: string | string[] service?: string | string[] @@ -1227,7 +1225,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const collectedNouns: HNSWNounWithMetadata[] = [] const targetCount = offset + limit - // v6.0.0: Iterate by shards (0x00-0xFF) instead of types + // Iterate by shards (0x00-0xFF) instead of types for (let shard = 0; shard < 256 && collectedNouns.length < targetCount; shard++) { const shardHex = shard.toString(16).padStart(2, '0') const shardDir = `entities/nouns/${shardHex}` @@ -1305,7 +1303,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get verbs with pagination (v5.5.0: Type-first implementation with billion-scale optimizations) + * Get verbs with pagination (Type-first implementation with billion-scale optimizations) * * CRITICAL: This method is required for brain.getRelations() to work! * Iterates through verb types with the same optimizations as nouns. @@ -1313,7 +1311,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * ARCHITECTURE: Reads storage directly (not indexes) to avoid circular dependencies. * Storage → Indexes (one direction only). GraphAdjacencyIndex built FROM storage. * - * OPTIMIZATIONS (v5.5.0): + * OPTIMIZATIONS: * - Skip empty types using verbCountsByType[] tracking (O(1) check) * - Early termination when offset + limit verbs collected * - Memory efficient: Never loads full dataset @@ -1322,7 +1320,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async getVerbsWithPagination(options: { limit: number offset: number - cursor?: string // v5.7.11: Currently ignored (offset-based pagination). Cursor support planned for v5.8.0 + cursor?: string // Currently ignored (offset-based pagination). Cursor support planned filter?: { verbType?: string | string[] sourceId?: string | string[] @@ -1353,7 +1351,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { ? new Set(Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId]) : null - // v6.0.0: Iterate by shards (0x00-0xFF) instead of types - single pass! + // Iterate by shards (0x00-0xFF) instead of types - single pass! for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) { const shardHex = shard.toString(16).padStart(2, '0') const shardDir = `entities/verbs/${shardHex}` @@ -1369,7 +1367,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const rawVerb = await this.readWithInheritance(verbPath) if (!rawVerb) continue - // v6.0.0: Deserialize connections Map from JSON storage format + // Deserialize connections Map from JSON storage format const verb = this.deserializeVerb(rawVerb) // Apply type filter @@ -1414,9 +1412,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // Apply pagination (v5.5.0: Efficient slicing after early termination) + // Apply pagination (Efficient slicing after early termination) const paginatedVerbs = collectedVerbs.slice(offset, offset + limit) - const hasMore = collectedVerbs.length > targetCount // v5.7.11: Fixed >= to > (was causing infinite loop) + const hasMore = collectedVerbs.length > targetCount // Fixed >= to > (was causing infinite loop) return { items: paginatedVerbs, @@ -1604,7 +1602,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // v6.2.9: Fast path for SINGLE sourceId + verbType combo (common VFS pattern) + // Fast path for SINGLE sourceId + verbType combo (common VFS pattern) // This avoids the slow type-iteration fallback for VFS operations // NOTE: Only use fast path for single sourceId to avoid incomplete results const isSingleSourceId = options.filter.sourceId && @@ -1718,12 +1716,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { let totalScanned = 0 const targetCount = offset + limit // We need this many verbs total (including offset) - // v5.5.0 BUG FIX: Check if optimization should be used + // BUG FIX: Check if optimization should be used // Only use type-skipping optimization if counts are non-zero (reliable) const totalVerbCountFromArray = this.verbCountsByType.reduce((sum, c) => sum + c, 0) const useOptimization = totalVerbCountFromArray > 0 - // v6.2.9 BUG FIX: Pre-compute requested verb types to avoid skipping them + // BUG FIX: Pre-compute requested verb types to avoid skipping them // When a specific verbType filter is provided, we MUST check that type // even if verbCountsByType shows 0 (counts can be stale after restart) const requestedVerbTypes = options?.filter?.verbType @@ -1736,7 +1734,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { for (let i = 0; i < VERB_TYPE_COUNT && collectedVerbs.length < targetCount; i++) { const type = TypeUtils.getVerbFromIndex(i) - // v6.2.9 FIX: Never skip a type that's explicitly requested in the filter + // FIX: Never skip a type that's explicitly requested in the filter // This fixes VFS bug where Contains relationships were skipped after restart // when verbCountsByType[Contains] was 0 due to stale statistics const isRequestedType = requestedVerbTypesSet?.has(type) ?? false @@ -1747,7 +1745,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { continue } - // v6.2.9: Log when we DON'T skip a requested type that would have been skipped + // Log when we DON'T skip a requested type that would have been skipped // This helps diagnose stale statistics issues in production if (useOptimization && countIsZero && isRequestedType) { prodLog.debug( @@ -1812,7 +1810,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Apply pagination (slice for offset) const paginatedVerbs = collectedVerbs.slice(offset, offset + limit) - const hasMore = collectedVerbs.length > targetCount // v5.7.11: Fixed >= to > (was causing infinite loop) + const hasMore = collectedVerbs.length > targetCount // Fixed >= to > (was causing infinite loop) return { items: paginatedVerbs, @@ -1851,7 +1849,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** * Get graph index (lazy initialization with concurrent access protection) - * v5.7.1: Fixed race condition where concurrent calls could trigger multiple rebuilds + * Fixed race condition where concurrent calls could trigger multiple rebuilds */ async getGraphIndex(): Promise { // If already initialized, return immediately @@ -1900,7 +1898,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { public abstract clear(): Promise /** - * v5.11.0: Removed checkClearMarker() and createClearMarker() abstract methods + * Removed checkClearMarker() and createClearMarker() abstract methods * COW is now always enabled - marker files are no longer used */ @@ -1951,7 +1949,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected abstract listObjectsUnderPath(prefix: string): Promise /** - * Save metadata to storage (v4.0.0: now typed) + * Save metadata to storage (now typed) * Routes to correct location (system or entity) based on key format */ public async saveMetadata(id: string, metadata: NounMetadata): Promise { @@ -1961,7 +1959,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get metadata from storage (v4.0.0: now typed) + * Get metadata from storage (now typed) * Routes to correct location (system or entity) based on key format */ public async getMetadata(id: string): Promise { @@ -1971,7 +1969,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Save noun metadata to storage (v4.0.0: now typed) + * Save noun metadata to storage (now typed) * Routes to correct sharded location based on UUID */ public async saveNounMetadata(id: string, metadata: NounMetadata): Promise { @@ -1981,10 +1979,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Internal method for saving noun metadata (v4.0.0: now typed) + * Internal method for saving noun metadata (now typed) * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded) * - * CRITICAL (v4.1.2): Count synchronization happens here + * CRITICAL: Count synchronization happens here * This ensures counts are updated AFTER metadata exists, fixing the race condition * where storage adapters tried to read metadata before it was saved. * @@ -1993,7 +1991,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise { await this.ensureInitialized() - // v6.0.0: ID-first path - no type needed! + // ID-first path - no type needed! const path = getNounMetadataPath(id) // Determine if this is a new entity by checking if metadata already exists @@ -2003,7 +2001,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Save the metadata (COW-aware - writes to branch-specific path) await this.writeObjectToBranch(path, metadata) - // CRITICAL FIX (v4.1.2): Increment count for new entities + // CRITICAL FIX: Increment count for new entities // This runs AFTER metadata is saved, guaranteeing type information is available // Uses synchronous increment since storage operations are already serialized // Fixes Bug #1: Count synchronization failure during add() and import() @@ -2019,18 +2017,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Get noun metadata from storage (METADATA-ONLY, NO VECTORS) * - * **Performance (v6.0.0)**: Direct O(1) ID-first lookup - NO type search needed! + * **Performance**: Direct O(1) ID-first lookup - NO type search needed! * - **All lookups**: 1 read, ~500ms on cloud (consistent performance) * - **No cache needed**: Type is in the metadata, not the path * - **No type search**: ID-first paths eliminate 42-type search entirely * - * **Clean architecture (v6.0.0)**: + * **Clean architecture**: * - Path: `entities/nouns/{SHARD}/{ID}/metadata.json` * - Type is just a field in metadata (`noun: "document"`) * - MetadataIndex handles type queries (no path scanning needed) * - Scales to billions without any overhead * - * **Performance (v5.11.1)**: Fast path for metadata-only reads + * **Performance**: Fast path for metadata-only reads * - **Speed**: 10ms vs 43ms (76-81% faster than getNoun) * - **Bandwidth**: 300 bytes vs 6KB (95% less) * - **Memory**: 300 bytes vs 6KB (87% less) @@ -2064,21 +2062,20 @@ export abstract class BaseStorage extends BaseStorageAdapter { * - No type search fallbacks * - Works in distributed systems without sync issues * - * @since v4.0.0 - * @since v5.4.0 - Type-first paths (removed in v6.0.0) - * @since v5.11.1 - Promoted to fast path for brain.get() optimization - * @since v6.0.0 - CLEAN FIX: ID-first paths eliminate all type-search complexity + * Type-first paths (removed) + * Promoted to fast path for brain.get() optimization + * CLEAN FIX: ID-first paths eliminate all type-search complexity */ public async getNounMetadata(id: string): Promise { await this.ensureInitialized() - // v6.0.0: Clean, simple, O(1) lookup - no type needed! + // Clean, simple, O(1) lookup - no type needed! const path = getNounMetadataPath(id) return this.readWithInheritance(path) } /** - * Batch fetch noun metadata from storage (v5.12.0 - Cloud Storage Optimization) + * Batch fetch noun metadata from storage * * **Performance**: Reduces N sequential calls → 1-2 batch calls * - Local storage: N × 10ms → 1 × 10ms parallel (N× faster) @@ -2107,7 +2104,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * } * ``` * - * @since v5.12.0 */ public async getNounMetadataBatch(ids: string[]): Promise> { await this.ensureInitialized() @@ -2115,7 +2111,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const results = new Map() if (ids.length === 0) return results - // v6.0.0: ID-first paths - no type grouping or search needed! + // ID-first paths - no type grouping or search needed! // Build direct paths for all IDs const pathsToFetch: Array<{ path: string; id: string }> = ids.map(id => ({ path: getNounMetadataPath(id), @@ -2137,7 +2133,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Batch get multiple nouns with vectors (v6.2.0 - N+1 fix) + * Batch get multiple nouns with vectors * * **Performance**: Eliminates N+1 pattern for vector loading * - Current: N × getNoun() = N × 50ms on GCS = 500ms for 10 entities @@ -2151,7 +2147,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * @param ids Array of entity IDs to fetch (with vectors) * @returns Map of id → HNSWNounWithMetadata (only successful reads included) * - * @since v6.2.0 */ public async getNounBatch(ids: string[]): Promise> { await this.ensureInitialized() @@ -2159,7 +2154,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const results = new Map() if (ids.length === 0) return results - // v6.2.0: Batch-fetch vectors and metadata in parallel + // Batch-fetch vectors and metadata in parallel // Build paths for vectors const vectorPaths: Array<{ path: string; id: string }> = ids.map(id => ({ path: getNounVectorPath(id), @@ -2188,7 +2183,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Deserialize noun const noun = this.deserializeNoun(vectorData) - // Extract standard fields to top-level (v4.8.0 pattern) + // Extract standard fields to top-level const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataData results.set(id, { @@ -2196,7 +2191,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { vector: noun.vector, connections: noun.connections, level: noun.level, - // v4.8.0: Standard fields at top-level + // Standard fields at top-level type: (nounType as NounType) || NounType.Thing, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), @@ -2215,7 +2210,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Batch read multiple storage paths with COW inheritance support (v5.12.0) + * Batch read multiple storage paths with COW inheritance support * * Core batching primitive that all batch operations build upon. * Handles write cache, branch inheritance, and adapter-specific batching. @@ -2230,7 +2225,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * @returns Map of path → data (only successful reads included) * * @protected - Available to subclasses and batch operations - * @since v5.12.0 */ protected async readBatchWithInheritance( paths: string[], @@ -2289,7 +2283,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Step 4: Handle COW inheritance for missing items (if not on main branch) if (targetBranch !== 'main' && missingPaths.length > 0) { // For now, fall back to individual inheritance lookups - // TODO v5.13.0: Optimize inheritance with batch commit walks + // TODO: Optimize inheritance with batch commit walks for (const originalPath of missingPaths) { try { const data = await this.readWithInheritance(originalPath, targetBranch) @@ -2306,7 +2300,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Adapter-level batch read with automatic batching strategy (v5.12.0) + * Adapter-level batch read with automatic batching strategy * * Uses adapter's native batch API when available: * - GCS: batch API (100 ops) @@ -2320,7 +2314,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * @returns Map of path → data * * @private - * @since v5.12.0 */ private async readBatchFromAdapter(paths: string[]): Promise> { if (paths.length === 0) return new Map() @@ -2366,13 +2359,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get batch configuration for this storage adapter (v5.12.0) + * Get batch configuration for this storage adapter * * Override in subclasses to provide adapter-specific batch limits. * Defaults to conservative limits for safety. * * @public - Inherited from BaseStorageAdapter - * @since v5.12.0 */ public getBatchConfig(): StorageBatchConfig { // Conservative defaults - adapters should override with their actual limits @@ -2389,18 +2381,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Delete noun metadata from storage (v6.0.0: ID-first, O(1) delete) + * Delete noun metadata from storage (ID-first, O(1) delete) */ public async deleteNounMetadata(id: string): Promise { await this.ensureInitialized() - // v6.0.0: Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path const path = getNounMetadataPath(id) await this.deleteObjectFromBranch(path) } /** - * Save verb metadata to storage (v4.0.0: now typed) + * Save verb metadata to storage (now typed) * Routes to correct sharded location based on UUID */ public async saveVerbMetadata(id: string, metadata: VerbMetadata): Promise { @@ -2409,10 +2401,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Internal method for saving verb metadata (v4.0.0: now typed) - * v5.4.0: Uses ID-first paths (must match getVerbMetadata) + * Internal method for saving verb metadata (now typed) + * Uses ID-first paths (must match getVerbMetadata) * - * CRITICAL (v4.1.2): Count synchronization happens here + * CRITICAL: Count synchronization happens here * This ensures verb counts are updated AFTER metadata exists, fixing the race condition * where storage adapters tried to read metadata before it was saved. * @@ -2423,7 +2415,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected async saveVerbMetadata_internal(id: string, metadata: VerbMetadata): Promise { await this.ensureInitialized() - // v5.4.0: Extract verb type from metadata for ID-first path + // Extract verb type from metadata for ID-first path const verbType = (metadata as any).verb as VerbType | undefined if (!verbType) { @@ -2433,7 +2425,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { return } - // v5.4.0: Use ID-first path + // Use ID-first path const path = getVerbMetadataPath(id) // Determine if this is a new verb by checking if metadata already exists @@ -2443,9 +2435,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Save the metadata (COW-aware - writes to branch-specific path) await this.writeObjectToBranch(path, metadata) - // v5.4.0: Cache verb type for faster lookups + // Cache verb type for faster lookups - // CRITICAL FIX (v4.1.2): Increment verb count for new relationships + // CRITICAL FIX: Increment verb count for new relationships // This runs AFTER metadata is saved // Uses synchronous increment since storage operations are already serialized // Fixes Bug #2: Count synchronization failure during relate() and import() @@ -2459,13 +2451,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get verb metadata from storage (v4.0.0: now typed) - * v5.4.0: Uses ID-first paths (must match saveVerbMetadata_internal) + * Get verb metadata from storage (now typed) + * Uses ID-first paths (must match saveVerbMetadata_internal) */ public async getVerbMetadata(id: string): Promise { await this.ensureInitialized() - // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed! + // Direct O(1) lookup with ID-first paths - no type search needed! const path = getVerbMetadataPath(id) try { @@ -2478,18 +2470,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Delete verb metadata from storage (v6.0.0: ID-first, O(1) delete) + * Delete verb metadata from storage (ID-first, O(1) delete) */ public async deleteVerbMetadata(id: string): Promise { await this.ensureInitialized() - // v6.0.0: Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path const path = getVerbMetadataPath(id) await this.deleteObjectFromBranch(path) } // ============================================================================ - // ID-FIRST HELPER METHODS (v6.0.0) + // ID-FIRST HELPER METHODS // Direct O(1) ID lookups - no type needed! // Clean, simple architecture for billion-scale performance // ============================================================================ @@ -2532,7 +2524,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Get noun counts by type (O(1) access to type statistics) - * v6.2.2: Exposed for MetadataIndexManager to use as single source of truth + * Exposed for MetadataIndexManager to use as single source of truth * @returns Uint32Array indexed by NounType enum value (42 types) */ public getNounCountsByType(): Uint32Array { @@ -2541,7 +2533,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Get verb counts by type (O(1) access to type statistics) - * v6.2.2: Exposed for MetadataIndexManager to use as single source of truth + * Exposed for MetadataIndexManager to use as single source of truth * @returns Uint32Array indexed by VerbType enum value (127 types) */ public getVerbCountsByType(): Uint32Array { @@ -2549,14 +2541,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Rebuild type counts from actual storage (v5.5.0) + * Rebuild type counts from actual storage * Called when statistics are missing or inconsistent * Ensures verbCountsByType is always accurate for reliable pagination */ protected async rebuildTypeCounts(): Promise { prodLog.info('[BaseStorage] Rebuilding type counts from storage...') - // v6.0.0: Rebuild by scanning shards (0x00-0xFF) and reading metadata + // Rebuild by scanning shards (0x00-0xFF) and reading metadata this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT) this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT) @@ -2625,12 +2617,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get noun type (v6.0.0: type no longer needed for paths!) + * Get noun type (type no longer needed for paths!) * With ID-first paths, this is only used for internal statistics tracking. * The actual type is stored in metadata and indexed by MetadataIndexManager. */ protected getNounType(noun: HNSWNoun): NounType { - // v6.0.0: Type cache removed - default to 'thing' for statistics + // Type cache removed - default to 'thing' for statistics // The real type is in metadata, accessible via getNounMetadata(id) return 'thing' } @@ -2640,7 +2632,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Verb type is a required field in HNSWVerb */ protected getVerbType(verb: HNSWVerb | GraphVerb): VerbType { - // v3.50.1+: verb is a required field in HNSWVerb + // verb is a required field in HNSWVerb if ('verb' in verb && verb.verb) { return verb.verb as VerbType } @@ -2657,7 +2649,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // ============================================================================ - // DESERIALIZATION HELPERS (v5.7.10) + // DESERIALIZATION HELPERS // Centralized Map/Set reconstruction from JSON storage format // ============================================================================ @@ -2667,7 +2659,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Converts plain object { "0": ["id1"], "1": ["id2"] } * into Map> * - * v5.7.10: Central helper to fix serialization bug across all code paths + * Central helper to fix serialization bug across all code paths * Root cause: JSON.stringify(Map) = {} (empty object), must reconstruct on read */ protected deserializeConnections(connections: any): Map> { @@ -2698,7 +2690,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Deserialize HNSWNoun from JSON storage format * - * v5.7.10: Ensures connections are properly reconstructed from Map → object → Map + * Ensures connections are properly reconstructed from Map → object → Map * Fixes: "TypeError: noun.connections.entries is not a function" */ protected deserializeNoun(data: any): HNSWNoun { @@ -2711,7 +2703,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Deserialize HNSWVerb from JSON storage format * - * v5.7.10: Ensures connections are properly reconstructed from Map → object → Map + * Ensures connections are properly reconstructed from Map → object → Map * Fixes same serialization bug for verbs */ protected deserializeVerb(data: any): HNSWVerb { @@ -2723,7 +2715,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // ============================================================================ - // ABSTRACT METHOD IMPLEMENTATIONS (v5.4.0) + // ABSTRACT METHOD IMPLEMENTATIONS // Converted from abstract to concrete - all adapters now have built-in type-aware // ============================================================================ @@ -2738,11 +2730,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { const typeIndex = TypeUtils.getNounIndex(type) this.nounCountsByType[typeIndex]++ - // COW-aware write (v5.0.1): Use COW helper for branch isolation + // COW-aware write: Use COW helper for branch isolation await this.writeObjectToBranch(path, noun) // Periodically save statistics - // v6.2.9: Also save on first noun of each type to ensure low-count types are tracked + // Also save on first noun of each type to ensure low-count types are tracked const shouldSave = this.nounCountsByType[typeIndex] === 1 || // First noun of type this.nounCountsByType[typeIndex] % 100 === 0 // Every 100th if (shouldSave) { @@ -2754,14 +2746,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Get a noun from storage (ID-first path) */ protected async getNoun_internal(id: string): Promise { - // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed! + // Direct O(1) lookup with ID-first paths - no type search needed! const path = getNounVectorPath(id) try { - // COW-aware read (v5.0.1): Use COW helper for branch isolation + // COW-aware read: Use COW helper for branch isolation const noun = await this.readWithInheritance(path) if (noun) { - // v5.7.10: Deserialize connections Map from JSON storage format + // Deserialize connections Map from JSON storage format return this.deserializeNoun(noun) } } catch (error) { @@ -2773,12 +2765,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get nouns by noun type (v6.0.0: Shard-based iteration!) + * Get nouns by noun type (Shard-based iteration!) */ protected async getNounsByNounType_internal( nounType: string ): Promise { - // v6.0.0: Iterate by shards (0x00-0xFF) instead of types + // Iterate by shards (0x00-0xFF) instead of types // Type is stored in metadata.noun field, we filter as we load const nouns: HNSWNoun[] = [] @@ -2816,10 +2808,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Delete a noun from storage (v6.0.0: ID-first, O(1) delete) + * Delete a noun from storage (ID-first, O(1) delete) */ protected async deleteNoun_internal(id: string): Promise { - // v6.0.0: Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path const path = getNounVectorPath(id) await this.deleteObjectFromBranch(path) @@ -2841,10 +2833,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { const typeIndex = TypeUtils.getVerbIndex(type) this.verbCountsByType[typeIndex]++ - // COW-aware write (v5.0.1): Use COW helper for branch isolation + // COW-aware write: Use COW helper for branch isolation await this.writeObjectToBranch(path, verb) - // v6.3.0: GraphAdjacencyIndex updates are now handled EXCLUSIVELY by Brainy.relate() + // GraphAdjacencyIndex updates are now handled EXCLUSIVELY by Brainy.relate() // via AddToGraphIndexOperation in the transaction system. This provides: // 1. Singleton pattern - only one graphIndex instance exists (via getGraphIndex()) // 2. Transaction rollback - if relate() fails, index update is rolled back @@ -2852,7 +2844,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // REMOVED: Direct graphIndex.addVerb() call that caused dual-ownership bugs // Periodically save statistics - // v6.2.9: Also save on first verb of each type to ensure low-count types are tracked + // Also save on first verb of each type to ensure low-count types are tracked // This prevents stale statistics after restart for types with < 100 verbs (common for VFS) const shouldSave = this.verbCountsByType[typeIndex] === 1 || // First verb of type this.verbCountsByType[typeIndex] % 100 === 0 // Every 100th @@ -2865,14 +2857,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Get a verb from storage (ID-first path) */ protected async getVerb_internal(id: string): Promise { - // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed! + // Direct O(1) lookup with ID-first paths - no type search needed! const path = getVerbVectorPath(id) try { - // COW-aware read (v5.0.1): Use COW helper for branch isolation + // COW-aware read: Use COW helper for branch isolation const verb = await this.readWithInheritance(path) if (verb) { - // v5.7.10: Deserialize connections Map from JSON storage format + // Deserialize connections Map from JSON storage format return this.deserializeVerb(verb) } } catch (error) { @@ -2884,7 +2876,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get verbs by source (v6.0.0: Uses GraphAdjacencyIndex when available) + * Get verbs by source (Uses GraphAdjacencyIndex when available) * Falls back to shard iteration during initialization to avoid circular dependency */ protected async getVerbsBySource_internal( @@ -2894,13 +2886,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`) - // v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded) + // Fast path - use GraphAdjacencyIndex if available (lazy-loaded) if (this.graphIndex && this.graphIndex.isInitialized) { try { const verbIds = await this.graphIndex.getVerbIdsBySource(sourceId) prodLog.debug(`[BaseStorage] GraphAdjacencyIndex found ${verbIds.length} verb IDs for sourceId=${sourceId}`) - // v6.0.2: PERFORMANCE FIX - Batch fetch verbs + metadata (eliminates N+1 pattern) + // PERFORMANCE FIX - Batch fetch verbs + metadata (eliminates N+1 pattern) // Before: N sequential calls (10 children = 20 × 300ms = 6000ms on GCS) // After: 2 parallel batch calls (10 children = 2 × 300ms = 600ms on GCS) // 10x improvement for cloud storage (GCS, S3, Azure) @@ -2922,7 +2914,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const metadata = metadataMap.get(metadataPath) if (rawVerb && metadata) { - // v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format + // CRITICAL - Deserialize connections Map from JSON storage format const verb = this.deserializeVerb(rawVerb) results.push({ @@ -2949,7 +2941,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // v6.0.0: Fallback - iterate by shards (WITH deserialization fix!) + // Fallback - iterate by shards (WITH deserialization fix!) prodLog.debug(`[BaseStorage] Using shard iteration fallback for sourceId=${sourceId}`) const results: HNSWVerbWithMetadata[] = [] let shardsScanned = 0 @@ -2972,7 +2964,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { verbsFound++ - // v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format + // CRITICAL - Deserialize connections Map from JSON storage format const verb = this.deserializeVerb(rawVerb) if (verb.sourceId === sourceId) { @@ -3009,7 +3001,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Batch get verbs by source IDs (v5.12.0 - Cloud Storage Optimization) + * Batch get verbs by source IDs * * **Performance**: Eliminates N+1 query pattern for relationship lookups * - Current: N × getVerbsBySource() = N × (list all verbs + filter) @@ -3038,7 +3030,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * } * ``` * - * @since v5.12.0 */ public async getVerbsBySourceBatch( sourceIds: string[], @@ -3057,7 +3048,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Convert sourceIds to Set for O(1) lookup const sourceIdSet = new Set(sourceIds) - // v6.0.0: Iterate by shards (0x00-0xFF) instead of types + // Iterate by shards (0x00-0xFF) instead of types for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') const shardDir = `entities/verbs/${shardHex}` @@ -3092,7 +3083,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { for (const [verbPath, rawVerbData] of verbDataMap.entries()) { if (!rawVerbData || !rawVerbData.sourceId) continue - // v6.0.0: Deserialize connections Map from JSON storage format + // Deserialize connections Map from JSON storage format const verbData = this.deserializeVerb(rawVerbData) // Check if this verb's source is in our requested set @@ -3135,15 +3126,15 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Get verbs by target (COW-aware implementation) - * v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock - * v5.4.0: Fixed to directly list verb files instead of directories + * Reverted to fix circular dependency deadlock + * Fixed to directly list verb files instead of directories */ protected async getVerbsByTarget_internal( targetId: string ): Promise { await this.ensureInitialized() - // v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded) + // Fast path - use GraphAdjacencyIndex if available (lazy-loaded) if (this.graphIndex && this.graphIndex.isInitialized) { try { const verbIds = await this.graphIndex.getVerbIdsByTarget(targetId) @@ -3177,7 +3168,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // v6.0.0: Fallback - iterate by shards (WITH deserialization fix!) + // Fallback - iterate by shards (WITH deserialization fix!) const results: HNSWVerbWithMetadata[] = [] for (let shard = 0; shard < 256; shard++) { @@ -3194,7 +3185,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const rawVerb = await this.readWithInheritance(verbPath) if (!rawVerb) continue - // v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format + // CRITICAL - Deserialize connections Map from JSON storage format const verb = this.deserializeVerb(rawVerb) if (verb.targetId === targetId) { @@ -3229,10 +3220,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Get verbs by type (v6.0.0: Shard iteration with type filtering) + * Get verbs by type (Shard iteration with type filtering) */ protected async getVerbsByType_internal(verbType: string): Promise { - // v6.0.0: Iterate by shards (0x00-0xFF) instead of type-first paths + // Iterate by shards (0x00-0xFF) instead of type-first paths const verbs: HNSWVerbWithMetadata[] = [] for (let shard = 0; shard < 256; shard++) { @@ -3249,23 +3240,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { const rawVerb = await this.readWithInheritance(verbPath) if (!rawVerb) continue - // v5.7.10: Deserialize connections Map from JSON storage format + // Deserialize connections Map from JSON storage format const hnswVerb = this.deserializeVerb(rawVerb) // Filter by verb type if (hnswVerb.verb !== verbType) continue - // Load metadata separately (optional in v4.0.0!) + // Load metadata separately (optional) const metadata = await this.getVerbMetadata(hnswVerb.id) - // v4.8.0: Extract standard fields from metadata to top-level + // Extract standard fields from metadata to top-level const metadataObj = (metadata || {}) as VerbMetadata const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj const verbWithMetadata: HNSWVerbWithMetadata = { id: hnswVerb.id, vector: [...hnswVerb.vector], - connections: hnswVerb.connections, // v5.7.10: Already deserialized + connections: hnswVerb.connections, // Already deserialized verb: hnswVerb.verb, sourceId: hnswVerb.sourceId, targetId: hnswVerb.targetId, @@ -3293,10 +3284,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Delete a verb from storage (v6.0.0: ID-first, O(1) delete) + * Delete a verb from storage (ID-first, O(1) delete) */ protected async deleteVerb_internal(id: string): Promise { - // v6.0.0: Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path const path = getVerbVectorPath(id) await this.deleteObjectFromBranch(path) diff --git a/src/storage/cow/BlobStorage.ts b/src/storage/cow/BlobStorage.ts index d08a253b..363bde85 100644 --- a/src/storage/cow/BlobStorage.ts +++ b/src/storage/cow/BlobStorage.ts @@ -161,7 +161,7 @@ export class BlobStorage { } /** - * v5.7.5: Ensure compression is ready before write operations + * Ensure compression is ready before write operations * Fixes race condition where write happens before async compression init completes */ private async ensureCompressionReady(): Promise { @@ -206,7 +206,7 @@ export class BlobStorage { return hash } - // v5.7.5: Ensure compression is initialized before writing + // Ensure compression is initialized before writing // Fixes race condition where write happens before async init completes await this.ensureCompressionReady() @@ -223,7 +223,7 @@ export class BlobStorage { } // Create metadata - // v5.7.5: Store ACTUAL compression state, not intended + // Store ACTUAL compression state, not intended // Prevents corruption if compression failed to initialize const actualCompression = finalData === data ? 'none' : compression const metadata: BlobMetadata = { @@ -277,7 +277,7 @@ export class BlobStorage { * @returns Blob data */ async read(hash: string, options: BlobReadOptions = {}): Promise { - // v5.3.4 fix: Guard against NULL hash (sentinel value) + // Guard against NULL hash (sentinel value) // NULL_HASH ('0000...0000') is used as a sentinel for "no parent" or "empty tree" // It should NEVER be read as actual blob data if (isNullHash(hash)) { @@ -309,7 +309,7 @@ export class BlobStorage { metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`) if (metadataBuffer) { prefix = tryPrefix - // v5.10.1: Unwrap metadata before parsing (defense-in-depth) + // Unwrap metadata before parsing (defense-in-depth) // Metadata should be JSON, but adapter might return wrapped format const unwrappedMetadata = unwrapBinaryData(metadataBuffer) metadata = JSON.parse(unwrappedMetadata.toString()) @@ -338,8 +338,8 @@ export class BlobStorage { finalData = await this.zstdDecompress(data) } - // v5.10.1: Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression) - // Even though COW adapter should unwrap (v5.7.5), verify it happened and re-unwrap if needed + // Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression) + // Even though COW adapter should unwrap, verify it happened and re-unwrap if needed // This prevents "Blob integrity check failed" errors if adapter returns wrapped data // Uses shared binaryDataCodec utility (single source of truth for unwrap logic) const unwrappedData = unwrapBinaryData(finalData) diff --git a/src/storage/cow/CommitLog.ts b/src/storage/cow/CommitLog.ts index fa46eead..ca9a1650 100644 --- a/src/storage/cow/CommitLog.ts +++ b/src/storage/cow/CommitLog.ts @@ -43,7 +43,7 @@ export interface CommitLogStats { /** * CommitLog: Efficient commit history traversal and querying * - * Pure v5.0.0 implementation - modern, clean, fast + * Pure implementation - modern, clean, fast */ export class CommitLog { private blobStorage: BlobStorage diff --git a/src/storage/cow/CommitObject.ts b/src/storage/cow/CommitObject.ts index f22c2fa5..57645695 100644 --- a/src/storage/cow/CommitObject.ts +++ b/src/storage/cow/CommitObject.ts @@ -337,7 +337,7 @@ export class CommitObject { let currentHash: string | null = startHash let depth = 0 - // v5.3.4 fix: Guard against NULL hash (sentinel for "no parent") + // Guard against NULL hash (sentinel for "no parent") // The initial commit has parent = null or NULL_HASH ('0000...0000') // We must stop walking when we reach it, not try to read it while (currentHash && !isNullHash(currentHash)) { diff --git a/src/storage/cow/RefManager.ts b/src/storage/cow/RefManager.ts index 73d2172b..453a07ba 100644 --- a/src/storage/cow/RefManager.ts +++ b/src/storage/cow/RefManager.ts @@ -55,7 +55,7 @@ export interface RefUpdateOptions { /** * RefManager: Manages branches, tags, and HEAD pointer * - * Pure implementation for v5.0.0 - no backward compatibility + * Pure implementation - no backward compatibility */ export class RefManager { private adapter: COWStorageAdapter diff --git a/src/storage/cow/binaryDataCodec.ts b/src/storage/cow/binaryDataCodec.ts index 58d98302..68c393dd 100644 --- a/src/storage/cow/binaryDataCodec.ts +++ b/src/storage/cow/binaryDataCodec.ts @@ -86,7 +86,7 @@ export function unwrapBinaryData(data: any): Buffer { /** * Wrap binary data for JSON storage * - * ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0) + * ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! * ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead. * ⚠️ This function exists for legacy/compatibility only. * @@ -94,7 +94,7 @@ export function unwrapBinaryData(data: any): Buffer { * This is FRAGILE because compressed binary can accidentally parse as valid JSON, * causing blob integrity failures. * - * v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention: + * SOLUTION: baseStorage.ts COW adapter now uses key naming convention: * - Keys with '-meta:' or 'ref:' prefix → Always JSON * - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary * No guessing needed! diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts index bb088c4c..07ac43e7 100644 --- a/src/storage/storageFactory.ts +++ b/src/storage/storageFactory.ts @@ -10,7 +10,7 @@ import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js' import { R2Storage } from './adapters/r2Storage.js' import { GcsStorage } from './adapters/gcsStorage.js' import { AzureBlobStorage } from './adapters/azureBlobStorage.js' -// TypeAwareStorageAdapter removed in v5.4.0 - type-aware now built into BaseStorage +// TypeAwareStorageAdapter removed - type-aware now built into BaseStorage // FileSystemStorage is dynamically imported to avoid issues in browser environments import { isBrowser } from '../utils/environment.js' import { OperationConfig } from '../utils/operationUtils.js' @@ -94,7 +94,7 @@ export interface StorageOptions { sessionToken?: string /** - * Initialization mode for fast cold starts (v7.3.0+) + * Initialization mode for fast cold starts * * - `'auto'` (default): Progressive in cloud environments (Lambda), * strict locally. Zero-config optimization. @@ -103,7 +103,6 @@ export interface StorageOptions { * - `'strict'`: Traditional blocking init. Validates bucket and loads counts * before init() returns. * - * @since v7.3.0 */ initMode?: 'progressive' | 'strict' | 'auto' } @@ -215,7 +214,7 @@ export interface StorageOptions { skipCountsFile?: boolean /** - * Initialization mode for fast cold starts (v7.3.0+) + * Initialization mode for fast cold starts * * - `'auto'` (default): Progressive in cloud environments (Cloud Run), * strict locally. Zero-config optimization. @@ -224,7 +223,6 @@ export interface StorageOptions { * - `'strict'`: Traditional blocking init. Validates bucket and loads counts * before init() returns. * - * @since v7.3.0 */ initMode?: 'progressive' | 'strict' | 'auto' } @@ -259,7 +257,7 @@ export interface StorageOptions { sasToken?: string /** - * Initialization mode for fast cold starts (v7.3.0+) + * Initialization mode for fast cold starts * * - `'auto'` (default): Progressive in cloud environments (Azure Functions), * strict locally. Zero-config optimization. @@ -268,7 +266,6 @@ export interface StorageOptions { * - `'strict'`: Traditional blocking init. Validates container and loads counts * before init() returns. * - * @since v7.3.0 */ initMode?: 'progressive' | 'strict' | 'auto' } @@ -388,7 +385,7 @@ export interface StorageOptions { /** * COW (Copy-on-Write) configuration for instant fork() capability - * v5.0.1: COW is now always enabled (automatic, zero-config) + * COW is now always enabled (automatic, zero-config) */ branch?: string // Current branch name (default: 'main') enableCompression?: boolean // Enable zstd compression for COW blobs (default: true) @@ -410,14 +407,14 @@ function getFileSystemPath(options: StorageOptions): string { } /** - * Configure COW (Copy-on-Write) options on a storage adapter (v5.4.0) + * Configure COW (Copy-on-Write) options on a storage adapter * TypeAware is now built-in to all adapters, no wrapper needed! * * @param storage - The storage adapter * @param options - Storage options (for COW configuration) */ function configureCOW(storage: any, options?: StorageOptions): void { - // v5.0.1: COW will be initialized AFTER storage.init() in Brainy + // COW will be initialized AFTER storage.init() in Brainy // Store COW options for later initialization if (typeof storage.initializeCOW === 'function') { storage._cowOptions = { @@ -672,10 +669,10 @@ export async function createStorage( } case 'type-aware': - // v5.0.0: TypeAware is now the default for ALL adapters + // TypeAware is now the default for ALL adapters // Redirect to the underlying type instead console.warn( - '⚠️ type-aware is deprecated in v5.0.0 - TypeAware is now always enabled.' + '⚠️ type-aware is deprecated - TypeAware is now always enabled.' ) console.warn( ' Just use the underlying storage type (e.g., "filesystem", "s3", etc.)' @@ -866,7 +863,7 @@ export async function createStorage( } /** - * Export storage adapters (v5.4.0: TypeAware is now built-in, no separate export) + * Export storage adapters (TypeAware is now built-in, no separate export) */ export { MemoryStorage, diff --git a/src/types/brainyInterface.ts b/src/types/brainyInterface.ts index 848bcee0..642fc675 100644 --- a/src/types/brainyInterface.ts +++ b/src/types/brainyInterface.ts @@ -18,7 +18,7 @@ export interface BrainyInterface { init(): Promise /** - * Promise that resolves when initialization is complete (v7.3.0+) + * Promise that resolves when initialization is complete * Can be awaited multiple times safely. */ readonly ready: Promise @@ -29,12 +29,12 @@ export interface BrainyInterface { readonly isInitialized: boolean /** - * Check if all initialization including background tasks is complete (v7.3.0+) + * Check if all initialization including background tasks is complete */ isFullyInitialized(): boolean /** - * Wait for all background initialization tasks to complete (v7.3.0+) + * Wait for all background initialization tasks to complete * For cloud storage adapters, this waits for bucket validation and count sync. */ awaitBackgroundInit(): Promise diff --git a/src/types/graphTypes.ts b/src/types/graphTypes.ts index 1f92b151..a5ee2622 100644 --- a/src/types/graphTypes.ts +++ b/src/types/graphTypes.ts @@ -303,7 +303,7 @@ * } * ``` * - * ## Stage 3 Changes from v5.4.0 + * ## Stage 3 Changes * * ### Nouns Added (+11) * agent, quality, timeInterval, function, proposition, socialGroup, institution, @@ -323,7 +323,7 @@ * createdBy (use inverse of creates), supervises (use inverse of reportsTo) * * **Net Change**: +9 nouns (31 → 40), +48 verbs (40 → 88) = +57 types total - * **Coverage**: 60% (v5.4.0) → 95% (Stage 3) + * **Coverage**: 60% → 95% (Stage 3) */ // Common metadata types diff --git a/src/types/typeMigration.ts b/src/types/typeMigration.ts index a1a3c058..de4d3e06 100644 --- a/src/types/typeMigration.ts +++ b/src/types/typeMigration.ts @@ -1,5 +1,5 @@ /** - * Type Migration Utilities for Stage 3 Taxonomy (v6.0.0) + * Type Migration Utilities for Stage 3 Taxonomy * * Provides migration helpers for code using removed types from v5.x * @@ -52,7 +52,7 @@ export function isRemovedVerbType(type: string): type is keyof typeof REMOVED_VE } /** - * Migrate a noun type from v5.x to v6.0 Stage 3 + * Migrate a noun type (Stage 3) * Returns the migrated type or the original if no migration needed */ export function migrateNounType(type: string): NounType { @@ -64,7 +64,7 @@ export function migrateNounType(type: string): NounType { } /** - * Migrate a verb type from v5.x to v6.0 Stage 3 + * Migrate a verb type (Stage 3) * Returns the migrated type or the original if no migration needed * * WARNING: Some verbs require inverting source/target relationships! @@ -138,7 +138,7 @@ export function migrateRelationship(params: { /** * Stage 3 Type Compatibility Check - * Helps developers identify code that needs updating for v6.0 + * Helps developers identify code that needs updating */ export function checkTypeCompatibility(nounTypes: string[], verbTypes: string[]): { valid: boolean diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index 1aff7e6b..8965e097 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -54,7 +54,7 @@ export class EntityIdMapper { async init(): Promise { try { const metadata = await this.storage.getMetadata(this.storageKey) - // v4.8.0: metadata IS the data (no nested 'data' property) + // metadata IS the data (no nested 'data' property) if (metadata && (metadata as any).nextId !== undefined) { const data = metadata as any as EntityIdMapperData this.nextId = data.nextId @@ -87,7 +87,7 @@ export class EntityIdMapper { } /** - * v7.5.0: Get integer ID for UUID with immediate persistence guarantee + * Get integer ID for UUID with immediate persistence guarantee * Unlike getOrAssign(), this method flushes to storage immediately after assigning * a new ID. This prevents UUID→int mapping divergence if the process crashes * before a normal flush() occurs. @@ -194,7 +194,7 @@ export class EntityIdMapper { } // Convert maps to plain objects for serialization - // v4.0.0: Add required 'noun' property for NounMetadata + // Add required 'noun' property for NounMetadata const data = { noun: 'EntityIdMapper', nextId: this.nextId, diff --git a/src/utils/fieldTypeInference.ts b/src/utils/fieldTypeInference.ts index 29bc5a94..b3d9fa1e 100644 --- a/src/utils/fieldTypeInference.ts +++ b/src/utils/fieldTypeInference.ts @@ -388,7 +388,7 @@ export class FieldTypeInference { const data = await this.storage.getMetadata(cacheKey) if (data) { - // v4.0.0: Double cast for type boundary crossing + // Double cast for type boundary crossing return data as unknown as FieldTypeInfo } } catch (error) { @@ -406,7 +406,7 @@ export class FieldTypeInference { this.typeCache.set(field, typeInfo) // Save to persistent storage (async, non-blocking) - // v4.0.0: Add required 'noun' property for NounMetadata + // Add required 'noun' property for NounMetadata const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}` const metadataObj = { noun: 'FieldTypeCache', @@ -487,7 +487,7 @@ export class FieldTypeInference { if (field) { this.typeCache.delete(field) const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}` - // v4.0.0: null signals deletion to storage adapter + // null signals deletion to storage adapter await this.storage.saveMetadata(cacheKey, null as any) } else { this.typeCache.clear() diff --git a/src/utils/import-progress-tracker.ts b/src/utils/import-progress-tracker.ts index 7090eee1..15b1d407 100644 --- a/src/utils/import-progress-tracker.ts +++ b/src/utils/import-progress-tracker.ts @@ -1,5 +1,5 @@ /** - * Import Progress Tracker (v4.5.0) + * Import Progress Tracker * * Comprehensive progress tracking for imports with: * - Multi-dimensional progress (bytes, entities, stages, timing) @@ -8,7 +8,6 @@ * - Throttled callbacks (avoid spam) * - Weighted overall progress * - * @since v4.5.0 */ import { diff --git a/src/utils/memoryDetection.ts b/src/utils/memoryDetection.ts index a0a67b52..df7c14a0 100644 --- a/src/utils/memoryDetection.ts +++ b/src/utils/memoryDetection.ts @@ -49,7 +49,7 @@ export interface CacheAllocationStrategy { /** Environment type detected */ environment: 'production' | 'development' | 'container' | 'unknown' - /** Model memory reserved (bytes) - v3.36.0+ */ + /** Model memory reserved (bytes) */ modelMemory: number /** Model precision (q8 or fp32) */ @@ -195,7 +195,7 @@ function detectCgroupV1Memory(): number | null { * Calculate optimal cache size based on available memory * Scales intelligently from 2GB to 128GB+ * - * v3.36.0+: Accounts for embedding model memory (150MB Q8, 250MB FP32) + * Accounts for embedding model memory (150MB Q8, 250MB FP32) */ export function calculateOptimalCacheSize( memoryInfo: MemoryInfo, @@ -219,7 +219,7 @@ export function calculateOptimalCacheSize( const minSize = options.minSize || 256 * 1024 * 1024 // 256MB minimum const maxSize = options.maxSize || null - // Detect model memory usage (v3.36.0+) + // Detect model memory usage const modelInfo = detectModelMemory({ precision: options.modelPrecision || 'q8' }) const modelMemory = modelInfo.bytes diff --git a/src/utils/metadataFilter.ts b/src/utils/metadataFilter.ts index 44bc2ad2..f38df4f5 100644 --- a/src/utils/metadataFilter.ts +++ b/src/utils/metadataFilter.ts @@ -323,7 +323,7 @@ export function filterSearchResultsByMetadata( /** * Filter nouns by metadata before search - * v4.0.0: Takes HNSWNounWithMetadata which includes metadata field + * Takes HNSWNounWithMetadata which includes metadata field */ export function filterNounsByMetadata( nouns: HNSWNounWithMetadata[], diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index fa0c3903..2f8fb18d 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -76,7 +76,7 @@ interface FieldStats { rangeQueryCount: number exactQueryCount: number avgQueryTime: number - indexType: 'hash' // v3.42.0: Only 'hash' since all fields use chunked sparse indices with zone maps + indexType: 'hash' // Only 'hash' since all fields use chunked sparse indices with zone maps normalizationStrategy?: 'none' | 'precision' | 'bucket' } @@ -121,19 +121,19 @@ export class MetadataIndexManager { private lockPromises = new Map>() private lockTimers = new Map() // Track timers for cleanup - // Adaptive Chunked Sparse Indexing (v3.42.0 → v3.44.1) + // Adaptive Chunked Sparse Indexing // Reduces file count from 560k → 89 files (630x reduction) // ALL fields now use chunking - no more flat files - // v3.44.1: Removed sparseIndices Map - now lazy-loaded via UnifiedCache only + // Removed sparseIndices Map - now lazy-loaded via UnifiedCache only // PROJECTED: Reduces metadata memory from 35GB → 5GB @ 1B scale (86% reduction from chunking strategy, not yet benchmarked) private chunkManager: ChunkManager private chunkingStrategy: AdaptiveChunkingStrategy - // Roaring Bitmap Support (v3.43.0) + // Roaring Bitmap Support // EntityIdMapper for UUID ↔ integer conversion private idMapper: EntityIdMapper - // Field Type Inference (v3.48.0 - Production-ready value-based type detection) + // Field Type Inference (Production-ready value-based type detection) // Replaces unreliable pattern matching with DuckDB-inspired value analysis private fieldTypeInference: FieldTypeInference @@ -179,20 +179,20 @@ export class MetadataIndexManager { // Get global unified cache for coordinated memory management this.unifiedCache = getGlobalCache() - // Initialize EntityIdMapper for roaring bitmap UUID ↔ integer mapping (v3.43.0) + // Initialize EntityIdMapper for roaring bitmap UUID ↔ integer mapping this.idMapper = new EntityIdMapper({ storage, storageKey: 'brainy:entityIdMapper' }) - // Initialize chunking system (v3.42.0) with roaring bitmap support + // Initialize chunking system with roaring bitmap support this.chunkManager = new ChunkManager(storage, this.idMapper) this.chunkingStrategy = new AdaptiveChunkingStrategy() - // Initialize Field Type Inference (v3.48.0) + // Initialize Field Type Inference this.fieldTypeInference = new FieldTypeInference(storage) - // v6.2.2: Removed lazyLoadCounts() call from constructor + // Removed lazyLoadCounts() call from constructor // It was a race condition (not awaited) and read from wrong source. // Now properly called in init() after warmCache() loads the sparse index. } @@ -205,18 +205,18 @@ export class MetadataIndexManager { // Initialize roaring-wasm library (browser bundle requires async init) await roaringLibraryInitialize() - // Load field registry to discover persisted indices (v4.2.1) + // Load field registry to discover persisted indices // Must run first to populate fieldIndexes directory before warming cache await this.loadFieldRegistry() // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage) await this.idMapper.init() - // Warm the cache with common fields (v3.44.1 - lazy loading optimization) + // Warm the cache with common fields (lazy loading optimization) // This loads the 'noun' sparse index which is needed for type counts await this.warmCache() - // v6.2.2: Load type counts AFTER warmCache (sparse index is now cached) + // Load type counts AFTER warmCache (sparse index is now cached) // Previously called in constructor without await and read from wrong source await this.lazyLoadCounts() @@ -224,16 +224,16 @@ export class MetadataIndexManager { // Now correctly happens AFTER lazyLoadCounts() finishes this.syncTypeCountsToFixed() - // v7.5.0: Detect index corruption and auto-rebuild if necessary + // Detect index corruption and auto-rebuild if necessary // The update() field asymmetry bug caused indexes to accumulate stale entries // This check runs on startup to detect and repair corrupted indexes automatically await this.detectAndRepairCorruption() } /** - * v7.5.0: Detect index corruption and automatically repair via rebuild + * Detect index corruption and automatically repair via rebuild * This catches the update() field asymmetry bug that causes 7 fields to accumulate per update - * Corruption threshold: 100 avg entries/entity (expected ~30) + * Corruption threshold: 100 avg metadata entries/entity, excluding __words__ (expected ~30) */ private async detectAndRepairCorruption(): Promise { const validation = await this.validateConsistency() @@ -260,7 +260,7 @@ export class MetadataIndexManager { } /** - * Warm the cache by preloading common field sparse indices (v3.44.1) + * Warm the cache by preloading common field sparse indices * This improves cache hit rates by loading frequently-accessed fields at startup * Target: >80% cache hit rate for typical workloads */ @@ -413,18 +413,18 @@ export class MetadataIndexManager { /** * Lazy load entity counts from the 'noun' field sparse index (O(n) where n = number of types) - * v6.2.2 FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed + * FIX: Previously read from stats.nounCount which was SERVICE-keyed, not TYPE-keyed * Now computes counts from the sparse index which has the correct type information */ private async lazyLoadCounts(): Promise { try { - // v6.2.4: CRITICAL FIX - Clear counts before loading to prevent accumulation + // CRITICAL FIX - Clear counts before loading to prevent accumulation // Previously, counts accumulated across restarts causing 100x inflation this.totalEntitiesByType.clear() this.entityCountsByTypeFixed.fill(0) this.verbCountsByTypeFixed.fill(0) - // v6.2.2: Load counts from sparse index (correct source) + // Load counts from sparse index (correct source) const nounSparseIndex = await this.loadSparseIndex('noun') if (!nounSparseIndex) { // No sparse index yet - counts will be populated as entities are added @@ -522,7 +522,7 @@ export class MetadataIndexManager { const stats = this.fieldStats.get(field)! const cardinality = stats.cardinality - // Track unique values by checking fieldIndex counts (v3.42.0 - removed indexCache) + // Track unique values by checking fieldIndex counts const fieldIndex = this.fieldIndexes.get(field) const normalizedValue = this.normalizeValue(value, field) const currentCount = fieldIndex?.values[normalizedValue] || 0 @@ -581,7 +581,7 @@ export class MetadataIndexManager { private updateIndexStrategy(field: string, stats: FieldStats): void { const hasHighCardinality = stats.cardinality.uniqueValues > this.HIGH_CARDINALITY_THRESHOLD - // All fields use chunked sparse indexing with zone maps (v3.42.0) + // All fields use chunked sparse indexing with zone maps stats.indexType = 'hash' // Determine normalization strategy for high cardinality NON-temporal fields @@ -603,7 +603,7 @@ export class MetadataIndexManager { } // ============================================================================ - // Adaptive Chunked Sparse Indexing (v3.42.0) + // Adaptive Chunked Sparse Indexing // All fields use chunking - simplified implementation // ============================================================================ @@ -652,8 +652,8 @@ export class MetadataIndexManager { } /** - * Get IDs for a value using chunked sparse index with roaring bitmaps (v3.43.0) - * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + * Get IDs for a value using chunked sparse index with roaring bitmaps + * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) */ private async getIdsFromChunks(field: string, value: any): Promise { // Load sparse index via UnifiedCache (lazy loading) @@ -690,9 +690,9 @@ export class MetadataIndexManager { } /** - * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps (v3.43.0) - * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) - * v4.5.4: Normalize min/max for timestamp bucketing before comparison + * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps + * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + * Normalize min/max for timestamp bucketing before comparison */ private async getIdsFromChunksForRange( field: string, @@ -707,7 +707,7 @@ export class MetadataIndexManager { return [] // No chunked index exists yet } - // v4.5.4: Normalize min/max for consistent comparison with indexed values + // Normalize min/max for consistent comparison with indexed values // (indexed values are bucketed for timestamps, so we must bucket the query bounds too) const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined @@ -751,9 +751,9 @@ export class MetadataIndexManager { } /** - * Get roaring bitmap for a field-value pair without converting to UUIDs (v3.43.0) + * Get roaring bitmap for a field-value pair without converting to UUIDs * This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND - * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) * @returns RoaringBitmap32 containing integer IDs, or null if no matches */ private async getBitmapFromChunks(field: string, value: any): Promise { @@ -806,7 +806,7 @@ export class MetadataIndexManager { } /** - * Get IDs for multiple field-value pairs using fast roaring bitmap intersection (v3.43.0) + * Get IDs for multiple field-value pairs using fast roaring bitmap intersection * * This method provides 500-900x faster multi-field queries by: * - Using hardware-accelerated bitmap AND operations (SIMD: AVX2/SSE4.2) @@ -862,7 +862,7 @@ export class MetadataIndexManager { /** * Add value-ID mapping to chunked index - * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) */ private async addToChunkedIndex(field: string, value: any, id: string): Promise { // Load or create sparse index via UnifiedCache (lazy loading) @@ -964,7 +964,7 @@ export class MetadataIndexManager { /** * Remove ID from chunked index - * v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) */ private async removeFromChunkedIndex(field: string, value: any, id: string): Promise { // Load sparse index via UnifiedCache (lazy loading) @@ -1013,7 +1013,7 @@ export class MetadataIndexManager { stats.rangeQueryCount++ } - // All fields use chunked sparse index with zone map optimization (v3.42.0) + // All fields use chunked sparse index with zone map optimization return await this.getIdsFromChunksForRange(field, min, max, includeMin, includeMax) } @@ -1047,7 +1047,7 @@ export class MetadataIndexManager { /** * Normalize value for consistent indexing with VALUE-BASED temporal detection * - * v3.48.0: Replaced unreliable field name pattern matching with production-ready + * Replaced unreliable field name pattern matching with production-ready * value-based detection (DuckDB-inspired). Analyzes actual data values, not names. * * NO FALLBACKS - Pure value-based detection only. @@ -1153,14 +1153,14 @@ export class MetadataIndexManager { /** * Extract indexable field-value pairs from entity or metadata * - * v4.8.0: Now handles BOTH entity structure (with top-level fields) AND plain metadata + * Now handles BOTH entity structure (with top-level fields) AND plain metadata * - Extracts from top-level fields (confidence, weight, timestamps, type, service, etc.) * - Also extracts from nested metadata field (custom user fields) * - Skips HNSW-specific fields (vector, connections, level, id) * - Maps 'type' → 'noun' for backward compatibility with existing indexes * - * BUG FIX (v3.50.1): Exclude vector embeddings and large arrays from indexing - * BUG FIX (v3.50.2): Also exclude purely numeric field names (array indices) + * BUG FIX: Exclude vector embeddings and large arrays from indexing + * BUG FIX: Also exclude purely numeric field names (array indices) * - Vector fields (384+ dimensions) were creating 825K chunk files for 1,144 entities * - Arrays converted to objects with numeric keys were still being indexed */ @@ -1186,7 +1186,7 @@ export class MetadataIndexManager { if (!this.shouldIndexField(fullKey)) continue // Special handling for metadata field at top level - // v4.8.0: Flatten metadata fields to top-level (no prefix) for cleaner queries + // Flatten metadata fields to top-level (no prefix) for cleaner queries // Standard fields are already at top-level, custom fields go in metadata // By flattening here, queries can use { category: 'B' } instead of { 'metadata.category': 'B' } if (key === 'metadata' && !prefix && typeof value === 'object' && !Array.isArray(value)) { @@ -1211,7 +1211,7 @@ export class MetadataIndexManager { } } else { // Primitive value: index it - // v4.8.0: Map 'type' → 'noun' for backward compatibility + // Map 'type' → 'noun' for backward compatibility const indexField = (!prefix && key === 'type') ? 'noun' : fullKey fields.push({ field: indexField, value }) } @@ -1222,8 +1222,8 @@ export class MetadataIndexManager { extract(data) } - // v7.7.0: Extract words for hybrid text search - // v7.8.0: Production-scale word limit (5000 words) + // Extract words for hybrid text search + // Production-scale word limit (5000 words) // - Handles articles, chapters, and large documents // - Roaring Bitmaps + Chunked Sparse Index + LRU caching // - Int32 hashes store words as 4-byte values, not strings @@ -1258,7 +1258,7 @@ export class MetadataIndexManager { } /** - * Extract text content from entity data for word indexing (v7.7.0) + * Extract text content from entity data for word indexing * * Recursively extracts string values from data, excluding: * - vector, embedding, connections, level, id (internal fields) @@ -1292,7 +1292,7 @@ export class MetadataIndexManager { } /** - * Tokenize text into words for indexing (v7.7.0) + * Tokenize text into words for indexing * * - Converts to lowercase * - Removes punctuation @@ -1314,7 +1314,7 @@ export class MetadataIndexManager { } /** - * Hash word to int32 using FNV-1a (v7.7.0) + * Hash word to int32 using FNV-1a * * FNV-1a is fast with low collision rate, suitable for word hashing. * Saves ~10GB at billion scale by avoiding string storage. @@ -1332,7 +1332,7 @@ export class MetadataIndexManager { } /** - * Get entity IDs matching a text query (v7.7.0) + * Get entity IDs matching a text query * * Performs word-based text search using the __words__ index. * Returns IDs ranked by match count (entities with more matching words first). @@ -1375,19 +1375,19 @@ export class MetadataIndexManager { /** * Add item to metadata indexes * - * v4.8.0: Now accepts either entity structure or plain metadata + * Now accepts either entity structure or plain metadata * - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} } * - Plain metadata: { noun, confidence, weight, createdAt, ... } * * @param id - Entity ID - * @param entityOrMetadata - Either full entity structure (v4.8.0+) or plain metadata (backward compat) + * @param entityOrMetadata - Either full entity structure or plain metadata (backward compat) * @param skipFlush - Skip automatic flush (used during batch operations) */ async addToIndex(id: string, entityOrMetadata: any, skipFlush: boolean = false): Promise { const fields = this.extractIndexableFields(entityOrMetadata) - // v6.7.0: Sanity check for excessive indexed fields (indicates possible data issue) - // v7.8.0: Separate threshold for metadata fields vs word fields + // Sanity check for excessive indexed fields (indicates possible data issue) + // Separate threshold for metadata fields vs word fields // - Metadata fields: warn if > 100 (indicates deeply nested metadata) // - Word fields: expected to be many for large documents, warn only for extreme cases const metadataFields = fields.filter(f => f.field !== '__words__') @@ -1418,7 +1418,7 @@ export class MetadataIndexManager { for (let i = 0; i < fields.length; i++) { const { field, value } = fields[i] - // All fields use chunked sparse indexing (v3.42.0) + // All fields use chunked sparse indexing await this.addToChunkedIndex(field, value, id) // Update statistics and tracking @@ -1432,7 +1432,7 @@ export class MetadataIndexManager { } } - // Adaptive auto-flush based on usage patterns (v3.42.0 - flush field indexes only) + // Adaptive auto-flush based on usage patterns if (!skipFlush) { const timeSinceLastFlush = Date.now() - this.lastFlushTime const shouldAutoFlush = @@ -1494,7 +1494,7 @@ export class MetadataIndexManager { /** * Remove item from metadata indexes * - * v4.8.0: Now accepts either entity structure or plain metadata (same as addToIndex) + * Now accepts either entity structure or plain metadata (same as addToIndex) * - Entity structure: { id, type, confidence, weight, createdAt, metadata: {...} } * - Plain metadata: { noun, confidence, weight, createdAt, ... } * @@ -1507,7 +1507,7 @@ export class MetadataIndexManager { const fields = this.extractIndexableFields(metadata) for (const { field, value } of fields) { - // All fields use chunked sparse indexing (v3.42.0) + // All fields use chunked sparse indexing await this.removeFromChunkedIndex(field, value, id) // Update statistics and tracking @@ -1521,7 +1521,7 @@ export class MetadataIndexManager { } else { // Remove from all indexes (slower, requires scanning all field indexes) // This should be rare - prefer providing metadata when removing - // v3.44.1: Scan via fieldIndexes, load sparse indices on-demand + // Scan via fieldIndexes, load sparse indices on-demand prodLog.warn(`Removing ID ${id} without metadata requires scanning all fields (slow)`) // Scan all fields via fieldIndexes @@ -1552,7 +1552,7 @@ export class MetadataIndexManager { * Get all IDs in the index */ async getAllIds(): Promise { - // Use storage as the source of truth (v3.42.0 - removed redundant indexCache scan) + // Use storage as the source of truth const allIds = new Set() // Storage.getNouns() is the definitive source of all entity IDs @@ -1586,7 +1586,7 @@ export class MetadataIndexManager { stats.exactQueryCount++ } - // All fields use chunked sparse indexing (v3.42.0) + // All fields use chunked sparse indexing return await this.getIdsFromChunks(field, value) } @@ -1739,7 +1739,7 @@ export class MetadataIndexManager { subIds.forEach(id => unionIds.add(id)) } - // v6.2.1: Fix - Check for outer-level field conditions that need AND application + // Fix - Check for outer-level field conditions that need AND application // This handles cases like { anyOf: [...], vfsType: { exists: false } } // where the anyOf results must be intersected with other field conditions const outerFields = Object.keys(filter).filter( @@ -1770,21 +1770,21 @@ export class MetadataIndexManager { let fieldResults: string[] = [] if (condition && typeof condition === 'object' && !Array.isArray(condition)) { - // Handle Brainy Field Operators (v4.5.4: canonical operators defined) + // Handle Brainy Field Operators (canonical operators defined) // See docs/api/README.md for complete operator reference for (const [op, operand] of Object.entries(condition)) { switch (op) { // ===== EQUALITY OPERATORS ===== - // Canonical: 'eq' | Alias: 'equals' | Deprecated: 'is' (remove in v5.0.0) - case 'is': // DEPRECATED (v4.5.4): Use 'eq' instead + // Canonical: 'eq' | Alias: 'equals' | Deprecated: 'is' + case 'is': // DEPRECATED: Use 'eq' instead case 'equals': // Alias for 'eq' case 'eq': fieldResults = await this.getIds(field, operand) break // ===== NEGATION OPERATORS ===== - // Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot' (remove in v5.0.0) - case 'isNot': // DEPRECATED (v4.5.4): Use 'ne' instead + // Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot' + case 'isNot': // DEPRECATED: Use 'ne' instead case 'notEquals': // Alias for 'ne' case 'ne': // For notEquals, we need all IDs EXCEPT those matching the value @@ -1824,8 +1824,8 @@ export class MetadataIndexManager { break // ===== GREATER THAN OR EQUAL OPERATORS ===== - // Canonical: 'gte' | Alias: 'greaterThanOrEqual' | Deprecated: 'greaterEqual' (remove in v5.0.0) - case 'greaterEqual': // DEPRECATED (v4.5.4): Use 'gte' instead + // Canonical: 'gte' | Alias: 'greaterThanOrEqual' | Deprecated: 'greaterEqual' + case 'greaterEqual': // DEPRECATED: Use 'gte' instead case 'greaterThanOrEqual': // Alias for 'gte' case 'gte': fieldResults = await this.getIdsForRange(field, operand, undefined, true, true) @@ -1839,8 +1839,8 @@ export class MetadataIndexManager { break // ===== LESS THAN OR EQUAL OPERATORS ===== - // Canonical: 'lte' | Alias: 'lessThanOrEqual' | Deprecated: 'lessEqual' (remove in v5.0.0) - case 'lessEqual': // DEPRECATED (v4.5.4): Use 'lte' instead + // Canonical: 'lte' | Alias: 'lessThanOrEqual' | Deprecated: 'lessEqual' + case 'lessEqual': // DEPRECATED: Use 'lte' instead case 'lessThanOrEqual': // Alias for 'lte' case 'lte': fieldResults = await this.getIdsForRange(field, undefined, operand, true, true) @@ -1865,8 +1865,8 @@ export class MetadataIndexManager { case 'exists': if (operand) { // exists: true - Get all IDs that have this field (any value) - // v3.43.0: From chunked sparse index with roaring bitmaps - // v3.44.1: Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) + // From chunked sparse index with roaring bitmaps + // Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) const allIntIds = new Set() // Load sparse index via UnifiedCache (lazy loading) @@ -1890,7 +1890,7 @@ export class MetadataIndexManager { fieldResults = this.idMapper.intsIterableToUuids(allIntIds) } else { // exists: false - Get all IDs that DON'T have this field - // v5.7.9: Fixed excludeVFS bug (was returning empty array) + // Fixed excludeVFS bug (was returning empty array) const allItemIds = await this.getAllIds() const existsIntIds = new Set() @@ -1921,7 +1921,7 @@ export class MetadataIndexManager { case 'missing': // missing: true is equivalent to exists: false // missing: false is equivalent to exists: true - // v5.7.9: Added for API consistency with in-memory metadataFilter + // Added for API consistency with in-memory metadataFilter if (operand) { // missing: true - field does NOT exist (same as exists: false) const allItemIds = await this.getAllIds() @@ -2021,7 +2021,6 @@ export class MetadataIndexManager { * @param order - Sort direction: 'asc' (default) or 'desc' * @returns Promise - Entity IDs sorted by specified field * - * @since v4.5.4 */ async getSortedIdsForFilter( filter: any, @@ -2084,7 +2083,6 @@ export class MetadataIndexManager { * @returns Promise - Field value or undefined if not found * * @public (called from brainy.ts for sorted queries) - * @since v4.5.4 */ async getFieldValueForEntity(entityId: string, field: string): Promise { // For timestamp fields, load ACTUAL value from entity metadata @@ -2152,7 +2150,6 @@ export class MetadataIndexManager { * @returns Denormalized value in original type * * @private - * @since v4.5.4 */ private denormalizeValue(normalized: string, field: string): any { // Try parsing as number (timestamps, integers, floats) @@ -2233,7 +2230,7 @@ export class MetadataIndexManager { /** * Flush dirty entries to storage (non-blocking version) - * NOTE (v3.42.0): Sparse indices are flushed immediately in add/remove operations + * NOTE: Sparse indices are flushed immediately in add/remove operations */ async flush(): Promise { // Check if we have anything to flush @@ -2245,7 +2242,7 @@ export class MetadataIndexManager { const BATCH_SIZE = 20 const allPromises: Promise[] = [] - // Flush field indexes in batches (v3.42.0 - removed flat file flushing) + // Flush field indexes in batches const dirtyFieldsArray = Array.from(this.dirtyFields) for (let i = 0; i < dirtyFieldsArray.length; i += BATCH_SIZE) { const batch = dirtyFieldsArray.slice(i, i + BATCH_SIZE) @@ -2264,10 +2261,10 @@ export class MetadataIndexManager { // Wait for all operations to complete await Promise.all(allPromises) - // Flush EntityIdMapper (UUID ↔ integer mappings) (v3.43.0) + // Flush EntityIdMapper (UUID ↔ integer mappings) await this.idMapper.flush() - // Save field registry for fast cold-start discovery (v4.2.1) + // Save field registry for fast cold-start discovery await this.saveFieldRegistry() this.dirtyFields.clear() @@ -2347,7 +2344,7 @@ export class MetadataIndexManager { const indexId = `__metadata_field_index__${filename}` const unifiedKey = `metadata:field:${filename}` - // v4.0.0: Add required 'noun' property for NounMetadata + // Add required 'noun' property for NounMetadata await this.storage.saveMetadata(indexId, { noun: 'MetadataFieldIndex', values: fieldIndex.values, @@ -2369,7 +2366,7 @@ export class MetadataIndexManager { /** * Save field registry to storage for fast cold-start discovery - * v4.2.1: Solves 100x performance regression by persisting field directory + * Solves 100x performance regression by persisting field directory * * This enables instant cold starts by discovering which fields have persisted indices * without needing to rebuild from scratch. Similar to how HNSW persists system metadata. @@ -2404,7 +2401,7 @@ export class MetadataIndexManager { /** * Load field registry from storage to populate fieldIndexes directory - * v4.2.1: Enables O(1) discovery of persisted sparse indices + * Enables O(1) discovery of persisted sparse indices * * Called during init() to discover which fields have persisted indices. * Populates fieldIndexes Map with skeleton entries - actual sparse indices @@ -2449,7 +2446,7 @@ export class MetadataIndexManager { /** * Get list of persisted fields from storage (not in-memory) - * v6.7.0: Used during rebuild to discover which chunk files need deletion + * Used during rebuild to discover which chunk files need deletion * * @returns Array of field names that have persisted sparse indices */ @@ -2470,7 +2467,7 @@ export class MetadataIndexManager { /** * Delete all chunk files for a specific field - * v6.7.0: Used during rebuild to ensure clean slate + * Used during rebuild to ensure clean slate * * @param field Field name whose chunks should be deleted */ @@ -2500,7 +2497,7 @@ export class MetadataIndexManager { /** * Clear ALL metadata index data from storage (for recovery) - * v6.7.0: Nuclear option for recovering from corrupted index state + * Nuclear option for recovering from corrupted index state * * WARNING: This deletes all indexed data - requires full rebuild after! * Use when index is corrupted beyond normal rebuild repair. @@ -2565,7 +2562,7 @@ export class MetadataIndexManager { /** * Get all entity types and their counts - O(1) operation - * v6.2.2: Fixed - totalEntitiesByType is correctly populated by updateTypeFieldAffinity + * Fixed - totalEntitiesByType is correctly populated by updateTypeFieldAffinity * during add operations. lazyLoadCounts was reading wrong data but that doesn't * affect freshly-added entities within the same session. */ @@ -2574,7 +2571,7 @@ export class MetadataIndexManager { } // ============================================================================ - // v6.2.1: VFS Statistics Methods (uses existing Roaring bitmap infrastructure) + // VFS Statistics Methods (uses existing Roaring bitmap infrastructure) // ============================================================================ /** @@ -2744,14 +2741,14 @@ export class MetadataIndexManager { * Get count of entities matching field-value criteria - queries chunked sparse index */ async getCountForCriteria(field: string, value: any): Promise { - // Use chunked sparse indexing (v3.42.0 - removed indexCache) + // Use chunked sparse indexing const ids = await this.getIds(field, value) return ids.length } /** * Get index statistics with enhanced counting information - * v3.44.1: Sparse indices now lazy-loaded via UnifiedCache + * Sparse indices now lazy-loaded via UnifiedCache * Note: This method may load sparse indices to calculate stats */ async getStats(): Promise { @@ -2759,8 +2756,9 @@ export class MetadataIndexManager { let totalEntries = 0 let totalIds = 0 - // Collect stats from field indexes (lightweight - always in memory) + // Collect stats from metadata field indexes only (excludes __words__ keyword index) for (const field of this.fieldIndexes.keys()) { + if (field === '__words__') continue // Keyword index not included in metadata stats fields.add(field) // Load sparse index to count entries (may trigger lazy load) @@ -2779,7 +2777,7 @@ export class MetadataIndexManager { } } - // v6.7.0: Sanity check for index corruption (77x overcounting bug detection) + // Sanity check for index corruption (only metadata fields, not __words__ keyword index) const entityCount = this.idMapper.size if (entityCount > 0) { const avgIdsPerEntity = totalIds / entityCount @@ -2801,11 +2799,12 @@ export class MetadataIndexManager { } /** - * v7.5.0: Validate index consistency and detect corruption + * Validate index consistency and detect corruption * Returns health status and recommendations for repair * + * Counts metadata field entries only (excludes __words__ keyword index). * Corruption typically manifests as high avg entries/entity (expected ~30, corrupted can be 100+) - * caused by the update() field asymmetry bug (fixed in v7.5.0) + * caused by the update() field asymmetry bug */ async validateConsistency(): Promise<{ healthy: boolean @@ -2827,9 +2826,10 @@ export class MetadataIndexManager { } } - // Count total index entries across all fields + // Count total index entries across all fields (excluding keyword index) let indexEntryCount = 0 for (const field of this.fieldIndexes.keys()) { + if (field === '__words__') continue // Keyword entries are expected to be high-volume const sparseIndex = await this.loadSparseIndex(field) if (sparseIndex) { for (const chunkId of sparseIndex.getAllChunkIds()) { @@ -2845,7 +2845,8 @@ export class MetadataIndexManager { const avgEntriesPerEntity = indexEntryCount / entityCount - // Threshold: 100 entries/entity is clearly corrupted (expected ~30) + // Threshold: 100 metadata entries/entity is clearly corrupted (expected ~30) + // __words__ keyword entries are excluded from this count since they can be 50-5000 per entity // This catches the update() asymmetry bug which causes 7 fields to accumulate per update const CORRUPTION_THRESHOLD = 100 const healthy = avgEntriesPerEntity <= CORRUPTION_THRESHOLD @@ -2868,7 +2869,7 @@ export class MetadataIndexManager { /** * Rebuild entire index from scratch using pagination * Non-blocking version that yields control back to event loop - * v3.44.1: Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map) + * Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map) */ async rebuild(): Promise { if (this.isRebuilding) return @@ -2879,12 +2880,12 @@ export class MetadataIndexManager { prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`) prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`) - // Clear existing indexes (v3.42.0 - use sparse indices instead of flat files) - // v3.44.1: No sparseIndices Map to clear - UnifiedCache handles eviction + // Clear existing indexes + // No sparseIndices Map to clear - UnifiedCache handles eviction this.fieldIndexes.clear() this.dirtyFields.clear() - // v6.2.4: CRITICAL FIX - Clear type counts to prevent accumulation + // CRITICAL FIX - Clear type counts to prevent accumulation // Previously, counts accumulated across rebuilds causing incorrect values this.totalEntitiesByType.clear() this.entityCountsByTypeFixed.fill(0) @@ -2892,12 +2893,12 @@ export class MetadataIndexManager { this.typeFieldAffinity.clear() // Clear all cached sparse indices in UnifiedCache - // This ensures rebuild starts fresh (v3.44.1) + // This ensures rebuild starts fresh this.unifiedCache.clear('metadata') - // v6.7.0: CRITICAL FIX - Delete existing chunk files from storage + // CRITICAL FIX - Delete existing chunk files from storage // Without this, old chunk data accumulates with each rebuild causing 77x overcounting! - // Previous fix (v6.2.4) cleared type counts but missed chunk file accumulation. + // Previous fix cleared type counts but missed chunk file accumulation. prodLog.info('🗑️ Clearing existing metadata index chunks from storage...') const existingFields = await this.getPersistedFieldList() @@ -2916,13 +2917,13 @@ export class MetadataIndexManager { prodLog.info(`✅ Cleared ${existingFields.length} field indexes from storage`) } - // Clear EntityIdMapper to start fresh (v6.7.0) + // Clear EntityIdMapper to start fresh await this.idMapper.clear() // Clear chunk manager cache this.chunkManager.clearCache() - // Adaptive rebuild strategy based on storage adapter (v4.2.3) + // Adaptive rebuild strategy based on storage adapter // FileSystem/Memory/OPFS: Load all at once (avoids getAllShardedFiles() overhead on every batch) // Cloud (GCS/S3/R2): Use pagination with small batches (prevent socket exhaustion) const storageType = this.storage.constructor.name @@ -3394,7 +3395,7 @@ export class MetadataIndexManager { // This is the type definition itself entityType = this.normalizeValue(value, field) // Pass field for bucketing! } else if (metadata && metadata.noun) { - // Extract entity type from metadata (v3.42.0 - removed indexCache scan) + // Extract entity type from metadata entityType = this.normalizeValue(metadata.noun, 'noun') } else { // No type information available, skip affinity tracking diff --git a/src/utils/metadataIndexChunking.ts b/src/utils/metadataIndexChunking.ts index 4149bd08..677a50b4 100644 --- a/src/utils/metadataIndexChunking.ts +++ b/src/utils/metadataIndexChunking.ts @@ -592,7 +592,7 @@ export class ChunkManager { const data = await this.storage.getMetadata(chunkPath) if (data) { - // v4.0.0: Cast NounMetadata to chunk data structure + // Cast NounMetadata to chunk data structure const chunkData = data as unknown as any // Deserialize: convert serialized roaring bitmaps back to RoaringBitmap32 objects @@ -633,7 +633,7 @@ export class ChunkManager { this.chunkCache.set(cacheKey, chunk) // Serialize: convert RoaringBitmap32 to portable format (Buffer) - // v4.0.0: Add required 'noun' property for NounMetadata + // Add required 'noun' property for NounMetadata const serializable = { noun: 'IndexChunk', // Required by NounMetadata interface chunkId: chunk.chunkId, @@ -825,7 +825,7 @@ export class ChunkManager { this.chunkCache.delete(cacheKey) const chunkPath = this.getChunkPath(field, chunkId) - // v4.0.0: null signals deletion to storage adapter + // null signals deletion to storage adapter await this.storage.saveMetadata(chunkPath, null as any) } diff --git a/src/utils/periodicCleanup.ts b/src/utils/periodicCleanup.ts index 141d1b24..0b7ebbc0 100644 --- a/src/utils/periodicCleanup.ts +++ b/src/utils/periodicCleanup.ts @@ -215,7 +215,7 @@ export class PeriodicCleanup { for (const noun of nounsResult.items) { try { - // v4.0.0: Cast NounMetadata to NamespacedMetadata for isDeleted check + // Cast NounMetadata to NamespacedMetadata for isDeleted check if (!noun.metadata || !isDeleted(noun.metadata as any)) { continue // Not deleted, skip } diff --git a/src/utils/rebuildCounts.ts b/src/utils/rebuildCounts.ts index 9456ba4e..389e7237 100644 --- a/src/utils/rebuildCounts.ts +++ b/src/utils/rebuildCounts.ts @@ -2,7 +2,7 @@ * Rebuild Counts Utility * * Scans storage and rebuilds counts.json from actual data - * Use this to fix databases affected by the v4.1.1 count synchronization bug + * Use this to fix databases affected by the count synchronization bug * * NO MOCKS - Production-ready implementation */ @@ -64,12 +64,12 @@ export async function rebuildCounts(storage: BaseStorage): Promise 10) { this.checkProactiveFairness(type) @@ -341,7 +341,7 @@ export class UnifiedCache { other: typeSizes.other / totalSize } - // Check for starvation (v3.40.2: more aggressive - 70% cache with <15% accesses) + // Check for starvation (more aggressive - 70% cache with <15% accesses) // Previous: 90% cache, <10% access (too lenient, caused thrashing) for (const type of ['hnsw', 'metadata', 'embedding', 'other'] as const) { if (sizeRatios[type] > 0.7 && accessRatios[type] < 0.15) { @@ -352,7 +352,7 @@ export class UnifiedCache { } /** - * Proactive fairness check (v3.40.2) + * Proactive fairness check * Called immediately when adding items to prevent imbalance formation * Uses same thresholds as periodic check but runs on-demand */ @@ -391,7 +391,7 @@ export class UnifiedCache { // Sort by score (lower is worse) candidates.sort((a, b) => a[1] - b[1]) - // Evict bottom 50% of this type (v3.40.2: was 20%, too slow to prevent thrashing) + // Evict bottom 50% of this type (was 20%, too slow to prevent thrashing) const evictCount = Math.max(1, Math.floor(candidates.length * 0.5)) for (let i = 0; i < evictCount && i < candidates.length; i++) { @@ -417,7 +417,7 @@ export class UnifiedCache { /** * Delete all items with keys starting with the given prefix - * v6.2.9: Added for VFS cache invalidation (fixes stale parent ID bug) + * Added for VFS cache invalidation (fixes stale parent ID bug) * @param prefix - The key prefix to match * @returns Number of items deleted */ @@ -526,7 +526,7 @@ export class UnifiedCache { totalAccessCount: this.totalAccessCount, hitRate, - // Memory management (v3.36.0+) + // Memory management memory: { available: this.memoryInfo.available, source: this.memoryInfo.source, diff --git a/src/versioning/VersionDiff.ts b/src/versioning/VersionDiff.ts index bc1656a8..69f7dc15 100644 --- a/src/versioning/VersionDiff.ts +++ b/src/versioning/VersionDiff.ts @@ -1,5 +1,5 @@ /** - * VersionDiff - Deep Object Comparison for Entity Versions (v5.3.0) + * VersionDiff - Deep Object Comparison for Entity Versions * * Provides deep diff between entity versions: * - Field-level change detection diff --git a/src/versioning/VersionIndex.ts b/src/versioning/VersionIndex.ts index d5dfc4a7..b6bb10ce 100644 --- a/src/versioning/VersionIndex.ts +++ b/src/versioning/VersionIndex.ts @@ -1,5 +1,5 @@ /** - * VersionIndex - Pure Key-Value Version Storage (v6.3.0) + * VersionIndex - Pure Key-Value Version Storage * * Stores version metadata using simple key-value storage: * - Version list per entity: __version_meta_{entityId}_{branch} diff --git a/src/versioning/VersionManager.ts b/src/versioning/VersionManager.ts index 5cd9d367..78041a42 100644 --- a/src/versioning/VersionManager.ts +++ b/src/versioning/VersionManager.ts @@ -1,14 +1,14 @@ /** - * VersionManager - Entity-Level Versioning Engine (v5.3.0, v6.3.0 fix) + * VersionManager - Entity-Level Versioning Engine * * Provides entity-level version control with: * - save() - Create entity version - * - restore() - Restore entity to specific version (v6.3.0: now updates all indexes) + * - restore() - Restore entity to specific version (now updates all indexes) * - list() - List all versions of an entity * - compare() - Deep diff between versions * - prune() - Remove old versions (retention policies) * - * Architecture (v6.3.0 - Clean Key-Value Storage): + * Architecture: * - Versions stored as key-value pairs, NOT as entities (no index pollution) * - Content-addressable: SHA-256 hashing for deduplication * - Space-efficient: Only stores unique content @@ -206,7 +206,7 @@ export class VersionManager { throw new Error(`Entity ${entityId} not found`) } - // v6.3.2 FIX: For VFS file entities, fetch current content from blob storage + // FIX: For VFS file entities, fetch current content from blob storage // The entity.data field contains stale embedding text, not actual file content // VFS files store their real content in BlobStorage (content-addressable) if (this.isVFSFile(entity)) { @@ -416,7 +416,7 @@ export class VersionManager { ) } - // v6.3.2 FIX: For VFS file entities, write content back to blob storage + // FIX: For VFS file entities, write content back to blob storage // The versioned data contains the actual file content (not stale embedding text) // Using vfs.writeFile() ensures proper blob creation and metadata update if (this.isVFSFile(versionedEntity)) { diff --git a/src/versioning/VersionStorage.ts b/src/versioning/VersionStorage.ts index 9da633ff..abc7d4cc 100644 --- a/src/versioning/VersionStorage.ts +++ b/src/versioning/VersionStorage.ts @@ -1,13 +1,13 @@ /** - * VersionStorage - Hybrid Storage for Entity Versions (v5.3.0, v6.3.0 fix) + * VersionStorage - Hybrid Storage for Entity Versions * * Implements content-addressable storage for entity versions: * - SHA-256 content hashing for deduplication - * - Uses BaseStorage.saveMetadata/getMetadata for storage (v6.3.0) + * - Uses BaseStorage.saveMetadata/getMetadata for storage * - Integrates with COW commit system * - Space-efficient: Only stores unique content * - * Storage structure (v6.3.0): + * Storage structure: * Version content is stored using system metadata keys: * __system_version_{entityId}_{contentHash} * @@ -160,7 +160,7 @@ export class VersionStorage { * @returns Storage key for version content */ private getVersionPath(entityId: string, contentHash: string): string { - // v6.3.0: Use system-prefixed key for BaseStorage.saveMetadata/getMetadata + // Use system-prefixed key for BaseStorage.saveMetadata/getMetadata // BaseStorage recognizes __system_ prefix and routes to _system/ directory return `__system_version_${entityId}_${contentHash}` } @@ -173,7 +173,7 @@ export class VersionStorage { */ private async contentExists(key: string): Promise { try { - // v6.3.0: Use getMetadata to check existence + // Use getMetadata to check existence const adapter = this.brain.storageAdapter if (!adapter) return false @@ -200,7 +200,7 @@ export class VersionStorage { throw new Error('Storage adapter not available') } - // v6.3.0: Use saveMetadata for storing version content + // Use saveMetadata for storing version content // The key is system-prefixed so it routes to _system/ directory await adapter.saveMetadata(key, entity) } @@ -218,7 +218,7 @@ export class VersionStorage { throw new Error('Storage adapter not available') } - // v6.3.0: Use getMetadata for reading version content + // Use getMetadata for reading version content const entity = await adapter.getMetadata(key) if (!entity) { throw new Error(`Version data not found: ${key}`) @@ -234,14 +234,14 @@ export class VersionStorage { * Deleting the version index entry (via VersionIndex.removeVersion) is sufficient. * The content may be shared with other versions (same contentHash). * - * v6.3.0: We don't actually delete version content to avoid breaking + * We don't actually delete version content to avoid breaking * other versions that may reference the same content hash. * A separate garbage collection process could clean up unreferenced content. * * @param key Storage key (unused - kept for API compatibility) */ private async deleteVersionData(_key: string): Promise { - // v6.3.0: Version content is content-addressed and may be shared. + // Version content is content-addressed and may be shared. // We don't delete it here to prevent breaking other versions. // The version INDEX is deleted via VersionIndex.removeVersion(). // A GC process could clean up unreferenced content in the future. diff --git a/src/versioning/VersioningAPI.ts b/src/versioning/VersioningAPI.ts index aec78f80..5a946bc9 100644 --- a/src/versioning/VersioningAPI.ts +++ b/src/versioning/VersioningAPI.ts @@ -1,5 +1,5 @@ /** - * VersioningAPI - Public API for Entity Versioning (v5.3.0) + * VersioningAPI - Public API for Entity Versioning * * User-friendly wrapper around VersionManager with: * - Clean, simple API diff --git a/src/vfs/MimeTypeDetector.ts b/src/vfs/MimeTypeDetector.ts index 4d46f277..3d62a9e8 100644 --- a/src/vfs/MimeTypeDetector.ts +++ b/src/vfs/MimeTypeDetector.ts @@ -1,5 +1,5 @@ /** - * MIME Type Detection Service (v5.2.0) + * MIME Type Detection Service * * Provides comprehensive MIME type detection using: * 1. Industry-standard `mime` library (2000+ IANA types) diff --git a/src/vfs/PathResolver.ts b/src/vfs/PathResolver.ts index 47db1bba..3c8ad897 100644 --- a/src/vfs/PathResolver.ts +++ b/src/vfs/PathResolver.ts @@ -71,7 +71,7 @@ export class PathResolver { /** * Resolve a path to an entity ID - * v6.1.0: Uses 3-tier caching + MetadataIndexManager for optimal performance + * Uses 3-tier caching + MetadataIndexManager for optimal performance * Works for ALL storage adapters (FileSystem, GCS, S3, Azure, R2, OPFS) */ async resolve(path: string, options?: { @@ -255,14 +255,14 @@ export class PathResolver { // Still need to verify it exists } - // v4.7.0: Use proper graph traversal to find children + // Use proper graph traversal to find children // VFS relationships are now part of the knowledge graph const relations = await this.brain.getRelations({ from: parentId, type: VerbType.Contains }) - // v6.0.2: PERFORMANCE FIX - Batch fetch all children (eliminates N+1 pattern) + // PERFORMANCE FIX - Batch fetch all children (eliminates N+1 pattern) // Before: N sequential get() calls (10 children = 10 × 300ms = 3000ms on GCS) // After: 1 batch call (10 children = 1 × 300ms = 300ms on GCS) // 10x improvement for cloud storage (GCS, S3, Azure) @@ -292,7 +292,7 @@ export class PathResolver { * Uses proper graph relationships to traverse the tree */ async getChildren(dirId: string): Promise { - // v4.7.0: Use O(1) graph relationships (VFS creates these in mkdir/writeFile) + // Use O(1) graph relationships (VFS creates these in mkdir/writeFile) // VFS relationships are now part of the knowledge graph (no special filtering needed) const relations = await this.brain.getRelations({ from: dirId, @@ -302,12 +302,12 @@ export class PathResolver { const validChildren: VFSEntity[] = [] const childNames = new Set() - // v5.12.0: Batch fetch all child entities (eliminates N+1 query pattern) + // Batch fetch all child entities (eliminates N+1 query pattern) // This is WIRED UP AND USED - no longer a stub! const childIds = relations.map(r => r.to) const childrenMap = await this.brain.batchGet(childIds) - // v7.4.1: Deduplicate by entity ID to handle duplicate relationship records + // Deduplicate by entity ID to handle duplicate relationship records // This can occur when multiple Brainy instances create relationships concurrently // for the same storage path (each instance has its own in-memory GraphAdjacencyIndex). // The Set lookup is O(1), adding negligible overhead. @@ -356,7 +356,7 @@ export class PathResolver { } /** - * Invalidate ALL caches (v6.3.0) + * Invalidate ALL caches * Call this when switching branches (checkout), clearing data (clear), or forking * This ensures no stale data from previous branch/state remains in cache */ @@ -381,13 +381,13 @@ export class PathResolver { /** * Invalidate cache entries for a path and its children - * v6.2.9 FIX: Also invalidates UnifiedCache to prevent stale entity IDs + * FIX: Also invalidates UnifiedCache to prevent stale entity IDs * This fixes the "Source entity not found" bug after delete+recreate operations */ invalidatePath(path: string, recursive = false): void { const normalizedPath = this.normalizePath(path) - // v6.2.9 FIX: Clear parent cache BEFORE deleting from pathCache + // FIX: Clear parent cache BEFORE deleting from pathCache // (we need the entityId from the cache entry) const cached = this.pathCache.get(normalizedPath) if (cached) { @@ -398,7 +398,7 @@ export class PathResolver { this.pathCache.delete(normalizedPath) this.hotPaths.delete(normalizedPath) - // v6.2.9 CRITICAL FIX: Also invalidate UnifiedCache (global LRU cache) + // CRITICAL FIX: Also invalidate UnifiedCache (global LRU cache) // This was missing before, causing stale entity IDs to be returned after delete const cacheKey = `vfs:path:${normalizedPath}` getGlobalCache().delete(cacheKey) @@ -411,12 +411,12 @@ export class PathResolver { if (cachedPath.startsWith(prefix)) { this.pathCache.delete(cachedPath) this.hotPaths.delete(cachedPath) - // v6.2.9: Also clear parent cache for this entry + // Also clear parent cache for this entry this.parentCache.delete(entry.entityId) } } - // v6.2.9 CRITICAL FIX: Also invalidate UnifiedCache entries with this prefix + // CRITICAL FIX: Also invalidate UnifiedCache entries with this prefix const globalCachePrefix = `vfs:path:${prefix}` getGlobalCache().deleteByPrefix(globalCachePrefix) } @@ -561,7 +561,7 @@ export class PathResolver { /** * Get cache statistics - * v6.1.0: Added MetadataIndexManager metrics + * Added MetadataIndexManager metrics */ getStats(): { cacheSize: number diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 84c77843..9142e2ac 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -81,10 +81,10 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Mutex for preventing race conditions in directory creation private mkdirLocks: Map> = new Map() - // v5.8.0: Singleton promise for root initialization (prevents duplicate roots) + // Singleton promise for root initialization (prevents duplicate roots) private rootInitPromise: Promise | null = null - // v5.10.0: Fixed VFS root ID (prevents duplicates across instances) + // Fixed VFS root ID (prevents duplicates across instances) // Uses deterministic UUID format for storage compatibility private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' @@ -99,7 +99,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * v5.2.0: Access to BlobStorage for unified file storage + * Access to BlobStorage for unified file storage */ private get blobStorage() { // TypeScript doesn't know about blobStorage on storage, use type assertion @@ -119,14 +119,14 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Merge config with defaults this.config = { ...this.getDefaultConfig(), ...config } - // v5.0.1: VFS is now auto-initialized during brain.init() + // VFS is now auto-initialized during brain.init() // Brain is guaranteed to be initialized when this is called // Removed brain.init() check to prevent infinite recursion // Create or find root entity this.rootEntityId = await this.initializeRoot() - // v5.10.0: Clean up old UUID-based roots from v5.9.0 (one-time migration) + // Clean up old UUID-based roots (one-time migration) await this.cleanupOldRoots() // Initialize projection registry with auto-discovery of built-in projections @@ -180,7 +180,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * v5.8.0: CRITICAL FIX - Prevent duplicate root creation + * CRITICAL FIX - Prevent duplicate root creation * Uses singleton promise pattern to ensure only ONE root initialization * happens even with concurrent init() calls */ @@ -206,7 +206,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * v5.10.0: Atomic root initialization with fixed ID + * Atomic root initialization with fixed ID * Uses deterministic ID to prevent duplicates across all VFS instances * * ARCHITECTURAL FIX: Instead of query-then-create (race condition), @@ -266,7 +266,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * v5.10.0: Get standard root metadata + * Get standard root metadata * Centralized to ensure consistency */ private getRootMetadata(): VFSMetadata { @@ -286,7 +286,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * v5.10.0: Cleanup old UUID-based VFS roots (migration from v5.9.0) + * Cleanup old UUID-based VFS roots * Called during init to remove duplicate roots created before fixed-ID fix * * This is a one-time migration helper that can be removed in future versions. @@ -308,7 +308,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { const duplicates = oldRoots.filter(r => r.id !== VirtualFileSystem.VFS_ROOT_ID) if (duplicates.length > 0) { - console.log(`VFS: Found ${duplicates.length} old UUID-based root(s) from v5.9.0, cleaning up...`) + console.log(`VFS: Found ${duplicates.length} old UUID-based root(s), cleaning up...`) for (const duplicate of duplicates) { try { @@ -352,23 +352,23 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'readFile') } - // v5.2.0: Unified blob storage - ONE path only + // Unified blob storage - ONE path only if (!entity.metadata.storage?.type || entity.metadata.storage.type !== 'blob') { throw new VFSError( VFSErrorCode.EIO, - `File has no blob storage: ${path}. Requires v5.2.0+ storage format.`, + `File has no blob storage: ${path}. Requires blob storage format.`, path, 'readFile' ) } - // v5.8.0: CRITICAL FIX - Isolate blob errors from VFS tree corruption + // CRITICAL FIX - Isolate blob errors from VFS tree corruption // Blob read errors MUST NOT cascade to VFS tree structure try { // Read from BlobStorage (handles decompression automatically) const content = await this.blobStorage.read(entity.metadata.storage.hash) - // v6.2.0: REMOVED updateAccessTime() for performance + // REMOVED updateAccessTime() for performance // Access time updates caused 50-100ms GCS write on EVERY file read // Modern file systems use 'noatime' for same reason (performance) // Field 'accessed' still exists in metadata for backward compat but won't update @@ -436,7 +436,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { existingId = null } - // v5.2.0: Unified blob storage for ALL files (no size-based branching) + // Unified blob storage for ALL files (no size-based branching) // Store in BlobStorage (content-addressable, auto-deduplication, streaming) const blobHash = await this.blobStorage.write(buffer) @@ -450,7 +450,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { compressed: blobMetadata?.compressed } - // Detect MIME type (v5.2.0: using comprehensive MimeTypeDetector) + // Detect MIME type (using comprehensive MimeTypeDetector) const mimeType = mimeDetector.detectMimeType(name, buffer) // Create metadata @@ -459,8 +459,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { name, parent: parentId, vfsType: 'file', - isVFS: true, // v4.3.3: Mark as VFS entity (internal) - isVFSEntity: true, // v5.3.0: Explicit flag for developer filtering + isVFS: true, // Mark as VFS entity (internal) + isVFSEntity: true, // Explicit flag for developer filtering size: buffer.length, mimeType, extension: this.getExtension(name), @@ -470,7 +470,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { accessed: Date.now(), modified: Date.now(), storage: storageStrategy - // v5.2.0: No rawData - content is in BlobStorage + // No rawData - content is in BlobStorage // Backward compatibility: readFile() checks for rawData for legacy files } @@ -481,7 +481,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { if (existingId) { // Update existing file - // v5.2.0: No entity.data - content is in BlobStorage + // No entity.data - content is in BlobStorage await this.brain.update({ id: existingId, metadata @@ -500,7 +500,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { from: parentId, to: existingId, type: VerbType.Contains, - metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship + metadata: { isVFS: true } // Mark as VFS relationship }) } } else { @@ -519,7 +519,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { from: parentId, to: entity, type: VerbType.Contains, - metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship + metadata: { isVFS: true } // Mark as VFS relationship }) // Update path resolver cache @@ -574,7 +574,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.EISDIR, `Is a directory: ${path}`, path, 'unlink') } - // v5.2.0: Delete blob from BlobStorage (decrements ref count) + // Delete blob from BlobStorage (decrements ref count) if (entity.metadata.storage?.type === 'blob') { await this.blobStorage.delete(entity.metadata.storage.hash) } @@ -617,7 +617,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * v6.2.0: Gather descendants using graph traversal + bulk fetch + * Gather descendants using graph traversal + bulk fetch * * ARCHITECTURE: * 1. Traverse graph to collect entity IDs (in-memory, fast) @@ -694,7 +694,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { /** * Get a properly structured tree for the given path * - * v6.2.0: Graph traversal + ONE batch fetch (55x faster on cloud storage) + * Graph traversal + ONE batch fetch (55x faster on cloud storage) * * Architecture: * 1. Resolve path to entity ID @@ -733,7 +733,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { /** * Get all descendants of a directory (flat list) * - * v6.2.0: Same optimization as getTreeStructure + * Same optimization as getTreeStructure */ async getDescendants(path: string, options?: { includeAncestor?: boolean @@ -876,8 +876,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { name, parent: parentId, vfsType: 'directory', - isVFS: true, // v4.3.3: Mark as VFS entity (internal) - isVFSEntity: true, // v5.3.0: Explicit flag for developer filtering + isVFS: true, // Mark as VFS entity (internal) + isVFSEntity: true, // Explicit flag for developer filtering size: 0, permissions: options?.mode || this.config.permissions?.defaultDirectory || 0o755, owner: 'user', @@ -900,8 +900,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { to: entity, type: VerbType.Contains, metadata: { - isVFS: true, // v4.5.1: Mark as VFS relationship - relationshipType: 'vfs' // v4.9.0: Standardized relationship type metadata + isVFS: true, // Mark as VFS relationship + relationshipType: 'vfs' // Standardized relationship type metadata } }) } @@ -921,7 +921,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { /** * Remove a directory * - * v6.4.0: Optimized for cloud storage using batch operations + * Optimized for cloud storage using batch operations * - Uses gatherDescendants() for efficient graph traversal + batch fetch * - Uses deleteMany() for chunked transactional deletion * - Parallel blob cleanup with chunking @@ -950,7 +950,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.ENOTEMPTY, `Directory not empty: ${path}`, path, 'rmdir') } - // v6.4.0: OPTIMIZED batch deletion for recursive case + // OPTIMIZED batch deletion for recursive case if (options?.recursive && children.length > 0) { // Phase 1: Gather all descendants in ONE batch fetch const descendants = await this.gatherDescendants(entityId, Infinity) @@ -1020,7 +1020,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { children = children.slice(0, options.limit) } - // v6.2.0: REMOVED updateAccessTime() for performance + // REMOVED updateAccessTime() for performance // Directory access time updates caused 50-100ms GCS write on EVERY readdir // await this.updateAccessTime(entityId) // ← REMOVED @@ -1117,7 +1117,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { offset: options?.offset, explain: options?.explain, where: { - vfsType: 'file' // v4.7.0: Search VFS files + vfsType: 'file' // Search VFS files } } @@ -1167,7 +1167,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { threshold: options?.threshold || 0.7, type: [NounType.File, NounType.Document, NounType.Media], where: { - vfsType: 'file' // v4.7.0: Find similar VFS files + vfsType: 'file' // Find similar VFS files } }) @@ -1294,7 +1294,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { return filename.substring(lastDot + 1).toLowerCase() } - // v5.2.0: MIME detection moved to MimeTypeDetector service + // MIME detection moved to MimeTypeDetector service // Removed detectMimeType() and isTextFile() - now using mimeDetector singleton private getFileNounType(mimeType: string): NounType { @@ -1307,7 +1307,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { return NounType.File } - // v5.2.0: Removed compression methods (shouldCompress, compress, decompress) + // Removed compression methods (shouldCompress, compress, decompress) // BlobStorage handles all compression automatically with zstd private async generateEmbedding(buffer: Buffer, mimeType: string): Promise { @@ -1371,7 +1371,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { return metadata } - // v6.2.0: REMOVED updateAccessTime() method entirely + // REMOVED updateAccessTime() method entirely // Access time updates caused 50-100ms GCS write on EVERY file/dir read // Modern file systems use 'noatime' for same reason // Field 'accessed' still exists in metadata for backward compat but won't update @@ -1674,7 +1674,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { from: newParentId, to: entityId, type: VerbType.Contains, - metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship + metadata: { isVFS: true } // Mark as VFS relationship }) } } @@ -1752,7 +1752,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { from: parentId, to: newEntity, type: VerbType.Contains, - metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship + metadata: { isVFS: true } // Mark as VFS relationship }) } @@ -1774,7 +1774,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { /** * Copy a directory recursively * - * v6.4.0: Optimized for cloud storage using batch operations + * Optimized for cloud storage using batch operations * - Uses gatherDescendants() for efficient graph traversal + batch fetch * - Uses addMany() for batch entity creation * - Uses relateMany() for batch relationship creation @@ -1941,7 +1941,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { from: parentId, to: entity, type: VerbType.Contains, - metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship + metadata: { isVFS: true } // Mark as VFS relationship }) // Update path resolver cache @@ -2151,7 +2151,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { from: fromEntityId, to: toEntityId, type: type as any, // Convert string to VerbType - metadata: { isVFS: true } // v4.5.1: Mark as VFS relationship + metadata: { isVFS: true } // Mark as VFS relationship }) // Invalidate caches for both paths @@ -2378,7 +2378,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { /** * Bulk write operations for performance * - * v6.5.0: Prevents race condition by processing mkdir operations + * Prevents race condition by processing mkdir operations * sequentially before parallel batch processing of other operations. */ async bulkWrite(operations: Array<{ diff --git a/src/vfs/importers/DirectoryImporter.ts b/src/vfs/importers/DirectoryImporter.ts index 1d4c6d01..9a087773 100644 --- a/src/vfs/importers/DirectoryImporter.ts +++ b/src/vfs/importers/DirectoryImporter.ts @@ -27,7 +27,7 @@ export interface ImportOptions { showProgress?: boolean // Log progress (default: false) filter?: (path: string) => boolean // Custom filter function - // v4.10.0: Import tracking + // Import tracking importId?: string // Unique import identifier (auto-generated if not provided) projectId?: string // Project identifier grouping related imports customMetadata?: Record // Custom metadata to attach @@ -66,7 +66,7 @@ export class DirectoryImporter { async import(sourcePath: string, options: ImportOptions = {}): Promise { const startTime = Date.now() - // v4.10.0: Generate tracking metadata + // Generate tracking metadata const importId = options.importId || uuidv4() const projectId = options.projectId || this.deriveProjectId(options.targetPath || '/') const trackingMetadata = { @@ -225,7 +225,7 @@ export class DirectoryImporter { try { await this.vfs.mkdir(dirPath, { recursive: true, - metadata: trackingMetadata // v4.10.0: Add tracking metadata + metadata: trackingMetadata // Add tracking metadata }) result.directoriesCreated++ } catch (error: any) { @@ -332,7 +332,7 @@ export class DirectoryImporter { originalPath: filePath, originalSize: stats.size, originalModified: stats.mtime.getTime(), - ...trackingMetadata // v4.10.0: Add tracking metadata + ...trackingMetadata // Add tracking metadata } }) @@ -382,6 +382,6 @@ export class DirectoryImporter { return false } - // v5.2.0: MIME detection moved to MimeTypeDetector service + // MIME detection moved to MimeTypeDetector service // Removed detectMimeType() - now using mimeDetector singleton } \ No newline at end of file diff --git a/src/vfs/index.ts b/src/vfs/index.ts index 0bbb7607..16684214 100644 --- a/src/vfs/index.ts +++ b/src/vfs/index.ts @@ -10,7 +10,7 @@ export { VirtualFileSystem } from './VirtualFileSystem.js' export { PathResolver } from './PathResolver.js' export * from './types.js' -// MIME Type Detection (v5.2.0) +// MIME Type Detection export { MimeTypeDetector, mimeDetector } from './MimeTypeDetector.js' // fs compatibility layer diff --git a/src/vfs/semantic/SemanticPathResolver.ts b/src/vfs/semantic/SemanticPathResolver.ts index cfedf257..db6b2249 100644 --- a/src/vfs/semantic/SemanticPathResolver.ts +++ b/src/vfs/semantic/SemanticPathResolver.ts @@ -317,7 +317,7 @@ export class SemanticPathResolver { } /** - * Invalidate ALL caches (v6.3.0) + * Invalidate ALL caches * Clears both traditional path cache AND semantic cache * Call this when switching branches, clearing data, or forking */ diff --git a/src/vfs/semantic/projections/AuthorProjection.ts b/src/vfs/semantic/projections/AuthorProjection.ts index f77e018d..92781a27 100644 --- a/src/vfs/semantic/projections/AuthorProjection.ts +++ b/src/vfs/semantic/projections/AuthorProjection.ts @@ -52,7 +52,7 @@ export class AuthorProjection extends BaseProjectionStrategy { * Resolve author to entity IDs using REAL Brainy.find() */ async resolve(brain: Brainy, vfs: VirtualFileSystem, authorName: string): Promise { - // v4.7.0: VFS entities are part of the knowledge graph + // VFS entities are part of the knowledge graph const results = await brain.find({ where: { vfsType: 'file', diff --git a/src/vfs/semantic/projections/TagProjection.ts b/src/vfs/semantic/projections/TagProjection.ts index 1ebe18eb..f61b33de 100644 --- a/src/vfs/semantic/projections/TagProjection.ts +++ b/src/vfs/semantic/projections/TagProjection.ts @@ -52,7 +52,7 @@ export class TagProjection extends BaseProjectionStrategy { * Resolve tag to entity IDs using REAL Brainy.find() */ async resolve(brain: Brainy, vfs: VirtualFileSystem, tagName: string): Promise { - // v4.7.0: VFS entities are part of the knowledge graph + // VFS entities are part of the knowledge graph const results = await brain.find({ where: { vfsType: 'file', diff --git a/src/vfs/semantic/projections/TemporalProjection.ts b/src/vfs/semantic/projections/TemporalProjection.ts index 4347e830..41b29b68 100644 --- a/src/vfs/semantic/projections/TemporalProjection.ts +++ b/src/vfs/semantic/projections/TemporalProjection.ts @@ -69,7 +69,7 @@ export class TemporalProjection extends BaseProjectionStrategy { const endOfDay = new Date(date) endOfDay.setHours(23, 59, 59, 999) - // v4.7.0: VFS entities are part of the knowledge graph + // VFS entities are part of the knowledge graph const results = await brain.find({ where: { vfsType: 'file', diff --git a/src/vfs/types.ts b/src/vfs/types.ts index 88803695..54e9902b 100644 --- a/src/vfs/types.ts +++ b/src/vfs/types.ts @@ -33,8 +33,8 @@ export interface VFSMetadata { name: string // Filename or directory name parent?: string // Parent directory entity ID vfsType: 'file' | 'directory' | 'symlink' - isVFS?: boolean // v4.3.3: Mark as VFS entity (internal, separates from knowledge graph) - isVFSEntity?: boolean // v5.3.0: Explicit developer-facing flag for filtering VFS entities + isVFS?: boolean // Mark as VFS entity (internal, separates from knowledge graph) + isVFSEntity?: boolean // Explicit developer-facing flag for filtering VFS entities // File attributes size: number // Size in bytes (0 for directories) @@ -50,7 +50,7 @@ export interface VFSMetadata { accessed: number // Last access timestamp (ms) modified: number // Last modification timestamp (ms) - // Content storage strategy (v5.2.0: unified blob storage) + // Content storage strategy (unified blob storage) storage?: { type: 'blob' // All files now use BlobStorage (was 'inline'|'reference'|'chunked') hash: string // SHA-256 content hash from BlobStorage