diff --git a/docs/api/README.md b/docs/api/README.md index 95da2014..efe1ae5d 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -1,9 +1,9 @@ -# 🧠 Brainy v7.1.0 API Reference +# 🧠 Brainy API Reference -> **Complete API documentation for Brainy v7.1.0** +> **Complete API documentation for Brainy** > Zero Configuration β€’ Triple Intelligence β€’ Git-Style Branching β€’ Entity Versioning β€’ Candle WASM Embeddings -**Updated:** 2026-01-06 for v7.1.0 +**Updated:** 2026-01-06 **All APIs verified against actual code** --- @@ -13,29 +13,29 @@ ```typescript import { Brainy, NounType, VerbType } from '@soulcraft/brainy' -const brain = new Brainy() // Zero config! -await brain.init() // VFS auto-initialized in v5.1.0! +const brain = new Brainy() // Zero config! +await brain.init() // VFS auto-initialized! // Add data (text auto-embeds!) const id = await brain.add({ - data: 'The future of AI is here', - type: NounType.Concept, - metadata: { category: 'technology' } + data: 'The future of AI is here', + type: NounType.Concept, + metadata: { category: 'technology' } }) // Search with Triple Intelligence const results = await brain.find({ - query: 'artificial intelligence', - where: { year: { greaterThan: 2020 } }, - connected: { from: id, depth: 2 } + query: 'artificial intelligence', + where: { year: { greaterThan: 2020 } }, + connected: { from: id, depth: 2 } }) -// Fork for safe experimentation (v5.0.0+) +// Fork for safe experimentation const experiment = await brain.fork('test-feature') await experiment.add({ data: 'test', type: NounType.Document }) await experiment.commit({ message: 'Add test data' }) -// Entity versioning (v5.3.0+) +// Entity versioning await brain.versions.save(id, { tag: 'v1.0', description: 'Initial version' }) await brain.update(id, { category: 'AI' }) await brain.versions.save(id, { tag: 'v2.0' }) @@ -54,10 +54,10 @@ Typed connections between entities - building knowledge graphs. ### 🧠 Triple Intelligence Vector search + Graph traversal + Metadata filtering in one unified query. -### 🌳 Git-Style Branching (v5.0.0+) +### 🌳 Git-Style Branching Fork, experiment, and commit - Snowflake-style copy-on-write isolation. -### πŸ“œ Entity Versioning (v5.3.0+) +### πŸ“œ Entity Versioning Time-travel and history tracking for individual entities - Git-like version control with content-addressable storage. --- @@ -68,15 +68,15 @@ Time-travel and history tracking for individual entities - Git-like version cont - [Search & Query](#search--query) - [Relationships](#relationships) - [Batch Operations](#batch-operations) -- [Branch Management (v5.0+)](#branch-management-v50) -- [Entity Versioning (v5.3.0+)](#entity-versioning-v530) +- [Branch Management](#branch-management) +- [Entity Versioning](#entity-versioning) - [Virtual Filesystem (VFS)](#virtual-filesystem-vfs) - [Neural API](#neural-api) - [Import & Export](#import--export) - [Configuration](#configuration) - [Storage Adapters](#storage-adapters) - [Utility Methods](#utility-methods) -- [Embedding & Analysis APIs (v7.1.0)](#embedding--analysis-apis-v710) +- [Embedding & Analysis APIs](#embedding--analysis-apis) - [Type System Reference](#type-system-reference) --- @@ -89,12 +89,12 @@ Add a single entity to the database. ```typescript const id = await brain.add({ - data: 'JavaScript is a programming language', // Text or pre-computed vector - type: NounType.Concept, // Required: Entity type - metadata: { // Optional metadata - category: 'programming', - year: 1995 - } + data: 'JavaScript is a programming language', // Text or pre-computed vector + type: NounType.Concept, // Required: Entity type + metadata: { // Optional metadata + category: 'programming', + year: 1995 + } }) ``` @@ -113,9 +113,9 @@ Retrieve a single entity by ID. ```typescript const entity = await brain.get(id) -console.log(entity?.data) // Original data -console.log(entity?.metadata) // Metadata -console.log(entity?.vector) // Embedding vector +console.log(entity?.data) // Original data +console.log(entity?.metadata) // Metadata +console.log(entity?.vector) // Embedding vector ``` **Parameters:** @@ -131,9 +131,9 @@ Update an existing entity. ```typescript await brain.update({ - id: entityId, - data: 'Updated content', // Optional: new data - metadata: { updated: true } // Optional: new metadata (merges) + id: entityId, + data: 'Updated content', // Optional: new data + metadata: { updated: true } // Optional: new metadata (merges) }) ``` @@ -173,72 +173,72 @@ const results = await brain.find('machine learning') // Advanced Triple Intelligence query const results = await brain.find({ - query: 'artificial intelligence', // Vector similarity - where: { // Metadata filtering - year: { greaterThan: 2020 }, - category: { oneOf: ['AI', 'ML'] } - }, - connected: { // Graph traversal - to: conceptId, - depth: 2, - type: VerbType.RelatedTo - }, - limit: 10 + query: 'artificial intelligence', // Vector similarity + where: { // Metadata filtering + year: { greaterThan: 2020 }, + category: { oneOf: ['AI', 'ML'] } + }, + connected: { // Graph traversal + to: conceptId, + depth: 2, + type: VerbType.RelatedTo + }, + limit: 10 }) ``` **Parameters:** - `query`: `string | FindParams` - - **Simple:** Just text for vector search - - **Advanced:** Object with vector + graph + metadata filters + - **Simple:** Just text for vector search + - **Advanced:** Object with vector + graph + metadata filters **FindParams:** - `query?`: `string` - Text for vector similarity - `where?`: `object` - Metadata filters (see [Query Operators](#query-operators)) - `connected?`: `object` - Graph traversal options - - `to?`: `string` - Target entity ID - - `from?`: `string` - Source entity ID - - `type?`: `VerbType` - Relationship type - - `depth?`: `number` - Traversal depth + - `to?`: `string` - Target entity ID + - `from?`: `string` - Source entity ID + - `type?`: `VerbType` - Relationship type + - `depth?`: `number` - Traversal depth - `limit?`: `number` - Max results (default: 10) - `offset?`: `number` - Skip results -- `searchMode?`: `'auto' | 'text' | 'semantic' | 'hybrid'` - Search strategy (v7.7.0): - - `'auto'` (default): Zero-config hybrid combining text + semantic search - - `'text'`: Pure keyword/text matching - - `'semantic'`: Pure vector similarity - - `'hybrid'`: Explicit hybrid mode +- `searchMode?`: `'auto' | 'text' | 'semantic' | 'hybrid'` - Search strategy: + - `'auto'` (default): Zero-config hybrid combining text + semantic search + - `'text'`: Pure keyword/text matching + - `'semantic'`: Pure vector similarity + - `'hybrid'`: Explicit hybrid mode - `hybridAlpha?`: `number` - Balance between text (0.0) and semantic (1.0) search. Auto-detected by query length if not specified. **Returns:** `Promise` - Matching entities with scores --- -### Hybrid Search (v7.7.0) +### Hybrid Search Brainy automatically combines text (keyword) and semantic (vector) search for optimal results. No configuration needed. ```typescript // Zero-config hybrid search (just works) const results = await brain.find({ - query: 'David Smith' // Finds both exact text matches AND semantically similar + query: 'David Smith' // Finds both exact text matches AND semantically similar }) // Force text-only search (exact keyword matching) const textResults = await brain.find({ - query: 'exact keyword', - searchMode: 'text' + query: 'exact keyword', + searchMode: 'text' }) // Force semantic-only search (vector similarity) const semanticResults = await brain.find({ - query: 'artificial intelligence concepts', - searchMode: 'semantic' + query: 'artificial intelligence concepts', + searchMode: 'semantic' }) // Custom hybrid weighting (0 = text only, 1 = semantic only) const customResults = await brain.find({ - query: 'David Smith', - hybridAlpha: 0.3 // Favor text matching + query: 'David Smith', + hybridAlpha: 0.3 // Favor text matching }) ``` @@ -249,7 +249,7 @@ const customResults = await brain.find({ --- -### Match Visibility (v7.8.0) +### Match Visibility Search results include detailed match information: @@ -257,10 +257,10 @@ Search results include detailed match information: const results = await brain.find({ query: 'david the warrior' }) // Each result now includes: -results[0].textMatches // ["david", "warrior"] - exact query words found -results[0].textScore // 0.25 - text match quality (0-1) -results[0].semanticScore // 0.87 - semantic similarity (0-1) -results[0].matchSource // 'both' | 'text' | 'semantic' +results[0].textMatches // ["david", "warrior"] - exact query words found +results[0].textScore // 0.25 - text match quality (0-1) +results[0].semanticScore // 0.87 - semantic similarity (0-1) +results[0].matchSource // 'both' | 'text' | 'semantic' ``` **Use cases:** @@ -270,7 +270,7 @@ results[0].matchSource // 'both' | 'text' | 'semantic' --- -### `highlight(params)` β†’ `Promise` ✨ *New v7.8.0, Enhanced v7.9.0* +### `highlight(params)` β†’ `Promise` ✨ Zero-config highlighting for both exact matches AND semantic concepts. Handles plain text, rich-text JSON (TipTap, Slate, Lexical, Draft.js, Quill), HTML, and Markdown automatically. @@ -278,37 +278,37 @@ Handles plain text, rich-text JSON (TipTap, Slate, Lexical, Draft.js, Quill), HT ```typescript // Plain text (works as before) const highlights = await brain.highlight({ - query: "david the warrior", - text: "David Smith is a brave fighter who battles dragons" + query: "david the warrior", + text: "David Smith is a brave fighter who battles dragons" }) // [ -// { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, -// { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, -// { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } +// { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, +// { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, +// { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } // ] -// Rich-text JSON (auto-detected, v7.9.0) +// Rich-text JSON (auto-detected) const highlights = await brain.highlight({ - query: "david the warrior", - text: JSON.stringify(tiptapDocument) // TipTap, Slate, Lexical, Draft.js, Quill + query: "david the warrior", + text: JSON.stringify(tiptapDocument) // TipTap, Slate, Lexical, Draft.js, Quill }) // Extracts text from nodes, annotates with contentCategory: // [ -// { text: "David", score: 1.0, matchType: 'text', contentCategory: 'heading' }, -// { text: "fighter", score: 0.78, matchType: 'semantic', contentCategory: 'prose' } +// { text: "David", score: 1.0, matchType: 'text', contentCategory: 'title' }, +// { text: "fighter", score: 0.78, matchType: 'semantic', contentCategory: 'content' } // ] -// HTML input (auto-detected, v7.9.0) +// HTML input (auto-detected) const highlights = await brain.highlight({ - query: "warrior", - text: "

David the Warrior

A brave fighter.

" + query: "warrior", + text: "

David the Warrior

A brave fighter.

" }) -// Custom extractor for proprietary formats (v7.9.0) +// Custom extractor for proprietary formats const highlights = await brain.highlight({ - query: "function", - text: sourceCode, - contentExtractor: (text) => treeSitterParse(text) // Your custom parser + query: "function", + text: sourceCode, + contentExtractor: (text) => treeSitterParse(text) // Your custom parser }) ``` @@ -317,17 +317,17 @@ const highlights = await brain.highlight({ - `text`: `string` - Text to highlight (plain text, JSON, HTML, or Markdown) - `granularity?`: `'word' | 'phrase' | 'sentence'` - Highlight unit (default: 'word') - `threshold?`: `number` - Min similarity for semantic matches (default: 0.5) -- `contentType?`: `ContentType` - Optional hint: `'plaintext' | 'richtext-json' | 'html' | 'markdown'`. Skips auto-detection when provided. *(v7.9.0)* -- `contentExtractor?`: `(text: string) => ExtractedSegment[]` - Custom parser. Bypasses built-in detection entirely. *(v7.9.0)* +- `contentType?`: `ContentType` - Optional hint: `'plaintext' | 'richtext-json' | 'html' | 'markdown'`. Skips auto-detection when provided. +- `contentExtractor?`: `(text: string) => ExtractedSegment[]` - Custom parser. Bypasses built-in detection entirely. **Returns:** `Promise` - `text` - The matched text - `score` - Match score (1.0 for text matches, varies for semantic) - `position` - [start, end] indices in extracted text - `matchType` - `'text'` (exact) or `'semantic'` (concept) -- `contentCategory?` - `'prose' | 'heading' | 'code' | 'label'` β€” Role of the source text in the document. Present when input is structured. *(v7.9.0)* +- `contentCategory?` - `'title' | 'annotation' | 'content' | 'value' | 'code' | 'structural'` β€” Role of the source text. Built-in extractors produce `'title'`, `'content'`, `'code'`. All 6 categories are available for custom parsers. -**Supported Rich-Text Formats (v7.9.0):** +**Supported Rich-Text Formats:** | Format | Detection | Text nodes | |--------|-----------|------------| @@ -339,17 +339,18 @@ const highlights = await brain.highlight({ | HTML | Tags like `

`, `

`, `` | Visible text content | | Markdown | `#` headings, ` ``` ` code blocks | Stripped markup | -**Timeout Protection (v7.9.0):** +**Timeout Protection:** Semantic matching has a 10-second timeout. If embedding takes too long (e.g., WASM stall), `highlight()` returns text-only matches instead of hanging. **UI Pattern:** ```typescript // Style differently based on match type and content category highlights.forEach(h => { - const style = h.matchType === 'text' ? 'font-weight: bold' : 'background: yellow' - if (h.contentCategory === 'heading') { /* render as heading highlight */ } - if (h.contentCategory === 'code') { /* render with code styling */ } - // Apply style from h.position[0] to h.position[1] + const style = h.matchType === 'text' ? 'font-weight: bold' : 'background: yellow' + if (h.contentCategory === 'title') { /* render as heading highlight */ } + if (h.contentCategory === 'code') { /* render with code styling */ } + if (h.contentCategory === 'annotation') { /* render as comment/caption */ } + // Apply style from h.position[0] to h.position[1] }) ``` @@ -384,13 +385,13 @@ Create a typed relationship between entities. ```typescript const relId = await brain.relate({ - from: sourceId, - to: targetId, - type: VerbType.RelatedTo, - metadata: { // Optional - strength: 0.9, - confidence: 0.85 - } + from: sourceId, + to: targetId, + type: VerbType.RelatedTo, + metadata: { // Optional + strength: 0.9, + confidence: 0.85 + } }) ``` @@ -417,8 +418,8 @@ const incoming = await brain.getRelations({ to: entityId }) // Filter by type const related = await brain.getRelations({ - from: entityId, - type: VerbType.Contains + from: entityId, + type: VerbType.Contains }) ``` @@ -439,14 +440,14 @@ Add multiple entities in one operation. ```typescript const result = await brain.addMany({ - items: [ - { data: 'Entity 1', type: NounType.Document }, - { data: 'Entity 2', type: NounType.Concept } - ] + items: [ + { data: 'Entity 1', type: NounType.Document }, + { data: 'Entity 2', type: NounType.Concept } + ] }) -console.log(result.successful) // Array of IDs -console.log(result.failed) // Array of errors +console.log(result.successful) // Array of IDs +console.log(result.failed) // Array of errors ``` **Returns:** `Promise>` - Success/failure results @@ -459,7 +460,7 @@ Delete multiple entities. ```typescript const result = await brain.deleteMany({ - ids: [id1, id2, id3] + ids: [id1, id2, id3] }) ``` @@ -471,10 +472,10 @@ Update multiple entities. ```typescript const result = await brain.updateMany({ - updates: [ - { id: id1, metadata: { updated: true } }, - { id: id2, data: 'New content' } - ] + updates: [ + { id: id1, metadata: { updated: true } }, + { id: id2, data: 'New content' } + ] }) ``` @@ -486,18 +487,18 @@ Create multiple relationships. ```typescript const ids = await brain.relateMany({ - relations: [ - { from: id1, to: id2, type: VerbType.RelatedTo }, - { from: id1, to: id3, type: VerbType.Contains } - ] + relations: [ + { from: id1, to: id2, type: VerbType.RelatedTo }, + { from: id1, to: id3, type: VerbType.Contains } + ] }) ``` --- -## Branch Management (v5.0+) +## Branch Management -**NEW in v5.0.0:** Git-style branching with Snowflake-style copy-on-write. +Git-style branching with Snowflake-style copy-on-write. ### `fork(branch?, options?)` β†’ `Promise` @@ -512,13 +513,13 @@ await experiment.add({ data: 'Test entity', type: NounType.Document }) await experiment.update({ id: someId, metadata: { modified: true } }) // Parent is unaffected! -const parentData = await brain.find({}) // Original data unchanged +const parentData = await brain.find({}) // Original data unchanged ``` **Parameters:** - `branch?`: `string` - Branch name (auto-generated if omitted) - `options?`: `object` - - `description?`: `string` - Branch description + - `description?`: `string` - Branch description **Returns:** `Promise` - New Brainy instance on forked branch @@ -568,9 +569,9 @@ Create a commit snapshot. ```typescript const commitId = await brain.commit({ - message: 'Add new features', - author: 'dev@example.com', - metadata: { ticket: 'PROJ-123' } + message: 'Add new features', + author: 'dev@example.com', + metadata: { ticket: 'PROJ-123' } }) ``` @@ -600,8 +601,8 @@ Get commit history. ```typescript const history = await brain.getHistory({ - branch: 'main', - limit: 10 + branch: 'main', + limit: 10 }) ``` @@ -618,13 +619,13 @@ const commitId = commits[0].id // Create snapshot (lazy-loading, no eager data loading) const snapshot = await brain.asOf(commitId, { - cacheSize: 10000 // LRU cache size (default: 10000) + cacheSize: 10000 // LRU cache size (default: 10000) }) // Query historical state - full Triple Intelligence works! const results = await snapshot.find({ - query: 'AI research', - where: { category: 'technology' } + query: 'AI research', + where: { category: 'technology' } }) // Get historical relationships @@ -637,7 +638,7 @@ await snapshot.close() **Parameters:** - `commitId`: `string` - Commit hash to snapshot from - `options?`: `object` - - `cacheSize?`: `number` - LRU cache size for lazy-loading (default: 10000) + - `cacheSize?`: `number` - LRU cache size for lazy-loading (default: 10000) **Returns:** `Promise` - Read-only Brainy instance with historical state @@ -651,9 +652,9 @@ await snapshot.close() --- -## Entity Versioning (v5.3.0+) +## Entity Versioning -**NEW in v5.3.0:** Git-style versioning for individual entities with content-addressable storage. +Git-style versioning for individual entities with content-addressable storage. ### Overview @@ -664,7 +665,7 @@ Entity Versioning provides time-travel and history tracking for individual entit - **Branch-Isolated** - Versions isolated per branch - **Selective Auto-Versioning** - Optional augmentation for automatic version creation - **Production-Scale** - Designed for billions of entities -- **VFS File Support (v6.3.2+)** - Full versioning for VFS files with actual blob content +- **VFS File Support** - Full versioning for VFS files with actual blob content --- @@ -675,22 +676,22 @@ Save a new version of an entity. ```typescript // Save version with tag const version = await brain.versions.save('user-123', { - tag: 'v1.0', - description: 'Initial user profile', - metadata: { author: 'dev@example.com' } + tag: 'v1.0', + description: 'Initial user profile', + metadata: { author: 'dev@example.com' } }) -console.log(version.version) // 1 -console.log(version.contentHash) // SHA-256 hash -console.log(version.createdAt) // Timestamp +console.log(version.version) // 1 +console.log(version.contentHash) // SHA-256 hash +console.log(version.createdAt) // Timestamp ``` **Parameters:** - `entityId`: `string` - Entity ID to version - `options?`: `object` - - `tag?`: `string` - Version tag (e.g., 'v1.0', 'beta') - - `description?`: `string` - Version description - - `metadata?`: `object` - Additional version metadata + - `tag?`: `string` - Version tag (e.g., 'v1.0', 'beta') + - `description?`: `string` - Version description + - `metadata?`: `object` - Additional version metadata **Returns:** `Promise` - Created version @@ -707,20 +708,20 @@ List all versions of an entity. ```typescript const versions = await brain.versions.list('user-123', { - limit: 10, - offset: 0 + limit: 10, + offset: 0 }) versions.forEach(v => { - console.log(`Version ${v.version}: ${v.tag} - ${v.description}`) + console.log(`Version ${v.version}: ${v.tag} - ${v.description}`) }) ``` **Parameters:** - `entityId`: `string` - Entity ID - `options?`: `object` - - `limit?`: `number` - Max versions to return - - `offset?`: `number` - Skip versions + - `limit?`: `number` - Max versions to return + - `offset?`: `number` - Skip versions **Returns:** `Promise` - Versions (newest first) @@ -751,10 +752,10 @@ Compare two versions. ```typescript const diff = await brain.versions.compare('user-123', 1, 2) -console.log(diff.totalChanges) // Total changes -console.log(diff.modified) // Modified fields -console.log(diff.added) // Added fields -console.log(diff.removed) // Removed fields +console.log(diff.totalChanges) // Total changes +console.log(diff.modified) // Modified fields +console.log(diff.added) // Added fields +console.log(diff.removed) // Removed fields // Check specific changes const nameChange = diff.modified.find(c => c.path === 'metadata.name') @@ -772,11 +773,11 @@ Get version content without restoring. ```typescript // View old version without changing current state const v1Content = await brain.versions.getContent('user-123', 1) -console.log(v1Content.metadata.name) // Old name +console.log(v1Content.metadata.name) // Old name // Current state unchanged const current = await brain.get('user-123') -console.log(current.metadata.name) // Current name +console.log(current.metadata.name) // Current name ``` --- @@ -803,9 +804,9 @@ Clean up old versions. ```typescript const result = await brain.versions.prune('user-123', { - keepRecent: 10, // Keep 10 most recent - keepTagged: true, // Always keep tagged versions - olderThan: Date.now() - 30 * 24 * 60 * 60 * 1000 // Older than 30 days + keepRecent: 10, // Keep 10 most recent + keepTagged: true, // Always keep tagged versions + olderThan: Date.now() - 30 * 24 * 60 * 60 * 1000 // Older than 30 days }) console.log(`Deleted ${result.deleted}, kept ${result.kept}`) @@ -825,7 +826,7 @@ Get latest version. ```typescript const latest = await brain.versions.getLatest('user-123') if (latest) { - console.log(`Latest: v${latest.version} (${latest.tag})`) + console.log(`Latest: v${latest.version} (${latest.tag})`) } ``` @@ -858,7 +859,7 @@ Check if entity has versions. ```typescript if (await brain.versions.hasVersions('user-123')) { - console.log('Entity has version history') + console.log('Entity has version history') } ``` @@ -873,14 +874,14 @@ import { VersioningAugmentation } from '@soulcraft/brainy' // Configure auto-versioning const versioning = new VersioningAugmentation({ - enabled: true, - onUpdate: true, // Version on update() - onDelete: false, // Don't version on delete - entities: ['user-*'], // Only version users - excludeEntities: ['temp-*'], - excludeTypes: ['temporary'], - keepRecent: 50, // Auto-prune old versions - keepTagged: true + enabled: true, + onUpdate: true, // Version on update() + onDelete: false, // Don't version on delete + entities: ['user-*'], // Only version users + excludeEntities: ['temp-*'], + excludeTypes: ['temporary'], + keepRecent: 50, // Auto-prune old versions + keepTagged: true }) // Apply augmentation @@ -930,7 +931,7 @@ await feature.versions.save('doc-1', { tag: 'feature-v1' }) const mainVersions = await brain.versions.list('doc-1') const featureVersions = await feature.versions.list('doc-1') -console.log(mainVersions.length !== featureVersions.length) // true +console.log(mainVersions.length !== featureVersions.length) // true ``` --- @@ -949,8 +950,8 @@ console.log(mainVersions.length !== featureVersions.length) // true **Storage Structure:** ``` -_version:{entityId}:{versionNum}:{branch} // Version metadata -_version_blob:{contentHash} // Content blob (deduplicated) +_version:{entityId}:{versionNum}:{branch} // Version metadata +_version_blob:{contentHash} // Content blob (deduplicated) ``` **Performance:** @@ -968,10 +969,10 @@ _version_blob:{contentHash} // Content blob (deduplicated) ```typescript // Create entity await brain.add({ - data: 'User profile', - id: 'user-123', - type: 'user', - metadata: { name: 'Alice', email: 'alice@example.com' } + data: 'User profile', + id: 'user-123', + type: 'user', + metadata: { name: 'Alice', email: 'alice@example.com' } }) // Save v1 @@ -1012,10 +1013,10 @@ await brain.versions.restore('app-config', 'beta') ```typescript // Track all changes const versioning = new VersioningAugmentation({ - enabled: true, - onUpdate: true, - entities: ['audit-*'], - keepRecent: 100 // Keep 100 versions for audit + enabled: true, + onUpdate: true, + entities: ['audit-*'], + keepRecent: 100 // Keep 100 versions for audit }) brain.augment(versioning) @@ -1027,11 +1028,11 @@ await brain.update('audit-record-1', { status: 'approved' }) // View complete history const versions = await brain.versions.list('audit-record-1') versions.forEach(v => { - console.log(`${v.createdAt}: ${v.description}`) + console.log(`${v.createdAt}: ${v.description}`) }) ``` -#### VFS File Versioning (v6.3.2+) +#### VFS File Versioning ```typescript // VFS files can be versioned with actual blob content @@ -1049,17 +1050,17 @@ await brain.vfs.writeFile('/docs/readme.md', 'Version 2 - updated content') // Save version 2 await brain.versions.save(stat.entityId, { tag: 'v2', description: 'Updated docs' }) -// Compare versions - content is DIFFERENT (fixed in v6.3.2) +// Compare versions - content is DIFFERENT const v1 = await brain.versions.getContent(stat.entityId, 1) const v2 = await brain.versions.getContent(stat.entityId, 2) -console.log(v1.data !== v2.data) // true +console.log(v1.data !== v2.data) // true // Restore to v1 - writes content back to blob storage await brain.versions.restore(stat.entityId, 'v1') // File is now back to v1 const content = await brain.vfs.readFile('/docs/readme.md') -console.log(content.toString()) // 'Version 1 content' +console.log(content.toString()) // 'Version 1 content' ``` --- @@ -1070,33 +1071,33 @@ console.log(content.toString()) // 'Version 1 content' ## Virtual Filesystem (VFS) -**Auto-initialized in v5.1.0!** Access via `brain.vfs` (property, not method). +Access via `brain.vfs` (property, not method). Auto-initialized during `brain.init()`. ### Filtering VFS Entities -**NEW in v5.3.0:** All VFS entities (files/folders) have `metadata.isVFSEntity: true` set automatically. +All VFS entities (files/folders) have `metadata.isVFSEntity: true` set automatically. Use this to filter VFS entities from semantic search results: ```typescript // Exclude VFS entities from semantic search const semanticOnly = await brain.find({ - query: 'artificial intelligence', - where: { - isVFSEntity: { notEquals: true } // Only semantic entities - } + query: 'artificial intelligence', + where: { + isVFSEntity: { notEquals: true } // Only semantic entities + } }) // Or filter to ONLY VFS entities const vfsOnly = await brain.find({ - where: { - isVFSEntity: { equals: true } // Only VFS files/folders - } + where: { + isVFSEntity: { equals: true } // Only VFS files/folders + } }) // Check if an entity is a VFS entity if (entity.metadata.isVFSEntity === true) { - console.log('This is a VFS file or folder') + console.log('This is a VFS file or folder') } ``` @@ -1123,7 +1124,7 @@ Write file content. ```typescript await brain.vfs.writeFile('/docs/README.md', 'New content', { - encoding: 'utf-8' + encoding: 'utf-8' }) ``` @@ -1161,7 +1162,7 @@ const files = await brain.vfs.readdir('/projects') // With file types const entries = await brain.vfs.readdir('/projects', { withFileTypes: true }) entries.forEach(entry => { - console.log(entry.name, entry.isDirectory() ? 'DIR' : 'FILE') + console.log(entry.name, entry.isDirectory() ? 'DIR' : 'FILE') }) ``` @@ -1183,9 +1184,9 @@ Get file/directory stats. ```typescript const stats = await brain.vfs.stat('/docs/README.md') -console.log(stats.size) // File size -console.log(stats.mtime) // Modified time -console.log(stats.isDirectory()) // Is directory? +console.log(stats.size) // File size +console.log(stats.mtime) // Modified time +console.log(stats.isDirectory()) // Is directory? ``` --- @@ -1198,8 +1199,8 @@ Semantic file search. ```typescript const results = await brain.vfs.search('React components with hooks', { - path: '/src', - limit: 10 + path: '/src', + limit: 10 }) ``` @@ -1211,8 +1212,8 @@ Find similar files. ```typescript const similar = await brain.vfs.findSimilar('/src/App.tsx', { - limit: 5, - threshold: 0.7 + limit: 5, + threshold: 0.7 }) ``` @@ -1226,7 +1227,7 @@ Get directory tree (prevents infinite recursion). ```typescript const tree = await brain.vfs.getTreeStructure('/projects', { - maxDepth: 3 + maxDepth: 3 }) ``` @@ -1238,7 +1239,7 @@ Get all descendants with optional filtering. ```typescript const files = await brain.vfs.getDescendants('/src', { - filter: (entity) => entity.name.endsWith('.tsx') + filter: (entity) => entity.name.endsWith('.tsx') }) ``` @@ -1252,8 +1253,8 @@ Get file metadata. ```typescript const meta = await brain.vfs.getMetadata('/src/App.tsx') -console.log(meta.todos) // Extracted TODOs -console.log(meta.tags) // Tags +console.log(meta.todos) // Extracted TODOs +console.log(meta.tags) // Tags ``` --- @@ -1299,7 +1300,7 @@ Get project statistics. const stats = await brain.vfs.getProjectStats('/projects/my-app') console.log(stats.fileCount) console.log(stats.totalSize) -console.log(stats.fileTypes) // Breakdown by extension +console.log(stats.fileTypes) // Breakdown by extension ``` --- @@ -1310,8 +1311,8 @@ Search for VFS entities by metadata. ```typescript const tsxFiles = await brain.vfs.searchEntities({ - type: 'file', - extension: '.tsx' + type: 'file', + extension: '.tsx' }) ``` @@ -1332,13 +1333,13 @@ Calculate semantic similarity. ```typescript // Simple similarity score const score = await brain.neural().similar( - 'renewable energy', - 'sustainable power' -) // 0.87 + 'renewable energy', + 'sustainable power' +) // 0.87 // Detailed result const result = await brain.neural().similar('text1', 'text2', { - detailed: true + detailed: true }) console.log(result.score) console.log(result.explanation) @@ -1352,15 +1353,15 @@ Automatic clustering. ```typescript const clusters = await brain.neural().clusters({ - algorithm: 'kmeans', - k: 5, - minSize: 3 + algorithm: 'kmeans', + k: 5, + minSize: 3 }) clusters.forEach(cluster => { - console.log(cluster.label) - console.log(cluster.items) - console.log(cluster.centroid) + console.log(cluster.label) + console.log(cluster.items) + console.log(cluster.centroid) }) ``` @@ -1372,8 +1373,8 @@ Find k-nearest neighbors. ```typescript const neighbors = await brain.neural().neighbors(entityId, { - k: 10, - threshold: 0.7 + k: 10, + threshold: 0.7 }) ``` @@ -1396,10 +1397,10 @@ Generate visualization data. ```typescript const vizData = await brain.neural().visualize({ - maxNodes: 100, - dimensions: 3, - algorithm: 'force', - includeEdges: true + maxNodes: 100, + dimensions: 3, + algorithm: 'force', + includeEdges: true }) // Use with D3.js, Cytoscape, GraphML tools ``` @@ -1414,8 +1415,8 @@ Fast clustering for large datasets. ```typescript const clusters = await brain.neural().clusterFast({ - k: 10, - maxIterations: 50 + k: 10, + maxIterations: 50 }) ``` @@ -1427,8 +1428,8 @@ Streaming clustering for very large datasets. ```typescript const clusters = await brain.neural().clusterLarge({ - k: 20, - batchSize: 1000 + k: 20, + batchSize: 1000 }) ``` @@ -1443,20 +1444,20 @@ Smart import with auto-detection (CSV, Excel, PDF, JSON, URLs). ```typescript // CSV import await brain.import('data.csv', { - format: 'csv', - createEntities: true + format: 'csv', + createEntities: true }) // Excel import await brain.import('sales.xlsx', { - format: 'excel', - sheets: ['Q1', 'Q2'] + format: 'excel', + sheets: ['Q1', 'Q2'] }) // PDF import await brain.import('research.pdf', { - format: 'pdf', - extractTables: true + format: 'pdf', + extractTables: true }) // URL import @@ -1466,14 +1467,14 @@ await brain.import('https://api.example.com/data.json') **Parameters:** - `source`: `string | Buffer | object` - File path, URL, buffer, or object - `options?`: Import configuration - - `format?`: `'csv' | 'excel' | 'pdf' | 'json'` - Auto-detected if omitted - - `createEntities?`: `boolean` - Create entities from rows - - `sheets?`: `string[]` - Excel sheets to import - - `extractTables?`: `boolean` - Extract tables from PDF + - `format?`: `'csv' | 'excel' | 'pdf' | 'json'` - Auto-detected if omitted + - `createEntities?`: `boolean` - Create entities from rows + - `sheets?`: `string[]` - Excel sheets to import + - `extractTables?`: `boolean` - Extract tables from PDF **Returns:** `Promise` - Import statistics -**Note:** Import always uses the current branch (v5.1.0 verified). +**Note:** Import always uses the current branch. **[πŸ“– Complete Import Guide β†’](../guides/import-anything.md)** @@ -1501,53 +1502,53 @@ const entities = await snapshot.find({ limit: 100 }) ```typescript const brain = new Brainy({ - // Storage configuration - storage: { - type: 'memory', // memory | opfs | filesystem | s3 | r2 | gcs | azure - path: './brainy-data', // For filesystem storage - compression: true, // Enable gzip compression (60-80% savings) + // Storage configuration + storage: { + type: 'memory', // memory | opfs | filesystem | s3 | r2 | gcs | azure + path: './brainy-data', // For filesystem storage + compression: true, // Enable gzip compression (60-80% savings) - // Cloud storage configs (see Storage Adapters section) - s3Storage: { ... }, - r2Storage: { ... }, - gcsStorage: { ... }, - azureStorage: { ... } - }, + // Cloud storage configs (see Storage Adapters section) + s3Storage: { ... }, + r2Storage: { ... }, + gcsStorage: { ... }, + azureStorage: { ... } + }, - // HNSW vector index config - hnsw: { - M: 16, // Connections per layer - efConstruction: 200, // Construction quality - efSearch: 100, // Search quality - typeAware: true // Enable type-aware indexing (v4.0+) - }, + // HNSW vector index config + hnsw: { + M: 16, // Connections per layer + efConstruction: 200, // Construction quality + efSearch: 100, // Search quality + typeAware: true // Enable type-aware indexing + }, - // Model configuration (embedded in WASM - zero config needed) - // Model: all-MiniLM-L6-v2 (384 dimensions) - // Device: CPU via WASM (works everywhere) + // Model configuration (embedded in WASM - zero config needed) + // Model: all-MiniLM-L6-v2 (384 dimensions) + // Device: CPU via WASM (works everywhere) - // Cache configuration - cache: { - enabled: true, - maxSize: 10000, - ttl: 3600000 // 1 hour in ms - } + // Cache configuration + cache: { + enabled: true, + maxSize: 10000, + ttl: 3600000 // 1 hour in ms + } }) -await brain.init() // Required! VFS auto-initialized in v5.1.0 +await brain.init() // Required! VFS auto-initialized ``` --- ## Storage Adapters -All 7 storage adapters support **copy-on-write branching** (v5.0+). +All 7 storage adapters support **copy-on-write branching**. ### Memory (Default) ```typescript const brain = new Brainy({ - storage: { type: 'memory' } + storage: { type: 'memory' } }) ``` @@ -1559,7 +1560,7 @@ const brain = new Brainy({ ```typescript const brain = new Brainy({ - storage: { type: 'opfs' } + storage: { type: 'opfs' } }) ``` @@ -1571,11 +1572,11 @@ const brain = new Brainy({ ```typescript const brain = new Brainy({ - storage: { - type: 'filesystem', - path: './brainy-data', - compression: true // 60-80% space savings - } + storage: { + type: 'filesystem', + path: './brainy-data', + compression: true // 60-80% space savings + } }) ``` @@ -1587,15 +1588,15 @@ const brain = new Brainy({ ```typescript const brain = new Brainy({ - storage: { - type: 's3', - s3Storage: { - bucketName: 'my-brainy-data', - region: 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY - } - } + storage: { + type: 's3', + s3Storage: { + bucketName: 'my-brainy-data', + region: 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + } + } }) // Enable Intelligent-Tiering for 96% cost savings @@ -1612,15 +1613,15 @@ await brain.storage.enableIntelligentTiering('entities/', 'auto-tier') ```typescript const brain = new Brainy({ - storage: { - type: 'r2', - r2Storage: { - accountId: process.env.CF_ACCOUNT_ID, - bucketName: 'my-brainy-data', - accessKeyId: process.env.CF_ACCESS_KEY_ID, - secretAccessKey: process.env.CF_SECRET_ACCESS_KEY - } - } + storage: { + type: 'r2', + r2Storage: { + accountId: process.env.CF_ACCOUNT_ID, + bucketName: 'my-brainy-data', + accessKeyId: process.env.CF_ACCESS_KEY_ID, + secretAccessKey: process.env.CF_SECRET_ACCESS_KEY + } + } }) ``` @@ -1634,19 +1635,19 @@ const brain = new Brainy({ ```typescript const brain = new Brainy({ - storage: { - type: 'gcs', - gcsStorage: { - bucketName: 'my-brainy-data', - projectId: process.env.GCP_PROJECT_ID, - keyFilename: './gcp-key.json' - } - } + storage: { + type: 'gcs', + gcsStorage: { + bucketName: 'my-brainy-data', + projectId: process.env.GCP_PROJECT_ID, + keyFilename: './gcp-key.json' + } + } }) // Enable auto-tiering await brain.storage.enableAutoclass({ - terminalStorageClass: 'ARCHIVE' + terminalStorageClass: 'ARCHIVE' }) ``` @@ -1660,14 +1661,14 @@ await brain.storage.enableAutoclass({ ```typescript const brain = new Brainy({ - storage: { - type: 'azure', - azureStorage: { - accountName: process.env.AZURE_STORAGE_ACCOUNT, - accountKey: process.env.AZURE_STORAGE_KEY, - containerName: 'brainy-data' - } - } + storage: { + type: 'azure', + azureStorage: { + accountName: process.env.AZURE_STORAGE_ACCOUNT, + accountKey: process.env.AZURE_STORAGE_KEY, + containerName: 'brainy-data' + } + } }) ``` @@ -1709,7 +1710,7 @@ const count = await brain.getVerbCount() --- -### `embed(data)` β†’ `Promise` ✨ *Enhanced v7.1.0* +### `embed(data)` β†’ `Promise` ✨ Generate embedding vector from text or data. @@ -1721,32 +1722,32 @@ console.log(vector.length) // 384 --- -### `embedBatch(texts)` β†’ `Promise` ✨ *New v7.1.0, Optimized v7.9.0* +### `embedBatch(texts)` β†’ `Promise` ✨ Batch embed multiple texts using native WASM batch API (single forward pass). ```typescript const embeddings = await brain.embedBatch([ - 'Machine learning is fascinating', - 'Deep neural networks', - 'Natural language processing' + 'Machine learning is fascinating', + 'Deep neural networks', + 'Natural language processing' ]) -console.log(embeddings.length) // 3 +console.log(embeddings.length) // 3 console.log(embeddings[0].length) // 384 ``` -> **v7.9.0**: Now uses the WASM engine's native `embed_batch()` for a single model forward pass instead of N individual calls. This is the same batch API used internally by `highlight()`. +> Uses the WASM engine's native `embed_batch()` for a single model forward pass instead of N individual calls. This is the same batch API used internally by `highlight()`. --- -### `similarity(textA, textB)` β†’ `Promise` ✨ *New v7.1.0* +### `similarity(textA, textB)` β†’ `Promise` ✨ Calculate semantic similarity between two texts. ```typescript const score = await brain.similarity( - 'The cat sat on the mat', - 'A feline was resting on the rug' + 'The cat sat on the mat', + 'A feline was resting on the rug' ) console.log(score) // ~0.85 (high semantic similarity) ``` @@ -1755,7 +1756,7 @@ console.log(score) // ~0.85 (high semantic similarity) --- -### `neighbors(entityId, options?)` β†’ `Promise` ✨ *New v7.1.0* +### `neighbors(entityId, options?)` β†’ `Promise` ✨ Get graph neighbors of an entity. @@ -1765,14 +1766,14 @@ const neighbors = await brain.neighbors(entityId) // Get outgoing connections only const outgoing = await brain.neighbors(entityId, { - direction: 'outgoing', - limit: 10 + direction: 'outgoing', + limit: 10 }) // Multi-hop traversal const extended = await brain.neighbors(entityId, { - depth: 2, - direction: 'both' + depth: 2, + direction: 'both' }) ``` @@ -1784,7 +1785,7 @@ const extended = await brain.neighbors(entityId, { --- -### `findDuplicates(options?)` β†’ `Promise` ✨ *New v7.1.0* +### `findDuplicates(options?)` β†’ `Promise` ✨ Find semantic duplicates in the database. @@ -1793,17 +1794,17 @@ Find semantic duplicates in the database. const duplicates = await brain.findDuplicates() for (const group of duplicates) { - console.log('Original:', group.entity.id) - for (const dup of group.duplicates) { - console.log(` Duplicate: ${dup.entity.id} (${dup.similarity.toFixed(2)})`) - } + console.log('Original:', group.entity.id) + for (const dup of group.duplicates) { + console.log(` Duplicate: ${dup.entity.id} (${dup.similarity.toFixed(2)})`) + } } // Find person duplicates with higher threshold const personDupes = await brain.findDuplicates({ - type: NounType.PERSON, - threshold: 0.9, - limit: 50 + type: NounType.PERSON, + threshold: 0.9, + limit: 50 }) ``` @@ -1814,7 +1815,7 @@ const personDupes = await brain.findDuplicates({ --- -### `indexStats()` β†’ `Promise` ✨ *New v7.1.0* +### `indexStats()` β†’ `Promise` ✨ Get comprehensive index statistics. @@ -1839,7 +1840,7 @@ console.log(`Fields: ${stats.metadataFields.join(', ')}`) --- -### `cluster(options?)` β†’ `Promise` ✨ *New v7.1.0* +### `cluster(options?)` β†’ `Promise` ✨ Cluster entities by semantic similarity. @@ -1848,15 +1849,15 @@ Cluster entities by semantic similarity. const clusters = await brain.cluster() for (const cluster of clusters) { - console.log(`${cluster.clusterId}: ${cluster.entities.length} entities`) + console.log(`${cluster.clusterId}: ${cluster.entities.length} entities`) } // Find document clusters with centroids const docClusters = await brain.cluster({ - type: NounType.Document, - threshold: 0.85, - minClusterSize: 3, - includeCentroid: true + type: NounType.Document, + threshold: 0.85, + minClusterSize: 3, + includeCentroid: true }) ``` @@ -1893,17 +1894,17 @@ console.log(stats.cacheHitRate) ```typescript const brain = new Brainy(config) -await brain.init() // Required! VFS auto-initialized here (v5.1.0) +await brain.init() // Required! VFS auto-initialized here ``` -**v5.1.0 Change:** VFS is now auto-initialized during `brain.init()` - no separate `vfs.init()` needed! +VFS is auto-initialized during `brain.init()` - no separate `vfs.init()` needed! --- ### Shutdown ```typescript -await brain.shutdown() // Graceful shutdown, flush caches +await brain.shutdown() // Graceful shutdown, flush caches ``` --- @@ -1915,9 +1916,9 @@ await brain.shutdown() // Graceful shutdown, flush caches ```typescript // Create const id = await brain.add({ - data: 'Quantum computing breakthrough', - type: NounType.Concept, - metadata: { category: 'tech', year: 2024 } + data: 'Quantum computing breakthrough', + type: NounType.Concept, + metadata: { category: 'tech', year: 2024 } }) // Read @@ -1925,8 +1926,8 @@ const entity = await brain.get(id) // Update await brain.update({ - id, - metadata: { updated: true } + id, + metadata: { updated: true } }) // Delete @@ -1940,25 +1941,25 @@ await brain.delete(id) ```typescript // Create entities const ai = await brain.add({ - data: 'Artificial Intelligence', - type: NounType.Concept + data: 'Artificial Intelligence', + type: NounType.Concept }) const ml = await brain.add({ - data: 'Machine Learning', - type: NounType.Concept + data: 'Machine Learning', + type: NounType.Concept }) // Create relationship await brain.relate({ - from: ml, - to: ai, - type: VerbType.IsA + from: ml, + to: ai, + type: VerbType.IsA }) // Traverse graph const results = await brain.find({ - connected: { from: ai, depth: 2 } + connected: { from: ai, depth: 2 } }) ``` @@ -1968,23 +1969,23 @@ const results = await brain.find({ ```typescript const results = await brain.find({ - query: 'modern frontend frameworks', // πŸ” Vector - where: { // πŸ“Š Document - year: { greaterThan: 2020 }, - category: { oneOf: ['framework', 'library'] } - }, - connected: { // πŸ•ΈοΈ Graph - to: reactId, - depth: 2, - type: VerbType.BuiltOn - }, - limit: 10 + query: 'modern frontend frameworks', // πŸ” Vector + where: { // πŸ“Š Document + year: { greaterThan: 2020 }, + category: { oneOf: ['framework', 'library'] } + }, + connected: { // πŸ•ΈοΈ Graph + to: reactId, + depth: 2, + type: VerbType.BuiltOn + }, + limit: 10 }) ``` --- -### Git-Style Workflow (v5.0+) +### Git-Style Workflow ```typescript // Fork for experimentation @@ -1992,14 +1993,14 @@ const experiment = await brain.fork('test-migration') // Make changes in isolation await experiment.add({ - data: 'New feature', - type: NounType.Document + data: 'New feature', + type: NounType.Document }) // Commit your work await experiment.commit({ - message: 'Add new feature', - author: 'dev@example.com' + message: 'Add new feature', + author: 'dev@example.com' }) // Switch to experimental branch to make it active @@ -2020,12 +2021,12 @@ const content = await brain.vfs.readFile('/docs/README.md') // Semantic search const reactFiles = await brain.vfs.search('React components with hooks', { - path: '/src' + path: '/src' }) // Get tree structure (safe, prevents infinite recursion) const tree = await brain.vfs.getTreeStructure('/projects', { - maxDepth: 3 + maxDepth: 3 }) ``` @@ -2033,7 +2034,7 @@ const tree = await brain.vfs.getTreeStructure('/projects', { ## Type System Reference -**NEW in v5.5.0:** Stage 3 CANONICAL taxonomy with 169 types (42 nouns + 127 verbs) +Stage 3 CANONICAL taxonomy with 169 types (42 nouns + 127 verbs) ### Noun Types (42) @@ -2107,8 +2108,7 @@ For the full taxonomy with all 169 types and their descriptions, see: - **[Stage 3 CANONICAL Taxonomy](../STAGE3-CANONICAL-TAXONOMY.md)** - Complete list with categories - **[Noun-Verb Taxonomy Architecture](../architecture/noun-verb-taxonomy.md)** - Design rationale -### Migration from v5.4.0 - +### Migration from **Breaking Changes:** - `NounType.Content` removed β†’ Use `Document`, `Message`, or `InformationContent` - `NounType.User` removed β†’ Use `Person` or `Agent` @@ -2120,34 +2120,24 @@ For the full taxonomy with all 169 types and their descriptions, see: --- -## What's New in v5.0 - -### v5.3.0 (Latest) +## Key Features - βœ… **Entity Versioning** - Git-style versioning for individual entities - βœ… **Content-Addressable Storage** - SHA-256 deduplication for versions - βœ… **Auto-Versioning Augmentation** - Automatic version creation on updates - βœ… **Branch-Isolated Versions** - Versions isolated per branch - βœ… **VFS Entity Filtering** - All VFS entities now have `isVFSEntity: true` flag -- βœ… **CRITICAL FIX:** commit() now updates branch refs correctly (brainy.ts:2385) -- βœ… **CRITICAL FIX:** VFS entities now properly flagged for filtering - -### v5.1.0 - - βœ… **VFS Auto-Initialization** - No more separate `vfs.init()` calls - βœ… **VFS Property Access** - Use `brain.vfs.method()` instead of `brain.vfs().method()` - βœ… **Complete COW Support** - All 20 TypeAware methods use COW helpers - βœ… **Verified Import/Export** - Work correctly with current branch - -### v5.0.0 - - βœ… **Instant Fork** - Snowflake-style copy-on-write (<100ms fork time) - βœ… **Git-Style Branching** - fork, commit, checkout, listBranches - βœ… **Full Branch Isolation** - Parent and fork fully isolated - βœ… **Read-Through Inheritance** - Forks see parent + own data - βœ… **Universal Storage Support** - All 7 adapters support branching -**[πŸ“– Complete v5.0 Changes β†’](../../.strategy/v5.1.0-CHANGES.md)** +**[πŸ“– Complete Changes β†’](../../.strategy/v5.1.0-CHANGES.md)** --- @@ -2175,5 +2165,5 @@ For the full taxonomy with all 169 types and their descriptions, see: --- -*Brainy v5.0+ - The Knowledge Operating System* +*Brainy - The Knowledge Operating System* *From prototype to planet-scale β€’ Zero configuration β€’ Triple Intelligenceβ„’ β€’ Git-Style Branching* diff --git a/src/brainy.ts b/src/brainy.ts index e7372ac0..a17f5897 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -88,7 +88,7 @@ import { BrainyInterface } from './types/brainyInterface.js' import type { IntegrationHub } from './integrations/core/IntegrationHub.js' /** - * Stopwords for semantic highlighting (v7.8.0) + * Stopwords for semantic highlighting * These common words are skipped when highlighting individual words * to focus on meaningful content words. */ @@ -149,20 +149,20 @@ export class Brainy implements BrainyInterface { private _tripleIntelligence?: TripleIntelligenceSystem private _versions?: VersioningAPI private _vfs?: VirtualFileSystem - private _vfsInitialized = false // v7.3.0: Track VFS init completion separately - private _hub?: IntegrationHub // v7.4.0: Integration Hub for external tools + private _vfsInitialized = false // Track VFS init completion separately + private _hub?: IntegrationHub // Integration Hub for external tools // State private initialized = false private dimensions?: number - // Ready Promise state (v7.3.0 - Unified readiness API) + // Ready Promise state (Unified readiness API) // Allows consumers to await brain.ready for initialization completion private _readyPromise: Promise | null = null private _readyResolve: (() => void) | null = null private _readyReject: ((error: Error) => void) | null = null - // Lazy rebuild state (v5.7.7 - Production-scale lazy loading) + // Lazy rebuild state (Production-scale lazy loading) // Prevents race conditions when multiple queries trigger rebuild simultaneously private lazyRebuildInProgress = false private lazyRebuildCompleted = false @@ -172,7 +172,7 @@ export class Brainy implements BrainyInterface { // Normalize configuration with defaults this.config = this.normalizeConfig(config) - // Configure memory limits (v5.11.0) + // Configure memory limits // This must happen early, before any validation occurs if (this.config.maxQueryLimit !== undefined || this.config.reservedQueryMemory !== undefined) { ValidationConfig.reconfigure({ @@ -192,7 +192,7 @@ export class Brainy implements BrainyInterface { this.setupDistributedComponents() } - // Initialize ready Promise (v7.3.0) + // Initialize ready Promise // This allows consumers to await brain.ready before using the database this._readyPromise = new Promise((resolve, reject) => { this._readyResolve = resolve @@ -260,7 +260,7 @@ export class Brainy implements BrainyInterface { this.storage = await this.setupStorage() await this.storage.init() - // Enable COW immediately after storage init (v5.0.1) + // Enable COW immediately after storage init // This ensures ALL data is stored in branch-scoped paths from the start // Lightweight: just sets cowEnabled=true and currentBranch, no RefManager/BlobStorage yet if (typeof (this.storage as any).enableCOWLightweight === 'function') { @@ -274,7 +274,7 @@ export class Brainy implements BrainyInterface { this.metadataIndex = new MetadataIndexManager(this.storage) await this.metadataIndex.init() - // v6.3.0: Get GraphAdjacencyIndex from storage (SINGLETON pattern) + // Get GraphAdjacencyIndex from storage (SINGLETON pattern) // Storage owns the single instance, Brainy accesses it via getGraphIndex() // This fixes the dual-ownership bug where Brainy and Storage had separate instances // causing verbIdSet to be out of sync and VFS tree queries to fail @@ -314,7 +314,7 @@ export class Brainy implements BrainyInterface { Brainy.shutdownHooksRegisteredGlobally = true } - // v5.2.0: Initialize COW (BlobStorage) before VFS + // Initialize COW (BlobStorage) before VFS // VFS now requires BlobStorage for unified file storage if (typeof (this.storage as any).initializeCOW === 'function') { await (this.storage as any).initializeCOW({ @@ -323,17 +323,17 @@ export class Brainy implements BrainyInterface { }) } - // Mark as initialized BEFORE VFS init (v5.0.1) + // Mark as initialized BEFORE VFS init // VFS.init() needs brain to be marked initialized to call brain methods this.initialized = true - // Initialize VFS (v5.0.1): Ensure VFS is ready when accessed as property + // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) await this._vfs.init() - this._vfsInitialized = true // v7.3.0: Mark VFS as fully initialized + this._vfsInitialized = true // Mark VFS as fully initialized - // v7.1.2: Eager embedding initialization for cloud deployments + // Eager embedding initialization for cloud deployments // When eagerEmbeddings is true, initialize the WASM embedding engine now // instead of lazily on first embed() call. This moves the 90-140 second // WASM compilation to container startup rather than first request. @@ -344,7 +344,7 @@ export class Brainy implements BrainyInterface { console.log('βœ… Embedding engine ready') } - // v7.4.0: Integration Hub initialization + // Integration Hub initialization // Creates the hub when integrations are enabled in config // Uses dynamic import for tree-shaking when integrations are disabled if (this.config.integrations) { @@ -360,12 +360,12 @@ export class Brainy implements BrainyInterface { }) } - // v7.3.0: Resolve ready Promise - consumers awaiting brain.ready will now proceed + // Resolve ready Promise - consumers awaiting brain.ready will now proceed if (this._readyResolve) { this._readyResolve() } } catch (error) { - // v7.3.0: Reject ready Promise - consumers awaiting brain.ready will receive error + // Reject ready Promise - consumers awaiting brain.ready will receive error if (this._readyReject) { this._readyReject(error instanceof Error ? error : new Error(String(error))) } @@ -374,7 +374,7 @@ export class Brainy implements BrainyInterface { } /** - * Register shutdown hooks for graceful count flushing (v3.32.3+) + * Register shutdown hooks for graceful count flushing * * Ensures pending count batches are persisted before container shutdown. * Critical for Cloud Run, Fargate, Lambda, and other containerized deployments. @@ -444,7 +444,7 @@ export class Brainy implements BrainyInterface { * This Promise is created in the constructor and resolves when init() completes. * It can be awaited multiple times safely - the result is cached. * - * **v7.3.0 Feature**: This enables reliable readiness detection for consumers, + * This enables reliable readiness detection for consumers, * especially in cloud environments where progressive initialization means * init() returns quickly but background tasks may still be running. * @@ -522,7 +522,7 @@ export class Brainy implements BrainyInterface { /** * Wait for all background initialization tasks to complete * - * For cloud storage adapters with progressive initialization (v7.3.0+), + * For cloud storage adapters with progressive initialization, * this waits for: * - Bucket/container validation * - Count synchronization @@ -572,8 +572,8 @@ export class Brainy implements BrainyInterface { * @param params.id - Custom ID (auto-generated if not provided) * @param params.vector - Pre-computed embedding vector * @param params.service - Service name for multi-tenancy - * @param params.confidence - Type classification confidence (0-1) *New in v4.3.0* - * @param params.weight - Entity importance/salience (0-1) *New in v4.3.0* + * @param params.confidence - Type classification confidence (0-1) + * @param params.weight - Entity importance/salience (0-1) * @returns Promise that resolves to the entity ID * * @example Basic entity creation @@ -586,7 +586,7 @@ export class Brainy implements BrainyInterface { * console.log(`Created entity: ${id}`) * ``` * - * @example Adding with confidence and weight (New in v4.3.0) + * @example Adding with confidence and weight * ```typescript * const id = await brain.add({ * data: "Machine learning model for sentiment analysis", @@ -631,7 +631,7 @@ export class Brainy implements BrainyInterface { async add(params: AddParams): Promise { await this.ensureInitialized() - // Zero-config validation (v7.3.0: static import for performance) + // Zero-config validation (static import for performance) validateAddParams(params) // Generate ID if not provided @@ -666,7 +666,7 @@ export class Brainy implements BrainyInterface { ...(params.createdBy && { createdBy: params.createdBy }) } - // v4.8.0: Build entity structure for indexing (NEW - with top-level fields) + // Build entity structure for indexing (NEW - with top-level fields) const entityForIndexing = { id, vector, @@ -684,10 +684,10 @@ export class Brainy implements BrainyInterface { metadata: params.metadata || {} } - // v5.8.0: Execute atomically with transaction system + // Execute atomically with transaction system // All operations succeed or all rollback - prevents partial failures await this.transactionManager.executeTransaction(async (tx) => { - // Operation 1: Save metadata FIRST (v5.0.1 - TypeAwareStorage caching) + // Operation 1: Save metadata FIRST (TypeAwareStorage caching) tx.addOperation( new SaveNounMetadataOperation(this.storage, id, storageMetadata) ) @@ -702,7 +702,7 @@ export class Brainy implements BrainyInterface { }) ) - // Operation 3: Add to HNSW index (v5.4.0 - after entity saved) + // Operation 3: Add to HNSW index (after entity saved) if (this.index instanceof TypeAwareHNSWIndex) { tx.addOperation( new AddToTypeAwareHNSWOperation( @@ -734,7 +734,7 @@ export class Brainy implements BrainyInterface { * @param id - The unique identifier of the entity to retrieve * @returns Promise that resolves to the entity if found, null if not found * - * **Entity includes (v4.3.0):** + * **Entity includes:** * - `confidence` - Type classification confidence (0-1) if set * - `weight` - Entity importance/salience (0-1) if set * - All standard fields: id, type, data, metadata, vector, timestamps @@ -750,7 +750,7 @@ export class Brainy implements BrainyInterface { * } * * @example - * // Accessing confidence and weight (New in v4.3.0) + * // Accessing confidence and weight * const entity = await brainy.get('concept-456') * if (entity) { * console.log(`Type: ${entity.type}`) @@ -816,7 +816,7 @@ export class Brainy implements BrainyInterface { /** * Get an entity by ID * - * **Performance (v5.11.1)**: Optimized for metadata-only reads by default + * **Performance**: Optimized for metadata-only reads by default * - **Default (metadata-only)**: 10ms, 300 bytes - 76-81% faster * - **Full entity (includeVectors: true)**: 43ms, 6KB - when vectors needed * @@ -856,17 +856,15 @@ export class Brainy implements BrainyInterface { * * @performance * - Metadata-only: 76-81% faster, 95% less bandwidth, 87% less memory - * - Full entity: Same as v5.11.0 (no regression) + * - Full entity: Same (no regression) * - VFS operations: 81% faster with zero code changes * - * @since v1.0.0 - * @since v5.11.1 - Metadata-only default for 76-81% speedup */ async get(id: string, options?: GetOptions): Promise | null> { await this.ensureInitialized() return this.augmentationRegistry.execute('get', { id, options }, async () => { - // v5.11.1: Route to metadata-only or full entity based on options + // Route to metadata-only or full entity based on options const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) if (includeVectors) { @@ -890,7 +888,7 @@ export class Brainy implements BrainyInterface { } /** - * Batch get multiple entities by IDs (v5.12.0 - Cloud Storage Optimization) + * Batch get multiple entities by IDs (Cloud Storage Optimization) * * **Performance**: Eliminates N+1 query pattern * - Current: N Γ— get() = N Γ— 300ms cloud latency = 3-6 seconds for 10-20 entities @@ -913,8 +911,6 @@ export class Brainy implements BrainyInterface { * const childrenMap = await brain.batchGet(childIds) * const children = childIds.map(id => childrenMap.get(id)).filter(Boolean) * ``` - * - * @since v5.12.0 */ async batchGet(ids: string[], options?: GetOptions): Promise>> { await this.ensureInitialized() @@ -925,7 +921,7 @@ export class Brainy implements BrainyInterface { const includeVectors = options?.includeVectors ?? false if (includeVectors) { - // v6.2.0: FULL PATH optimized with batch vector loading (10x faster on GCS) + // FULL PATH optimized with batch vector loading (10x faster on GCS) // GCS: 10 entities with vectors = 1Γ—50ms vs 10Γ—50ms = 500ms (10x faster) const nounsMap = await this.storage.getNounBatch(ids) @@ -968,24 +964,24 @@ export class Brainy implements BrainyInterface { } /** - * Convert a noun from storage to an entity (v4.8.0 - SIMPLIFIED!) + * Convert a noun from storage to an entity (SIMPLIFIED!) * - * v4.8.0: Dramatically simplified - standard fields moved to top-level + * Dramatically simplified - standard fields moved to top-level * - Extracts standard fields from metadata (storage format) * - Returns entity with standard fields at top-level (in-memory format) * - metadata contains ONLY custom user fields */ private async convertNounToEntity(noun: any): Promise> { - // v4.8.0: Storage adapters ALREADY extract standard fields to top-level! + // Storage adapters ALREADY extract standard fields to top-level! // Just read from top-level fields of HNSWNounWithMetadata - // v4.8.0: Clean structure with standard fields at top-level + // Clean structure with standard fields at top-level const entity: Entity = { id: noun.id, vector: noun.vector, type: noun.type || NounType.Thing, - // Standard fields at top-level (v4.8.0) + // Standard fields at top-level confidence: noun.confidence, weight: noun.weight, createdAt: noun.createdAt || Date.now(), @@ -994,7 +990,7 @@ export class Brainy implements BrainyInterface { data: noun.data, createdBy: noun.createdBy, - // ONLY custom user fields in metadata (v4.8.0: already separated by storage adapter) + // ONLY custom user fields in metadata (already separated by storage adapter) metadata: noun.metadata as T } @@ -1002,7 +998,7 @@ export class Brainy implements BrainyInterface { } /** - * Convert metadata-only to entity (v5.11.1 - FAST PATH!) + * Convert metadata-only to entity (FAST PATH!) * * Used when vectors are NOT needed (94% of brain.get() calls): * - VFS operations (readFile, stat, readdir) @@ -1018,13 +1014,12 @@ export class Brainy implements BrainyInterface { * @param metadata - Metadata from storage.getNounMetadata() * @returns Entity with stub vector (Float32Array(0)) * - * @since v5.11.1 */ private async convertMetadataToEntity(id: string, metadata: any): Promise> { - // v5.11.1: Metadata-only entity (no vector loading) + // Metadata-only entity (no vector loading) // This is 76-81% faster for operations that don't need semantic similarity - // v4.8.0: Extract standard fields, rest are custom metadata + // Extract standard fields, rest are custom metadata // Same destructuring as baseStorage.getNoun() to ensure consistency const { noun, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata @@ -1042,7 +1037,7 @@ export class Brainy implements BrainyInterface { data, createdBy, - // Custom user fields (v4.8.0: standard fields removed, only custom remain) + // Custom user fields (standard fields removed, only custom remain) metadata: customMetadata as T } @@ -1055,11 +1050,11 @@ export class Brainy implements BrainyInterface { async update(params: UpdateParams): Promise { await this.ensureInitialized() - // Zero-config validation (v7.3.0: static import for performance) + // Zero-config validation (static import for performance) validateUpdateParams(params) return this.augmentationRegistry.execute('update', params, async () => { - // Get existing entity with vectors (v6.7.0: fix for v5.11.1 regression) + // Get existing entity with vectors (fix for regression) // We need includeVectors: true because: // 1. SaveNounOperation requires the vector // 2. HNSW reindexing operations need the original vector @@ -1090,7 +1085,7 @@ export class Brainy implements BrainyInterface { const updatedMetadata = { ...newMetadata, ...dataFields, - data: params.data !== undefined ? params.data : existing.data, // v4.8.0: Store data field + data: params.data !== undefined ? params.data : existing.data, // Store data field noun: params.type || existing.type, service: existing.service, createdAt: existing.createdAt, @@ -1102,7 +1097,7 @@ export class Brainy implements BrainyInterface { ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }) } - // v4.8.0: Build entity structure for metadata index (with top-level fields) + // Build entity structure for metadata index (with top-level fields) const entityForIndexing = { id: params.id, vector, @@ -1120,9 +1115,9 @@ export class Brainy implements BrainyInterface { metadata: newMetadata } - // v5.8.0: Execute atomically with transaction system + // Execute atomically with transaction system await this.transactionManager.executeTransaction(async (tx) => { - // Operation 1: Update metadata FIRST (v5.1.0 - updates type cache) + // Operation 1: Update metadata FIRST (updates type cache) tx.addOperation( new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) ) @@ -1167,7 +1162,7 @@ export class Brainy implements BrainyInterface { } // Operation 5-6: Update metadata index (remove old, add new) - // v7.5.0 FIX: Include ALL indexed fields in removalMetadata (not just type) + // FIX: Include ALL indexed fields in removalMetadata (not just type) // Previously, only metadata + type was removed, but entityForIndexing includes: // confidence, weight, createdAt, updatedAt, service, data, createdBy // This asymmetry caused 7 fields to accumulate on EVERY update, eventually @@ -1177,7 +1172,7 @@ export class Brainy implements BrainyInterface { // console.log('[UPDATE DEBUG] existing.metadata:', JSON.stringify(existing.metadata)) // console.log('[UPDATE DEBUG] entityForIndexing keys:', Object.keys(entityForIndexing)) // - // v7.5.0 FIX: removalMetadata must MATCH entityForIndexing structure + // FIX: removalMetadata must MATCH entityForIndexing structure // entityForIndexing has: { type, confidence, ..., metadata: {...} } // So removalMetadata must also have: { type, confidence, ..., metadata: {...} } const removalMetadata = { @@ -1220,7 +1215,7 @@ export class Brainy implements BrainyInterface { const targetVerbs = await this.storage.getVerbsByTarget(id) const allVerbs = [...verbs, ...targetVerbs] - // v5.8.0: Execute atomically with transaction system + // Execute atomically with transaction system await this.transactionManager.executeTransaction(async (tx) => { // Operation 1: Remove from vector index if (noun && metadata) { @@ -1376,7 +1371,7 @@ export class Brainy implements BrainyInterface { async relate(params: RelateParams): Promise { await this.ensureInitialized() - // Zero-config validation (v7.3.0: static import for performance) + // Zero-config validation (static import for performance) validateRelateParams(params) // Verify entities exist @@ -1390,13 +1385,13 @@ export class Brainy implements BrainyInterface { throw new Error(`Target entity ${params.to} not found`) } - // CRITICAL FIX (v3.43.2): Check for duplicate relationships + // CRITICAL FIX: Check for duplicate relationships // This prevents infinite loops where same relationship is created repeatedly // Bug #1 showed incrementing verb counts (7β†’8β†’9...) indicating duplicates - // v5.8.0 OPTIMIZATION: Use GraphAdjacencyIndex for O(log n) lookup instead of O(n) storage scan + // OPTIMIZATION: Use GraphAdjacencyIndex for O(log n) lookup instead of O(n) storage scan const verbIds = await this.graphIndex.getVerbIdsBySource(params.from) - // v6.2.0: Batch-load verbs for 5x faster duplicate checking on GCS + // Batch-load verbs for 5x faster duplicate checking on GCS // GCS: 5 verbs = 1Γ—50ms vs 5Γ—50ms = 250ms (5x faster) if (verbIds.length > 0) { const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds) @@ -1420,8 +1415,8 @@ export class Brainy implements BrainyInterface { ) return this.augmentationRegistry.execute('relate', params, async () => { - // v4.0.0: Prepare verb metadata - // CRITICAL (v4.1.2): Include verb type in metadata for count tracking + // Prepare verb metadata + // CRITICAL: Include verb type in metadata for count tracking const verbMetadata = { verb: params.type, // Store verb type for count synchronization weight: params.weight ?? 1.0, @@ -1429,7 +1424,7 @@ export class Brainy implements BrainyInterface { createdAt: Date.now() } - // Save to storage (v4.0.0: vector and metadata separately) + // Save to storage (vector and metadata separately) const verb: GraphVerb = { id, vector: relationVector, @@ -1444,7 +1439,7 @@ export class Brainy implements BrainyInterface { createdAt: Date.now() } as any - // v5.8.0: Execute atomically with transaction system + // Execute atomically with transaction system await this.transactionManager.executeTransaction(async (tx) => { // Operation 1: Save verb vector data tx.addOperation( @@ -1518,7 +1513,7 @@ export class Brainy implements BrainyInterface { // Get verb data before deletion for rollback const verb = await this.storage.getVerb(id) - // v5.8.0: Execute atomically with transaction system + // Execute atomically with transaction system await this.transactionManager.executeTransaction(async (tx) => { // Operation 1: Remove from graph index if (verb) { @@ -1564,8 +1559,6 @@ export class Brainy implements BrainyInterface { * const page2 = await brain.getRelations({ offset: 100, limit: 100 }) * ``` * - * @since v4.1.3 - Fixed bug where calling without parameters returned empty array - * @since v4.1.3 - Added string ID shorthand syntax: getRelations(id) */ async getRelations( paramsOrId?: string | GetRelationsParams @@ -1607,7 +1600,7 @@ export class Brainy implements BrainyInterface { filter.service = params.service } - // v4.7.0: VFS relationships are no longer filtered + // VFS relationships are no longer filtered // VFS is part of the knowledge graph - users can filter explicitly if needed // Fetch from storage with pagination at storage layer (efficient!) @@ -1633,7 +1626,7 @@ export class Brainy implements BrainyInterface { * @param query - Natural language string or structured FindParams object * @returns Promise that resolves to array of search results with scores * - * **Result Structure (v4.3.0):** + * **Result Structure:** * Each result includes flattened entity fields for convenient access: * - `metadata`, `type`, `data` - Direct access (flattened from entity) * - `confidence`, `weight` - Entity confidence/importance (if set) @@ -1658,7 +1651,7 @@ export class Brainy implements BrainyInterface { * } * }) * - * // NEW in v4.3.0: Access flattened fields directly + * // Access flattened fields directly * for (const result of results) { * console.log(`Score: ${result.score}`) * console.log(`Type: ${result.type}`) // Flattened! @@ -1781,13 +1774,13 @@ export class Brainy implements BrainyInterface { * } * * @example - * // VFS Filtering (v4.4.0): Exclude VFS entities by default + * // VFS Filtering: Exclude VFS entities by default * // Knowledge graph queries stay clean - no VFS files in results * const knowledge = await brainy.find({ query: 'AI concepts' }) * // Returns only knowledge entities, VFS files excluded * * @example - * // v4.7.0: VFS entities included by default + * // VFS entities included by default * const everything = await brainy.find({ * query: 'documentation' * }) @@ -1803,13 +1796,13 @@ export class Brainy implements BrainyInterface { * // Exclude VFS entities (if needed) * const concepts = await brainy.find({ * query: 'machine learning', - * excludeVFS: true // v4.7.0: Exclude VFS files + * excludeVFS: true // Exclude VFS files * }) */ async find(query: string | FindParams): Promise[]> { await this.ensureInitialized() - // v5.7.7: Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) + // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) // This is a production-safe, concurrency-controlled lazy load await this.ensureIndexesLoaded() @@ -1817,7 +1810,7 @@ export class Brainy implements BrainyInterface { const params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query - // Zero-config validation (v7.3.0: static import for performance) + // Zero-config validation (static import for performance) validateFindParams(params) const startTime = Date.now() @@ -1851,7 +1844,7 @@ export class Brainy implements BrainyInterface { } } - // v5.7.13: excludeVFS helper - ONLY exclude VFS infrastructure entities + // ExcludeVFS helper - ONLY exclude VFS infrastructure entities // Applied AFTER type filter to avoid execution order bugs // Excludes entities where: // - vfsType is 'file' or 'directory' (VFS files/folders) @@ -1865,7 +1858,7 @@ export class Brainy implements BrainyInterface { filter.isVFSEntity = { ne: true } } - // v4.5.4: Apply sorting if requested, otherwise just filter + // Apply sorting if requested, otherwise just filter let filteredIds: string[] if (params.orderBy) { // Get sorted IDs using production-scale sorted filtering @@ -1884,7 +1877,7 @@ export class Brainy implements BrainyInterface { const offset = params.offset || 0 const pageIds = filteredIds.slice(offset, offset + limit) - // v6.2.0: Batch-load entities for 10x faster cloud storage performance + // Batch-load entities for 10x faster cloud storage performance // GCS: 10 entities = 1Γ—50ms vs 10Γ—50ms = 500ms (10x faster) const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { @@ -1902,7 +1895,7 @@ export class Brainy implements BrainyInterface { const limit = params.limit || 20 const offset = params.offset || 0 - // v5.7.13: excludeVFS helper - exclude VFS infrastructure entities + // ExcludeVFS helper - exclude VFS infrastructure entities // VFS files/folders have vfsType set, extracted entities do NOT let filter: any = {} if (params.excludeVFS === true) { @@ -1915,7 +1908,7 @@ export class Brainy implements BrainyInterface { const filteredIds = await this.metadataIndex.getIdsForFilter(filter) const pageIds = filteredIds.slice(offset, offset + limit) - // v6.2.0: Batch-load entities for 10x faster cloud storage performance + // Batch-load entities for 10x faster cloud storage performance const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) @@ -1941,7 +1934,7 @@ export class Brainy implements BrainyInterface { return results } - // v7.7.0: Zero-Config Hybrid Search + // Zero-Config Hybrid Search // Determine search mode: auto (default) combines text + semantic for query searches const searchMode = params.searchMode || 'auto' const limit = params.limit || 10 @@ -1965,7 +1958,7 @@ export class Brainy implements BrainyInterface { // Use user-specified alpha or auto-detect based on query length const alpha = params.hybridAlpha ?? this.autoAlpha(params.query) - // v7.8.0: Tokenize query for match visibility + // Tokenize query for match visibility const queryWords = this.metadataIndex.tokenize(params.query) // RRF fusion combines both result sets with match visibility @@ -2007,7 +2000,7 @@ export class Brainy implements BrainyInterface { if (params.where) Object.assign(filter, params.where) if (params.service) filter.service = params.service - // v5.7.13: excludeVFS helper - exclude VFS infrastructure entities + // ExcludeVFS helper - exclude VFS infrastructure entities // VFS files/folders have vfsType set, extracted entities do NOT if (params.excludeVFS === true) { filter.vfsType = { exists: false } @@ -2046,7 +2039,7 @@ export class Brainy implements BrainyInterface { results.sort((a, b) => b.score - a.score) results = results.slice(offset, offset + limit) - // v6.2.0: Batch-load entities only for the paginated results (10x faster on GCS) + // Batch-load entities only for the paginated results (10x faster on GCS) const idsToLoad = results.filter(r => !r.entity).map(r => r.id) if (idsToLoad.length > 0) { const entitiesMap = await this.batchGet(idsToLoad) @@ -2071,7 +2064,7 @@ export class Brainy implements BrainyInterface { const offset = params.offset || 0 const pageIds = filteredIds.slice(offset, offset + limit) - // v6.2.0: Batch-load entities for current page - O(page_size) instead of O(total_results) + // Batch-load entities for current page - O(page_size) instead of O(total_results) // GCS: 10 entities = 1Γ—50ms vs 10Γ—50ms = 500ms (10x faster) const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { @@ -2083,7 +2076,7 @@ export class Brainy implements BrainyInterface { // Early return for metadata-only queries with pagination applied if (!params.query && !params.connected) { - // v4.5.4: Apply sorting if requested for metadata-only queries + // Apply sorting if requested for metadata-only queries if (params.orderBy) { const sortedIds = await this.metadataIndex.getSortedIdsForFilter( filter, @@ -2096,7 +2089,7 @@ export class Brainy implements BrainyInterface { const offset = params.offset || 0 const pageIds = sortedIds.slice(offset, offset + limit) - // v6.2.0: Batch-load entities for paginated results (10x faster on GCS) + // Batch-load entities for paginated results (10x faster on GCS) const sortedResults: Result[] = [] const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { @@ -2125,7 +2118,7 @@ export class Brainy implements BrainyInterface { } // OPTIMIZED: Sort first, then apply efficient pagination - // v4.5.4: Support custom orderBy for vector + metadata queries + // Support custom orderBy for vector + metadata queries if (params.orderBy && results.length > 0) { // For vector + metadata queries, sort by specified field instead of score // Load sort field values for all results (small set, already filtered) @@ -2177,7 +2170,7 @@ export class Brainy implements BrainyInterface { * @param params.where - Metadata filters * @returns Promise that resolves to array of Result objects with similarity scores (same structure as find()) * - * **Returns (v4.3.0):** + * **Returns:** * Same Result structure as find() with flattened fields for convenient access * * @example @@ -2187,7 +2180,7 @@ export class Brainy implements BrainyInterface { * limit: 10 * }) * - * // NEW in v4.3.0: Access flattened fields + * // Access flattened fields * for (const result of similarDocs) { * console.log(`Similarity: ${result.score}`) * console.log(`Type: ${result.type}`) // Flattened! @@ -2293,7 +2286,7 @@ export class Brainy implements BrainyInterface { let targetVector: Vector if (typeof params.to === 'string') { - // v5.11.1: Need vector for similarity, so use includeVectors: true + // Need vector for similarity, so use includeVectors: true const entity = await this.get(params.to, { includeVectors: true }) if (!entity) { throw new Error(`Entity ${params.to} not found`) @@ -2302,7 +2295,7 @@ export class Brainy implements BrainyInterface { } else if (Array.isArray(params.to)) { targetVector = params.to as Vector } else { - // v5.11.1: Entity object passed - check if vectors are loaded + // Entity object passed - check if vectors are loaded const entityVector = (params.to as Entity).vector if (!entityVector || entityVector.length === 0) { throw new Error( @@ -2321,7 +2314,7 @@ export class Brainy implements BrainyInterface { type: params.type, where: params.where, service: params.service, - excludeVFS: params.excludeVFS // v4.7.0: Pass through VFS filtering + excludeVFS: params.excludeVFS // Pass through VFS filtering }) } @@ -2333,7 +2326,7 @@ export class Brainy implements BrainyInterface { async addMany(params: AddManyParams): Promise> { await this.ensureInitialized() - // Get optimal batch configuration from storage adapter (v4.11.0) + // Get optimal batch configuration from storage adapter // This automatically adapts to storage characteristics: // - GCS: 50 batch size, 100ms delay, sequential // - S3/R2: 100 batch size, 50ms delay, parallel @@ -2441,7 +2434,7 @@ export class Brainy implements BrainyInterface { const startTime = Date.now() - // v6.2.0: Batch deletes into chunks for 10x faster performance with proper error handling + // Batch deletes into chunks for 10x faster performance with proper error handling // Single transaction per chunk (10 entities) = atomic within chunk, graceful failure across chunks const chunkSize = 10 @@ -2606,7 +2599,7 @@ export class Brainy implements BrainyInterface { async relateMany(params: RelateManyParams): Promise { await this.ensureInitialized() - // Get optimal batch configuration from storage adapter (v4.11.0) + // Get optimal batch configuration from storage adapter // Automatically adapts to storage characteristics const storageConfig = this.storage.getBatchConfig() @@ -2697,7 +2690,7 @@ export class Brainy implements BrainyInterface { // Clear storage await this.storage.clear() - // v7.4.1: Invalidate GraphAdjacencyIndex to prevent stale in-memory data + // Invalidate GraphAdjacencyIndex to prevent stale in-memory data // The index has LSMTree data and verbIdSet pointing to deleted entities. // Without this, relate()'s duplicate check uses stale data, potentially // allowing duplicate relationships or missing valid duplicates. @@ -2714,7 +2707,7 @@ export class Brainy implements BrainyInterface { this.index = this.setupIndex() } - // v5.10.4: Recreate metadata index to clear cached data + // Recreate metadata index to clear cached data // Bug: Metadata index cache was not being cleared, causing find() with type filters to return stale data this.metadataIndex = new MetadataIndexManager(this.storage) await this.metadataIndex.init() @@ -2727,7 +2720,7 @@ export class Brainy implements BrainyInterface { this._nlp = undefined this._tripleIntelligence = undefined - // v7.3.1: Re-initialize COW (BlobStorage) after storage.clear() + // Re-initialize COW (BlobStorage) after storage.clear() // storage.clear() sets blobStorage=undefined for FileSystem/cloud adapters // VFS depends on blobStorage being available (unified blob storage for all files) // Must be done BEFORE VFS reinitialization @@ -2738,7 +2731,7 @@ export class Brainy implements BrainyInterface { }) } - // v7.3.1: Reset VFS state - root entity was deleted by storage.clear() + // Reset VFS state - root entity was deleted by storage.clear() // Bug: VFS instance remained in memory pointing to deleted root entity // Following checkout() pattern exactly (see lines 2907-2914) if (this._vfs) { @@ -2757,7 +2750,7 @@ export class Brainy implements BrainyInterface { }) } - // ============= COW (COPY-ON-WRITE) API - v5.0.0 ============= + // ============= COW (COPY-ON-WRITE) API ============= /** * Fork the brain (instant clone via Snowflake-style COW) @@ -2766,7 +2759,7 @@ export class Brainy implements BrainyInterface { * Fork shares storage and HNSW data structures with parent, copying only * when modified (lazy deep copy). * - * **How It Works (v5.0.0)**: + * **How It Works**: * 1. HNSW Index: Shallow copy via `enableCOW()` (~10ms for 1M+ nodes) * 2. Metadata Index: Fast rebuild from shared storage (<100ms) * 3. Graph Index: Fast rebuild from shared storage (<500ms) @@ -2801,7 +2794,6 @@ export class Brainy implements BrainyInterface { * console.log((await experiment.find({})).length) // 2 (Alice + Bob) * ``` * - * @since v5.0.0 */ async fork(branch?: string, options?: { author?: string @@ -2813,7 +2805,7 @@ export class Brainy implements BrainyInterface { return this.augmentationRegistry.execute('fork', { branch, options }, async () => { const branchName = branch || `fork-${Date.now()}` - // v5.0.1: Lazy COW initialization - enable automatically on first fork() + // Lazy COW initialization - enable automatically on first fork() // This is zero-config and transparent to users if (!('refManager' in this.storage) || !(this.storage as any).refManager) { // Storage supports COW but isn't initialized yet - initialize now @@ -2827,7 +2819,7 @@ export class Brainy implements BrainyInterface { throw new Error( 'Fork requires COW-enabled storage. ' + 'This storage adapter does not support branching. ' + - 'Please use v5.0.0+ storage adapters.' + 'Please use compatible storage adapters.' ) } } @@ -2852,7 +2844,7 @@ export class Brainy implements BrainyInterface { // Step 2: Copy storage ref (COW layer - instant!) await refManager.copyRef(currentBranch, branchName) - // CRITICAL FIX (v5.3.6): Verify branch was actually created to prevent silent failures + // CRITICAL FIX: Verify branch was actually created to prevent silent failures // Without this check, fork() could complete successfully but branch wouldn't exist, // causing subsequent checkout() calls to fail (see Workshop bug report). const verifyBranch = await refManager.getRef(branchName) @@ -2892,7 +2884,7 @@ export class Brainy implements BrainyInterface { clone.metadataIndex = new MetadataIndexManager(clone.storage) await clone.metadataIndex.init() - // v6.3.0: GraphAdjacencyIndex SINGLETON pattern for fork() + // GraphAdjacencyIndex SINGLETON pattern for fork() // Object.create() causes prototype inheritance, so clone.storage.graphIndex // would point to parent's graphIndex. We must break this inheritance and // create a fresh instance for the clone's branch. @@ -2937,7 +2929,7 @@ export class Brainy implements BrainyInterface { return this.augmentationRegistry.execute('listBranches', {}, async () => { if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage (v5.0.0+)') + throw new Error('Branch management requires COW-enabled storage') } const refManager = (this.storage as any).refManager @@ -2987,7 +2979,7 @@ export class Brainy implements BrainyInterface { return this.augmentationRegistry.execute('checkout', { branch }, async () => { if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage (v5.0.0+)') + throw new Error('Branch management requires COW-enabled storage') } // Verify branch exists @@ -2999,26 +2991,26 @@ export class Brainy implements BrainyInterface { // Update storage currentBranch (this.storage as any).currentBranch = branch - // v5.3.5 fix: Reload indexes from new branch WITHOUT recreating storage + // Fix: Reload indexes from new branch WITHOUT recreating storage // Previous implementation called init() which recreated storage, losing currentBranch this.index = this.setupIndex() this.metadataIndex = new (MetadataIndexManager as any)(this.storage) await this.metadataIndex.init() - // v6.3.0: GraphAdjacencyIndex SINGLETON pattern for checkout() + // GraphAdjacencyIndex SINGLETON pattern for checkout() // Invalidate the old graphIndex (it has data from the old branch) // and get a fresh instance for the new branch ;(this.storage as any).invalidateGraphIndex() this.graphIndex = await (this.storage as any).getGraphIndex() - // v5.7.7: Reset lazy loading state when switching branches + // Reset lazy loading state when switching branches // Indexes contain data from previous branch, must rebuild for new branch this.lazyRebuildCompleted = false // Rebuild indexes from new branch data (force=true to override disableAutoRebuild) await this.rebuildIndexesIfNeeded(true) - // v6.3.0: Clear VFS caches before recreating VFS for new branch + // Clear VFS caches before recreating VFS for new branch // UnifiedCache is global, so old branch's VFS path cache entries would persist if (this._vfs) { // Clear old PathResolver's caches including UnifiedCache entries @@ -3050,13 +3042,13 @@ export class Brainy implements BrainyInterface { message?: string author?: string metadata?: Record - captureState?: boolean // v5.3.7: Capture entity snapshots for time-travel + captureState?: boolean // Capture entity snapshots for time-travel }): Promise { await this.ensureInitialized() return this.augmentationRegistry.execute('commit', { options }, async () => { if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) { - throw new Error('Commit requires COW-enabled storage (v5.0.0+)') + throw new Error('Commit requires COW-enabled storage') } const refManager = (this.storage as any).refManager @@ -3070,10 +3062,10 @@ export class Brainy implements BrainyInterface { const entityCount = await this.getNounCount() const relationshipCount = await this.getVerbCount() - // v5.3.4: Import NULL_HASH constant + // Import NULL_HASH constant const { NULL_HASH } = await import('./storage/cow/constants.js') - // v5.3.7: Capture entity state if requested (for time-travel) + // Capture entity state if requested (for time-travel) let treeHash = NULL_HASH if (options?.captureState) { treeHash = await this.captureStateToTree() @@ -3114,7 +3106,7 @@ export class Brainy implements BrainyInterface { } /** - * Capture current entity and relationship state to tree object (v5.4.0) + * Capture current entity and relationship state to tree object * Used by commit({ captureState: true }) for time-travel * * Serializes ALL entities + relationships to blobs and builds a tree. @@ -3210,7 +3202,7 @@ export class Brainy implements BrainyInterface { } /** - * Create a read-only snapshot of the workspace at a specific commit (v5.4.0) + * Create a read-only snapshot of the workspace at a specific commit * * Time-travel API for historical queries. Returns a new Brainy instance that: * - Contains all entities and relationships from that commit @@ -3252,7 +3244,7 @@ export class Brainy implements BrainyInterface { }): Promise { await this.ensureInitialized() - // v5.4.0: Lazy-loading historical adapter with bounded memory + // Lazy-loading historical adapter with bounded memory // No eager loading of entire commit state! const { HistoricalStorageAdapter } = await import('./storage/adapters/historicalStorageAdapter.js') const { BaseStorage } = await import('./storage/baseStorage.js') @@ -3305,7 +3297,7 @@ export class Brainy implements BrainyInterface { return this.augmentationRegistry.execute('deleteBranch', { branch }, async () => { if (!('refManager' in this.storage)) { - throw new Error('Branch management requires COW-enabled storage (v5.0.0+)') + throw new Error('Branch management requires COW-enabled storage') } const currentBranch = await this.getCurrentBranch() @@ -3346,7 +3338,7 @@ export class Brainy implements BrainyInterface { return this.augmentationRegistry.execute('getHistory', { options }, async () => { if (!('commitLog' in this.storage) || !('refManager' in this.storage)) { - throw new Error('History requires COW-enabled storage (v5.0.0+)') + throw new Error('History requires COW-enabled storage') } const commitLog = (this.storage as any).commitLog @@ -3418,7 +3410,7 @@ export class Brainy implements BrainyInterface { await this.ensureInitialized() if (!('commitLog' in this.storage) || !('refManager' in this.storage)) { - throw new Error('History streaming requires COW-enabled storage (v5.0.0+)') + throw new Error('History streaming requires COW-enabled storage') } const commitLog = (this.storage as any).commitLog @@ -3468,7 +3460,7 @@ export class Brainy implements BrainyInterface { } /** - * Get memory statistics and limits (v5.11.0) + * Get memory statistics and limits * * Returns detailed memory information including: * - Current heap usage @@ -3593,7 +3585,7 @@ export class Brainy implements BrainyInterface { } /** - * Versioning API - Entity version control (v5.3.0) + * Versioning API - Entity version control * * Provides entity-level versioning with: * - save() - Create version of entity @@ -3671,7 +3663,7 @@ export class Brainy implements BrainyInterface { /** * Extract entities from text (alias for extract()) - * v5.7.6: Added for API clarity and Workshop team request + * Added for API clarity and Workshop team request * * Uses NeuralEntityExtractor with SmartExtractor ensemble (4-signal architecture): * - ExactMatch (40%) - Dictionary lookups @@ -3770,7 +3762,7 @@ export class Brainy implements BrainyInterface { * groupBy: 'type', // Organize by entity type * preserveSource: true, // Keep original file * - * // Progress tracking (v4.5.0 - STANDARDIZED FOR ALL 7 FORMATS!) + * // Progress tracking (STANDARDIZED FOR ALL 7 FORMATS!) * onProgress: (p) => { * console.log(`[${p.stage}] ${p.message}`) * console.log(`Entities: ${p.entities || 0}, Rels: ${p.relationships || 0}`) @@ -3780,7 +3772,7 @@ export class Brainy implements BrainyInterface { * // THIS SAME HANDLER WORKS FOR CSV, PDF, Excel, JSON, Markdown, YAML, DOCX! * ``` * - * @example Universal Progress Handler (v4.5.0) + * @example Universal Progress Handler * ```typescript * // ONE handler for ALL 7 formats - no format-specific code needed! * const universalProgress = (p) => { @@ -3815,12 +3807,12 @@ export class Brainy implements BrainyInterface { * * @see {@link https://brainy.dev/docs/api/import API Documentation} * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} - * @see {@link https://brainy.dev/docs/guides/standard-import-progress Standard Progress API (v4.5.0)} + * @see {@link https://brainy.dev/docs/guides/standard-import-progress Standard Progress API} * * @remarks * **⚠️ Breaking Changes from v3.x:** * - * The import API was redesigned in v4.0.0 for clarity and better feature control. + * The import API was redesigned for clarity and better feature control. * Old v3.x option names are **no longer recognized** and will throw errors. * * **Option Changes:** @@ -3873,7 +3865,7 @@ export class Brainy implements BrainyInterface { }) => void } ) { - // Execute through augmentation pipeline (v5.2.0: Enables IntelligentImportAugmentation) + // Execute through augmentation pipeline (Enables IntelligentImportAugmentation) // If source is an ImportSource object (not a Buffer), spread it so augmentations can access properties const params = typeof source === 'object' && !Buffer.isBuffer(source) ? { ...source as object, ...options } // Spread ImportSource: { type, data, filename, ...options } @@ -3891,7 +3883,7 @@ export class Brainy implements BrainyInterface { } /** - * Virtual File System API - Knowledge Operating System (v5.0.1+) + * Virtual File System API - Knowledge Operating System * * Returns a cached VFS instance that is auto-initialized during brain.init(). * No separate initialization needed! @@ -3925,15 +3917,14 @@ export class Brainy implements BrainyInterface { * **Pattern:** The VFS instance is cached, so multiple calls to brain.vfs * return the same instance. This ensures import and user code share state. * - * @since v5.0.1 - Auto-initialization during brain.init() */ get vfs(): VirtualFileSystem { if (!this._vfs) { - // VFS is initialized during brain.init() (v5.0.1) + // VFS is initialized during brain.init() // If not initialized yet, create instance but user should call brain.init() first this._vfs = new VirtualFileSystem(this) } - // v7.3.0: Warn if VFS accessed before init() completed + // Warn if VFS accessed before init() completed if (!this._vfsInitialized && this.initialized) { console.warn('[Brainy] VFS accessed before initialization complete. Call await brain.init() first.') } @@ -3973,7 +3964,6 @@ export class Brainy implements BrainyInterface { * }) * ``` * - * @since v7.4.0 * @throws Error if integrations are not enabled in config */ get hub(): IntegrationHub { @@ -4134,7 +4124,7 @@ export class Brainy implements BrainyInterface { /** * Flush all indexes and caches to persistent storage - * CRITICAL FIX (v3.43.2): Ensures data survives server restarts + * CRITICAL FIX: Ensures data survives server restarts * * Flushes all 4 core indexes: * 1. Storage counts (entity/verb counts by type) @@ -4193,7 +4183,7 @@ export class Brainy implements BrainyInterface { } /** - * Get index loading status (v5.7.7 - Diagnostic for lazy loading) + * Get index loading status (Diagnostic for lazy loading) * * Returns detailed information about index population and lazy loading state. * Useful for debugging empty query results or performance troubleshooting. @@ -4482,7 +4472,7 @@ export class Brainy implements BrainyInterface { relationships: () => this.graphIndex.getTotalRelationshipCount(), // O(1) count by type (string-based, backward compatible) - // v6.2.1: Added optional excludeVFS using Roaring bitmap intersection + // Added optional excludeVFS using Roaring bitmap intersection byType: async (typeOrOptions?: string | { excludeVFS?: boolean }, options?: { excludeVFS?: boolean }) => { // Handle overloaded signature: byType(type), byType({ excludeVFS }), byType(type, { excludeVFS }) let type: string | undefined @@ -4569,7 +4559,7 @@ export class Brainy implements BrainyInterface { getAllTypeCounts: () => this.metadataIndex.getAllEntityCounts(), // Get complete statistics - // v6.2.1: Added optional excludeVFS using Roaring bitmap intersection + // Added optional excludeVFS using Roaring bitmap intersection getStats: async (options?: { excludeVFS?: boolean }) => { if (options?.excludeVFS) { const allCounts = this.metadataIndex.getAllEntityCounts() @@ -4629,7 +4619,7 @@ export class Brainy implements BrainyInterface { /** * Get complete statistics - convenience method * For more granular counting, use brain.counts API - * v6.2.1: Added optional excludeVFS using Roaring bitmap intersection + * Added optional excludeVFS using Roaring bitmap intersection * @param options Optional settings - excludeVFS: filter out VFS entities * @returns Complete statistics including entities, relationships, and density */ @@ -4637,7 +4627,7 @@ export class Brainy implements BrainyInterface { return this.counts.getStats(options) } - // ============= NEW EMBEDDING & ANALYSIS APIs (v7.1.0) ============= + // ============= NEW EMBEDDING & ANALYSIS APIs ============= /** * Batch embed multiple texts at once @@ -4701,7 +4691,7 @@ export class Brainy implements BrainyInterface { } /** - * Zero-config hybrid highlighting (v7.8.0) + * Zero-config hybrid highlighting * * Returns both exact text matches AND semantically similar concepts. * Perfect for UI highlighting at different levels: @@ -4736,7 +4726,7 @@ export class Brainy implements BrainyInterface { return [] } - // v7.9.0: Extract text from structured content (JSON, HTML, Markdown) + // Extract text from structured content (JSON, HTML, Markdown) // Custom extractor takes priority, then built-in detection type ChunkWithCategory = { text: string, position: [number, number], contentCategory?: import('./types/brainy.types.js').ContentCategory } @@ -4767,7 +4757,7 @@ export class Brainy implements BrainyInterface { return [] } - // v7.8.0 Production safety: Limit chunks to prevent memory explosion + // Production safety: Limit chunks to prevent memory explosion // At 500 words Γ— 384 dimensions Γ— 4 bytes = 768KB temp memory (acceptable) const MAX_HIGHLIGHT_CHUNKS = 500 const chunks = allChunks.slice(0, MAX_HIGHLIGHT_CHUNKS) @@ -4857,7 +4847,7 @@ export class Brainy implements BrainyInterface { } /** - * Split text into chunks for highlighting (v7.8.0) + * Split text into chunks for highlighting * @internal */ private splitForHighlighting(text: string, granularity: string): Array<{ text: string, position: [number, number] }> { @@ -4959,11 +4949,11 @@ export class Brainy implements BrainyInterface { } /** - * v7.5.0: Validate metadata index consistency and detect corruption + * Validate metadata index consistency and detect corruption * * Returns health status and recommendations for repair. Corruption typically * manifests as high avg entries/entity (expected ~30, corrupted can be 100+) - * caused by the update() field asymmetry bug (fixed in v7.5.0). + * caused by the update() field asymmetry bug (fixed). * * @returns Promise resolving to validation results * @@ -4986,7 +4976,7 @@ export class Brainy implements BrainyInterface { } /** - * v7.5.0: Get metadata index statistics + * Get metadata index statistics * * Returns detailed statistics about the metadata index including * total entries, IDs indexed, and fields indexed. @@ -5485,7 +5475,7 @@ export class Brainy implements BrainyInterface { ? await this.index.search(vector, limit * 2, params.type as any) : await this.index.search(vector, limit * 2) - // v7.3.0: Batch-load entities for 10-50x faster cloud storage performance + // Batch-load entities for 10-50x faster cloud storage performance // GCS: 10 results = 1Γ—50ms vs 10Γ—50ms = 500ms (10x faster) const ids = searchResults.map(([id]) => id) const entitiesMap = await this.batchGet(ids) @@ -5523,7 +5513,7 @@ export class Brainy implements BrainyInterface { return score >= threshold }) - // v7.3.0: Batch-load entities for 10-50x faster cloud storage performance + // Batch-load entities for 10-50x faster cloud storage performance const ids = filteredResults.map(([id]) => id) const entitiesMap = await this.batchGet(ids) @@ -5565,7 +5555,7 @@ export class Brainy implements BrainyInterface { return existingResults.filter(r => connectedIdSet.has(r.id)) } - // v6.2.0: Batch-load connected entities for 10x faster cloud storage performance + // Batch-load connected entities for 10x faster cloud storage performance // GCS: 20 entities = 1Γ—50ms vs 20Γ—50ms = 1000ms (20x faster) const results: Result[] = [] const entitiesMap = await this.batchGet(connectedIds) @@ -5617,7 +5607,7 @@ export class Brainy implements BrainyInterface { } /** - * Execute text search using word index (v7.7.0) + * Execute text search using word index * * Performs keyword-based search using the __words__ index in MetadataIndexManager. * Returns results ranked by word match count. @@ -5652,7 +5642,7 @@ export class Brainy implements BrainyInterface { } /** - * Auto-detect optimal alpha for hybrid search (v7.7.0) + * Auto-detect optimal alpha for hybrid search * * Short queries (1-2 words) favor text search (lower alpha) * Long queries (5+ words) favor semantic search (higher alpha) @@ -5668,7 +5658,7 @@ export class Brainy implements BrainyInterface { } /** - * Reciprocal Rank Fusion (RRF) for combining search results (v7.7.0) + * Reciprocal Rank Fusion (RRF) for combining search results * * RRF is a proven fusion algorithm that: * - Doesn't require score normalization @@ -5677,7 +5667,7 @@ export class Brainy implements BrainyInterface { * * Formula: score(d) = sum(1 / (k + rank(d))) for each list * - * v7.8.0: Now includes match visibility (textMatches, textScore, semanticScore, matchSource) + * Now includes match visibility (textMatches, textScore, semanticScore, matchSource) * * @param textResults - Results from text search * @param semanticResults - Results from semantic search @@ -5742,7 +5732,7 @@ export class Brainy implements BrainyInterface { } } - // v7.8.0 Performance: Build set of text result IDs for O(1) lookup + // Performance: Build set of text result IDs for O(1) lookup // This avoids re-extracting text for entities that weren't in text results const textResultIds = new Set(textResults.map(r => r.id)) @@ -5779,7 +5769,7 @@ export class Brainy implements BrainyInterface { } /** - * Find which query words match in an entity's text content (v7.8.0) + * Find which query words match in an entity's text content * * Performance: O(query_words Γ— text_length) - only called when needed * At scale: Use textResultIds set for O(1) lookup instead of re-extracting @@ -5847,7 +5837,7 @@ export class Brainy implements BrainyInterface { } /** - * Convert verbs to relations (v4.8.0 - read from top-level) + * Convert verbs to relations (read from top-level) */ private verbsToRelations(verbs: GraphVerb[]): Relation[] { return verbs.map((v) => ({ @@ -5855,7 +5845,7 @@ export class Brainy implements BrainyInterface { from: v.sourceId, to: v.targetId, type: (v.verb || v.type) as VerbType, - weight: v.weight ?? 1.0, // v4.8.0: weight is at top-level + weight: v.weight ?? 1.0, // Weight is at top-level metadata: v.metadata, service: v.service as string, createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now() @@ -6039,7 +6029,7 @@ export class Brainy implements BrainyInterface { } /** - * Explicitly warm up the embedding engine (v7.1.2) + * Explicitly warm up the embedding engine * * Use this to pre-initialize the Candle WASM embedding engine before * processing requests. The WASM module (93MB with embedded model) takes @@ -6075,7 +6065,7 @@ export class Brainy implements BrainyInterface { } /** - * Check if embedding engine is initialized (v7.1.2) + * Check if embedding engine is initialized * * @returns true if embedding engine is ready for immediate use */ @@ -6105,7 +6095,7 @@ export class Brainy implements BrainyInterface { /** * Detect storage type from the storage instance class name * - * v7.1.1: Fixes storage type detection for HNSW persistence mode. + * Fixes storage type detection for HNSW persistence mode. * Previously relied on this.config.storage.type which was often not set * after storage creation, causing cloud storage to use 'immediate' mode * and resulting in 50-100x slower add() operations. @@ -6135,7 +6125,7 @@ export class Brainy implements BrainyInterface { * - 10x faster type-specific queries * - Automatic type routing * - * v6.2.8: Smart defaults for HNSW persistence mode + * Smart defaults for HNSW persistence mode * - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50Γ— faster adds * - Local storage (FileSystem/Memory/OPFS): 'immediate' (already fast) */ @@ -6145,13 +6135,13 @@ export class Brainy implements BrainyInterface { distanceFunction: this.distance } - // v6.2.8: Determine persist mode (user config > smart default) - // v7.1.1: Fixed to use getStorageType() for reliable detection + // Determine persist mode (user config > smart default) + // Fixed to use getStorageType() for reliable detection let persistMode: 'immediate' | 'deferred' = this.config.hnswPersistMode || 'immediate' // Smart default: Use deferred mode for cloud storage adapters if (!this.config.hnswPersistMode) { - // v7.1.1 FIX: Use instance-based detection as fallback + // FIX: Use instance-based detection as fallback // Previously this.config.storage.type was often undefined after storage creation, // causing cloud storage to incorrectly use 'immediate' mode (50-100x slower) const storageType = this.config.storage?.type || this.getStorageType() @@ -6262,20 +6252,20 @@ export class Brainy implements BrainyInterface { disableAutoOptimize: config?.disableAutoOptimize ?? false, batchWrites: config?.batchWrites ?? true, maxConcurrentOperations: config?.maxConcurrentOperations ?? 10, - // Memory management options (v5.11.0) + // Memory management options maxQueryLimit: config?.maxQueryLimit ?? undefined as any, reservedQueryMemory: config?.reservedQueryMemory ?? undefined as any, - // HNSW persistence mode (v6.2.8) - undefined = smart default in setupIndex + // HNSW persistence mode - undefined = smart default in setupIndex hnswPersistMode: config?.hnswPersistMode ?? undefined as any, - // Embedding initialization (v7.1.2) - false = lazy init on first embed() + // Embedding initialization - false = lazy init on first embed() eagerEmbeddings: config?.eagerEmbeddings ?? false, - // Integration Hub (v7.4.0) - undefined/false = disabled + // Integration Hub - undefined/false = disabled integrations: config?.integrations ?? undefined as any } } /** - * Ensure indexes are loaded (v5.7.7 - Production-scale lazy loading) + * Ensure indexes are loaded (Production-scale lazy loading) * * Called by query methods (find, search, get, etc.) when disableAutoRebuild is true. * Handles concurrent queries safely - multiple calls wait for same rebuild. @@ -6341,13 +6331,13 @@ export class Brainy implements BrainyInterface { } /** - * Rebuild indexes from persisted data if needed (v3.35.0+, v5.7.7 LAZY LOADING) + * Rebuild indexes from persisted data if needed (LAZY LOADING) * * FIXES FOR CRITICAL BUGS: * - Bug #1: GraphAdjacencyIndex rebuild never called βœ… FIXED * - Bug #2: Early return blocks recovery when count=0 βœ… FIXED * - Bug #4: HNSW index has no rebuild mechanism βœ… FIXED - * - Bug #5: disableAutoRebuild leaves indexes empty forever βœ… FIXED (v5.7.7) + * - Bug #5: disableAutoRebuild leaves indexes empty forever βœ… FIXED * * Production-grade rebuild with: * - Handles BILLIONS of entities via streaming pagination @@ -6362,7 +6352,7 @@ export class Brainy implements BrainyInterface { */ private async rebuildIndexesIfNeeded(force = false): Promise { try { - // v5.7.7: Check if auto-rebuild is explicitly disabled (ONLY during init, not for lazy loading) + // Check if auto-rebuild is explicitly disabled (ONLY during init, not for lazy loading) // force=true means this is a lazy rebuild triggered by first query if (this.config.disableAutoRebuild === true && !force) { if (!this.config.silent) { @@ -6398,7 +6388,7 @@ export class Brainy implements BrainyInterface { // Intelligent decision: Auto-rebuild based on dataset size // Production scale: Handles billions via streaming pagination - const AUTO_REBUILD_THRESHOLD = 10000 // Auto-rebuild if < 10K items (v5.7.7: increased from 1K) + const AUTO_REBUILD_THRESHOLD = 10000 // Auto-rebuild if < 10K items (increased from 1K) // Check if indexes need rebuilding const metadataStats = await this.metadataIndex.getStats() @@ -6415,7 +6405,7 @@ export class Brainy implements BrainyInterface { return } - // v5.7.7: Determine rebuild strategy + // Determine rebuild strategy const isLazyRebuild = force && this.config.disableAutoRebuild === true const isSmallDataset = totalCount < AUTO_REBUILD_THRESHOLD const shouldRebuild = isLazyRebuild || isSmallDataset || this.config.disableAutoRebuild === false @@ -6469,11 +6459,11 @@ export class Brainy implements BrainyInterface { /** * Close and cleanup * - * v6.2.8: Now flushes HNSW dirty nodes before closing + * Now flushes HNSW dirty nodes before closing * This ensures deferred persistence mode data is saved */ async close(): Promise { - // v6.2.8: Flush HNSW dirty nodes before closing + // Flush HNSW dirty nodes before closing // In deferred persistence mode, this persists all pending HNSW graph data if (this.index && typeof this.index.flush === 'function') { await this.index.flush() diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index bb754946..b5f14632 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -83,11 +83,12 @@ export interface Result { // Score transparency explanation?: ScoreExplanation - // Match visibility (v7.8.0) - shows what matched in hybrid search + // Match visibility - shows what matched in hybrid search textMatches?: string[] // Query words found in entity (e.g., ["david", "warrior"]) textScore?: number // Normalized text match score (0-1) semanticScore?: number // Semantic similarity score (0-1) matchSource?: 'text' | 'semantic' | 'both' // Where this result came from + rrfScore?: number // Raw RRF fusion score (for advanced ranking analysis) } /** @@ -188,7 +189,7 @@ export interface FindParams { offset?: number // Skip N results cursor?: string // Cursor-based pagination - // Sorting (v4.5.4) + // Sorting orderBy?: string // Field to sort by (e.g., 'createdAt', 'title', 'metadata.priority') order?: 'asc' | 'desc' // Sort direction: 'asc' (default) or 'desc' @@ -196,10 +197,10 @@ export interface FindParams { mode?: SearchMode // Search strategy explain?: boolean // Return scoring explanation includeRelations?: boolean // Include entity relationships - excludeVFS?: boolean // v4.7.0: Exclude VFS entities from results (default: false - VFS included) + excludeVFS?: boolean // Exclude VFS entities from results (default: false - VFS included) service?: string // Multi-tenancy filter - // Hybrid search options (v7.7.0) + // Hybrid search options searchMode?: 'auto' | 'text' | 'semantic' | 'vector' | 'hybrid' // Search strategy: auto (default), text-only, semantic/vector-only, or explicit hybrid hybridAlpha?: number // Weight between text (0.0) and semantic (1.0) search, default: auto-detected by query length @@ -252,7 +253,7 @@ export interface SimilarParams { type?: NounType | NounType[] // Restrict to types where?: Partial // Additional filters service?: string // Multi-tenancy - excludeVFS?: boolean // v4.7.0: Exclude VFS entities (default: false - VFS included) + excludeVFS?: boolean // Exclude VFS entities (default: false - VFS included) } /** @@ -289,8 +290,8 @@ export interface SimilarParams { * }) * ``` * - * @since v4.1.3 - Fixed bug where calling without parameters returned empty array - * @since v4.1.3 - Added string ID shorthand syntax + * Fixed bug where calling without parameters returned empty array + * Added string ID shorthand syntax */ export interface GetRelationsParams { /** @@ -376,7 +377,7 @@ export interface DeleteManyParams { where?: any // Delete by metadata limit?: number // Max to delete (safety) onProgress?: (done: number, total: number) => void - continueOnError?: boolean // v6.2.0: Continue processing if a delete fails + continueOnError?: boolean // Continue processing if a delete fails } /** @@ -403,7 +404,7 @@ export interface BatchResult { duration: number // Time taken in ms } -// ============= Import Progress (v4.5.0) ============= +// ============= Import Progress ============= /** * Import stage enumeration @@ -435,7 +436,6 @@ export type ImportStatus = * - Time estimates * - Performance metrics * - * @since v4.5.0 */ export interface ImportProgress { // Overall Progress @@ -540,7 +540,7 @@ export interface ImportResult { /** * Options for brain.get() entity retrieval * - * **Performance Optimization (v5.11.1)**: + * **Performance Optimization**: * By default, brain.get() loads ONLY metadata (not vectors), resulting in: * - **76-81% faster** reads (10ms vs 43ms for metadata-only) * - **95% less bandwidth** (300 bytes vs 6KB per entity) @@ -573,7 +573,6 @@ export interface ImportResult { * await vfs.readFile('/file.txt') // 53ms β†’ 10ms (81% faster) * ``` * - * @since v5.11.1 */ export interface GetOptions { /** @@ -635,7 +634,7 @@ export type AggregateMetric = // ============= Configuration ============= /** - * Integration Hub configuration (v7.4.0) + * Integration Hub configuration * * Enables external tool integrations: Excel, Power BI, Google Sheets, etc. * Works in all environments with zero external dependencies. @@ -722,18 +721,18 @@ export interface BrainyConfig { batchWrites?: boolean // Enable write batching for better performance maxConcurrentOperations?: number // Limit concurrent file operations - // HNSW persistence mode (v6.2.8) + // HNSW persistence mode // Controls when HNSW graph connections are persisted to storage // - 'immediate': Persist on every add (slow but durable, default for filesystem) // - 'deferred': Persist only on flush/close (fast, default for cloud storage) // Cloud storage (GCS/S3/R2/Azure) should use 'deferred' for 30-50Γ— faster adds hnswPersistMode?: 'immediate' | 'deferred' - // Memory management options (v5.11.0) + // Memory management options maxQueryLimit?: number // Override auto-detected query result limit (max: 100000) reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB) - // Embedding initialization (v7.1.2) + // Embedding initialization // Controls when the WASM embedding engine is initialized // - false (default): Lazy initialization on first embed() call // - true: Eager initialization during brain.init() @@ -745,7 +744,7 @@ export interface BrainyConfig { verbose?: boolean // Enable verbose logging silent?: boolean // Suppress all logging output - // Integration Hub (v7.4.0) + // Integration Hub // Enable external tool integrations: Excel, Power BI, Google Sheets, etc. // - true: Enable all integrations with default paths // - false/undefined: Disable integrations (default) @@ -789,54 +788,55 @@ export interface NeuralAnomalyParams { returnScores?: boolean // Return anomaly scores } -// ============= Content Extraction Types (v7.9.0) ============= +// ============= Content Extraction Types ============= /** * Detected content type for smart text extraction * - * @since v7.9.0 */ export type ContentType = 'plaintext' | 'richtext-json' | 'html' | 'markdown' /** * Content category for extracted segments * - * Allows UI to apply different styling based on content role: - * - 'prose': Regular text content (paragraphs, list items) - * - 'heading': Heading/title text (h1-h6) - * - 'code': Code blocks or inline code - * - 'label': Labels, captions, or metadata-like text + * Universal categories that work across documents, code, and UI: + * - 'title': Names the subject β€” headings, identifiers, labels, JSON keys + * - 'annotation': Human explanation β€” comments, docstrings, captions, alt text + * - 'content': Body substance β€” paragraphs, list items, flowing text + * - 'value': Data literals β€” strings, numbers, form values, error messages + * - 'code': Unparsed code blocks (custom parsers decompose into above) + * - 'structural': Boilerplate β€” keywords, operators, punctuation, formatting * - * @since v7.9.0 + * Built-in extractors produce: 'title', 'content', 'code'. + * All 6 categories are available for custom parsers. */ -export type ContentCategory = 'prose' | 'heading' | 'code' | 'label' +export type ContentCategory = 'title' | 'annotation' | 'content' | 'value' | 'code' | 'structural' /** * A segment of extracted text with its content category * - * @since v7.9.0 */ export interface ExtractedSegment { /** The extracted text content */ text: string - /** What role this text plays in the document */ + /** What role this text plays: 'title' | 'annotation' | 'content' | 'value' | 'code' | 'structural' */ contentCategory: ContentCategory } -// ============= Semantic Highlighting Types (v7.8.0) ============= +// ============= Semantic Highlighting Types ============= /** - * Parameters for hybrid highlighting (v7.8.0, enhanced v7.9.0) + * Parameters for hybrid highlighting * * Zero-config highlighting that returns both text (exact) and semantic (concept) matches. * Perfect for UI highlighting at different levels. * - * v7.9.0: Added contentType hint and contentExtractor callback for structured text + * Added contentType hint and contentExtractor callback for structured text * (rich-text JSON, HTML, Markdown). Auto-detects format when not specified. * * @example * ```typescript - * // Plain text (unchanged from v7.8.0) + * // Plain text * const highlights = await brain.highlight({ * query: "david the warrior", * text: "David Smith is a brave fighter who battles dragons" @@ -872,7 +872,6 @@ export interface HighlightParams { /** * Optional content type hint to skip auto-detection. * When omitted, the content type is detected from the text content. - * @since v7.9.0 */ contentType?: ContentType @@ -880,7 +879,6 @@ export interface HighlightParams { * Optional custom content extractor function. * When provided, bypasses built-in detection and extraction entirely. * Use this to plug in custom parsers (tree-sitter, Monaco, proprietary formats). - * @since v7.9.0 */ contentExtractor?: (text: string) => ExtractedSegment[] } @@ -906,9 +904,8 @@ export interface Highlight { matchType: 'text' | 'semantic' /** - * Content category of the source segment (heading, code, prose, label). + * Content category of the source segment: 'title' | 'annotation' | 'content' | 'value' | 'code' | 'structural'. * Present when the input text was structured (JSON, HTML, Markdown). - * @since v7.9.0 */ contentCategory?: ContentCategory } diff --git a/src/utils/contentExtractor.ts b/src/utils/contentExtractor.ts index f0d19b08..3eee5f0a 100644 --- a/src/utils/contentExtractor.ts +++ b/src/utils/contentExtractor.ts @@ -1,8 +1,8 @@ /** - * Content Extractor (v7.9.0) + * Content Extractor * * Detects content type (plaintext, rich-text JSON, HTML, Markdown) and extracts - * text segments with content categories (prose, heading, code, label). + * text segments with content categories (title, content, code, etc.). * * Supports common rich-text editor formats: * - TipTap / ProseMirror: { type: 'doc', content: [...] } @@ -12,7 +12,7 @@ * - Quill Delta: { ops: [{ insert }] } * * Falls back gracefully: structured text that doesn't match known patterns - * is extracted as plain prose via recursive text collection. + * is extracted as plain content via recursive text collection. */ import type { ContentType, ContentCategory, ExtractedSegment } from '../types/brainy.types.js' @@ -65,7 +65,7 @@ export function extractForHighlighting( return extractFromMarkdown(text) case 'plaintext': default: - return [{ text, contentCategory: 'prose' }] + return [{ text, contentCategory: 'content' }] } } @@ -81,7 +81,7 @@ function extractFromJson(text: string): ExtractedSegment[] { parsed = JSON.parse(text.trim()) } catch { // Invalid JSON β€” treat as plain text - return [{ text, contentCategory: 'prose' }] + return [{ text, contentCategory: 'content' }] } const segments = walkRichTextNodes(parsed) @@ -94,7 +94,7 @@ function extractFromJson(text: string): ExtractedSegment[] { // Fallback: collect all string values from the JSON const fallbackText = extractTextFromJsonValue(parsed) if (fallbackText.trim()) { - return [{ text: fallbackText, contentCategory: 'prose' }] + return [{ text: fallbackText, contentCategory: 'content' }] } return [] @@ -121,13 +121,13 @@ function walkRichTextNodes(node: any): ExtractedSegment[] { // Leaf: text content (TipTap/ProseMirror/Lexical/Slate) if (typeof node.text === 'string' && node.text.length > 0) { - segments.push({ text: node.text, contentCategory: 'prose' }) + segments.push({ text: node.text, contentCategory: 'content' }) return segments } // Leaf: Quill Delta insert if (typeof node.insert === 'string' && node.insert.length > 0) { - segments.push({ text: node.insert, contentCategory: 'prose' }) + segments.push({ text: node.insert, contentCategory: 'content' }) return segments } @@ -146,8 +146,8 @@ function walkRichTextNodes(node: any): ExtractedSegment[] { if (Array.isArray(children)) { for (const child of children) { const childSegments = walkRichTextNodes(child) - // Apply parent's category if it's more specific than 'prose' - if (category !== 'prose') { + // Apply parent's category if it's more specific than 'content' + if (category !== 'content') { childSegments.forEach(s => { s.contentCategory = category }) } segments.push(...childSegments) @@ -166,11 +166,11 @@ function walkRichTextNodes(node: any): ExtractedSegment[] { * Categorize a node type string into a ContentCategory */ function categorizeNodeType(type?: string): ContentCategory { - if (!type) return 'prose' + if (!type) return 'content' const t = type.toLowerCase() - if (t === 'heading' || /^h[1-6]$/.test(t)) return 'heading' + if (t === 'heading' || /^h[1-6]$/.test(t)) return 'title' if (t === 'code' || t === 'codeblock' || t === 'code_block') return 'code' - return 'prose' + return 'content' } /** @@ -201,7 +201,7 @@ function extractTextFromJsonValue(value: any): string { function extractFromHtml(html: string): ExtractedSegment[] { const segments: ExtractedSegment[] = [] let current = '' - let currentCategory: ContentCategory = 'prose' + let currentCategory: ContentCategory = 'content' let i = 0 let insideTag = false let tagName = '' @@ -304,10 +304,10 @@ function extractFromHtml(html: string): ExtractedSegment[] { function getCategoryFromTagStack(stack: string[]): ContentCategory { for (let i = stack.length - 1; i >= 0; i--) { const tag = stack[i] - if (/^h[1-6]$/.test(tag)) return 'heading' + if (/^h[1-6]$/.test(tag)) return 'title' if (tag === 'code' || tag === 'pre') return 'code' } - return 'prose' + return 'content' } /** @@ -328,6 +328,41 @@ function decodeHtmlEntity(entity: string): string { // ============= Markdown Extraction ============= +/** + * Split a prose line into alternating content/code segments based on + * backtick-delimited inline code spans. Lines without backticks return + * a single 'content' segment. + */ +function splitInlineCode(line: string): ExtractedSegment[] { + const segments: ExtractedSegment[] = [] + const pattern = /`([^`]+)`/g + let lastIndex = 0 + let match: RegExpExecArray | null + + while ((match = pattern.exec(line)) !== null) { + // Text before the backtick span + if (match.index > lastIndex) { + const before = line.substring(lastIndex, match.index).trim() + if (before.length > 0) { + segments.push({ text: before, contentCategory: 'content' }) + } + } + // The code span (without backticks) + segments.push({ text: match[1], contentCategory: 'code' }) + lastIndex = match.index + match[0].length + } + + // Remaining text after last backtick span (or entire line if no backticks) + if (lastIndex < line.length) { + const remaining = line.substring(lastIndex).trim() + if (remaining.length > 0) { + segments.push({ text: remaining, contentCategory: 'content' }) + } + } + + return segments +} + /** * Extract text from Markdown with category detection */ @@ -381,14 +416,14 @@ function extractFromMarkdown(text: string): ExtractedSegment[] { // Heading (# style) const headingMatch = line.match(/^#{1,6}\s+(.+)$/) if (headingMatch) { - segments.push({ text: headingMatch[1].trim(), contentCategory: 'heading' }) + segments.push({ text: headingMatch[1].trim(), contentCategory: 'title' }) i++ continue } - // Regular prose line + // Regular content line β€” split out inline code spans if (line.trim().length > 0) { - segments.push({ text: line.trim(), contentCategory: 'prose' }) + segments.push(...splitInlineCode(line.trim())) } i++ diff --git a/tests/unit/brainy/hybrid-search.test.ts b/tests/unit/brainy/hybrid-search.test.ts index 2aa9cc57..ff4ea6b7 100644 --- a/tests/unit/brainy/hybrid-search.test.ts +++ b/tests/unit/brainy/hybrid-search.test.ts @@ -769,7 +769,7 @@ describe('Hybrid Search (v7.7.0)', () => { expect(warriorMatch).toBeDefined() expect(warriorMatch?.matchType).toBe('text') // Should annotate heading content - expect(warriorMatch?.contentCategory).toBe('heading') + expect(warriorMatch?.contentCategory).toBe('title') }) it('should extract text from Slate.js JSON', async () => { @@ -799,7 +799,7 @@ describe('Hybrid Search (v7.7.0)', () => { const introMatch = highlights.find(h => h.text === 'Introduction') expect(introMatch).toBeDefined() expect(introMatch?.matchType).toBe('text') - expect(introMatch?.contentCategory).toBe('heading') + expect(introMatch?.contentCategory).toBe('title') }) it('should extract text from Quill Delta JSON', async () => { @@ -856,12 +856,12 @@ describe('Hybrid Search (v7.7.0)', () => { // Should find "Warrior" from heading const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior') expect(warriorMatch).toBeDefined() - expect(warriorMatch?.contentCategory).toBe('heading') + expect(warriorMatch?.contentCategory).toBe('title') // Should find "fighter" from paragraph const fighterMatch = highlights.find(h => h.text.toLowerCase() === 'fighter') expect(fighterMatch).toBeDefined() - expect(fighterMatch?.contentCategory).toBe('prose') + expect(fighterMatch?.contentCategory).toBe('content') }) it('should extract text from Markdown with headings and code', async () => { @@ -876,8 +876,8 @@ describe('Hybrid Search (v7.7.0)', () => { expect(highlights.length).toBeGreaterThan(0) // Should find text from heading - const headingMatch = highlights.find(h => h.text === 'Guide' && h.contentCategory === 'heading') - || highlights.find(h => h.text === 'Warrior' && h.contentCategory === 'heading') + const headingMatch = highlights.find(h => h.text === 'Guide' && h.contentCategory === 'title') + || highlights.find(h => h.text === 'Warrior' && h.contentCategory === 'title') expect(headingMatch).toBeDefined() }) @@ -896,8 +896,8 @@ describe('Hybrid Search (v7.7.0)', () => { expect(warriorMatch).toBeDefined() expect(warriorMatch?.matchType).toBe('text') expect(warriorMatch?.score).toBe(1.0) - // Plain text gets 'prose' category - expect(warriorMatch?.contentCategory).toBe('prose') + // Plain text gets 'content' category + expect(warriorMatch?.contentCategory).toBe('content') }) it('should use contentType hint to skip auto-detection', async () => { @@ -920,7 +920,7 @@ describe('Hybrid Search (v7.7.0)', () => { it('should use custom contentExtractor when provided', async () => { const customExtractor = (text: string) => { return [ - { text: 'custom extracted warrior content', contentCategory: 'prose' as const }, + { text: 'custom extracted warrior content', contentCategory: 'content' as const }, { text: 'Code Section', contentCategory: 'code' as const } ] } @@ -937,8 +937,9 @@ describe('Hybrid Search (v7.7.0)', () => { const warriorMatch = highlights.find(h => h.text.toLowerCase() === 'warrior') expect(warriorMatch).toBeDefined() expect(warriorMatch?.matchType).toBe('text') - expect(warriorMatch?.contentCategory).toBe('prose') + expect(warriorMatch?.contentCategory).toBe('content') }) + }) describe('Timeout Protection (v7.9.0)', () => { diff --git a/tests/unit/utils/contentExtractor.test.ts b/tests/unit/utils/contentExtractor.test.ts index e1222129..d54e51fc 100644 --- a/tests/unit/utils/contentExtractor.test.ts +++ b/tests/unit/utils/contentExtractor.test.ts @@ -75,10 +75,10 @@ describe('Content Extractor (v7.9.0)', () => { expect(segments.length).toBe(3) expect(segments[0].text).toBe('My Heading') - expect(segments[0].contentCategory).toBe('heading') + expect(segments[0].contentCategory).toBe('title') expect(segments[1].text).toBe('Regular paragraph text') - expect(segments[1].contentCategory).toBe('prose') + expect(segments[1].contentCategory).toBe('content') expect(segments[2].text).toBe('const x = 1') expect(segments[2].contentCategory).toBe('code') @@ -124,9 +124,9 @@ describe('Content Extractor (v7.9.0)', () => { const segments = extractForHighlighting(doc) expect(segments.length).toBe(3) expect(segments[0].text).toBe('Slate Heading') - expect(segments[0].contentCategory).toBe('heading') + expect(segments[0].contentCategory).toBe('title') expect(segments[1].text).toBe('First part ') - expect(segments[1].contentCategory).toBe('prose') + expect(segments[1].contentCategory).toBe('content') }) }) @@ -150,9 +150,9 @@ describe('Content Extractor (v7.9.0)', () => { const segments = extractForHighlighting(doc) expect(segments.length).toBe(2) expect(segments[0].text).toBe('Lexical Heading') - expect(segments[0].contentCategory).toBe('heading') + expect(segments[0].contentCategory).toBe('title') expect(segments[1].text).toBe('Lexical body text') - expect(segments[1].contentCategory).toBe('prose') + expect(segments[1].contentCategory).toBe('content') }) }) @@ -223,11 +223,11 @@ describe('Content Extractor (v7.9.0)', () => { const headingSegment = segments.find(s => s.text === 'Title') expect(headingSegment).toBeDefined() - expect(headingSegment?.contentCategory).toBe('heading') + expect(headingSegment?.contentCategory).toBe('title') const bodySegment = segments.find(s => s.text === 'Body text here.') expect(bodySegment).toBeDefined() - expect(bodySegment?.contentCategory).toBe('prose') + expect(bodySegment?.contentCategory).toBe('content') const codeSegment = segments.find(s => s.text === 'let x = 1') expect(codeSegment).toBeDefined() @@ -257,7 +257,7 @@ describe('Content Extractor (v7.9.0)', () => { const segments = extractForHighlighting(html) const heading = segments.find(s => s.text === 'Nested Heading') expect(heading).toBeDefined() - expect(heading?.contentCategory).toBe('heading') + expect(heading?.contentCategory).toBe('title') }) }) @@ -271,11 +271,11 @@ describe('Content Extractor (v7.9.0)', () => { const body = segments.find(s => s.text === 'Some body text') expect(heading1).toBeDefined() - expect(heading1?.contentCategory).toBe('heading') + expect(heading1?.contentCategory).toBe('title') expect(heading2).toBeDefined() - expect(heading2?.contentCategory).toBe('heading') + expect(heading2?.contentCategory).toBe('title') expect(body).toBeDefined() - expect(body?.contentCategory).toBe('prose') + expect(body?.contentCategory).toBe('content') }) it('should extract fenced code blocks', () => { @@ -296,16 +296,51 @@ describe('Content Extractor (v7.9.0)', () => { expect(code).toBeDefined() expect(code?.text).toContain('code line 1') }) + + it('should split inline code spans into separate segments', () => { + const md = '# Title\n\nUse the `authenticate()` function to verify users' + + const segments = extractForHighlighting(md) + // Should have: title segment, content "Use the", code "authenticate()", content "function to verify users" + const codeSegment = segments.find(s => s.text === 'authenticate()' && s.contentCategory === 'code') + expect(codeSegment).toBeDefined() + + const beforeCode = segments.find(s => s.text === 'Use the' && s.contentCategory === 'content') + expect(beforeCode).toBeDefined() + + const afterCode = segments.find(s => s.text === 'function to verify users' && s.contentCategory === 'content') + expect(afterCode).toBeDefined() + }) + + it('should handle multiple inline code spans in a single line', () => { + const md = '# API\n\nCall `foo()` then `bar()` for results' + + const segments = extractForHighlighting(md, 'markdown') + const fooSegment = segments.find(s => s.text === 'foo()' && s.contentCategory === 'code') + const barSegment = segments.find(s => s.text === 'bar()' && s.contentCategory === 'code') + expect(fooSegment).toBeDefined() + expect(barSegment).toBeDefined() + }) + + it('should not split backticks inside fenced code blocks', () => { + const md = '# Title\n\n```\nconst x = `template`\n```' + + const segments = extractForHighlighting(md) + const code = segments.find(s => s.contentCategory === 'code') + expect(code).toBeDefined() + // The fenced block content should be intact, not split by backticks + expect(code?.text).toContain('const x = `template`') + }) }) describe('extractForHighlighting() β€” Plaintext', () => { - it('should return single prose segment for plain text', () => { + it('should return single content segment for plain text', () => { const text = 'Just some plain text content' const segments = extractForHighlighting(text) expect(segments.length).toBe(1) expect(segments[0].text).toBe(text) - expect(segments[0].contentCategory).toBe('prose') + expect(segments[0].contentCategory).toBe('content') }) it('should respect contentType override', () => { @@ -315,7 +350,7 @@ describe('Content Extractor (v7.9.0)', () => { expect(segments.length).toBe(1) expect(segments[0].text).toBe(html) // Raw HTML, not extracted - expect(segments[0].contentCategory).toBe('prose') + expect(segments[0].contentCategory).toBe('content') }) }) })