diff --git a/README.md b/README.md index 5dcd2e48..4d0c9b63 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,8 @@ GitHub Pages that showcases Brainy's main features. npm install @soulcraft/brainy ``` -TensorFlow.js packages are included as bundled dependencies and will be automatically installed without any additional configuration. +TensorFlow.js packages are included as bundled dependencies and will be automatically installed without any additional +configuration. ### Additional Packages @@ -84,7 +85,7 @@ REST API web service wrapper that provides HTTP endpoints for search operations Brainy uses a unified build that automatically adapts to your environment (Node.js, browser, or serverless): ```typescript -import { BrainyData, NounType, VerbType } from '@soulcraft/brainy' +import {BrainyData, NounType, VerbType} from '@soulcraft/brainy' // Create and initialize the database const db = new BrainyData() @@ -92,13 +93,13 @@ await db.init() // Add data (automatically converted to vectors) const catId = await db.add("Cats are independent pets", { - noun: NounType.Thing, - category: 'animal' + noun: NounType.Thing, + category: 'animal' }) const dogId = await db.add("Dogs are loyal companions", { - noun: NounType.Thing, - category: 'animal' + noun: NounType.Thing, + category: 'animal' }) // Search for similar items @@ -107,8 +108,8 @@ console.log(results) // Add a relationship between items await db.addVerb(catId, dogId, { - verb: VerbType.RelatedTo, - description: 'Both are common household pets' + verb: VerbType.RelatedTo, + description: 'Both are common household pets' }) ``` @@ -116,28 +117,30 @@ await db.addVerb(catId, dogId, { ```typescript // Standard import - automatically adapts to any environment -import { BrainyData } from '@soulcraft/brainy' +import {BrainyData} from '@soulcraft/brainy' // Minified version for production -import { BrainyData } from '@soulcraft/brainy/min' +import {BrainyData} from '@soulcraft/brainy/min' ``` -> **Note**: The CLI functionality is available as a separate package `@soulcraft/brainy-cli` to reduce the bundle size of the main package. Install it globally with `npm install -g @soulcraft/brainy-cli` to use the command-line interface. +> **Note**: The CLI functionality is available as a separate package `@soulcraft/brainy-cli` to reduce the bundle size +> of the main package. Install it globally with `npm install -g @soulcraft/brainy-cli` to use the command-line +> interface. ### Browser Usage ```html ``` @@ -292,13 +295,13 @@ The pipeline runs automatically when you: ```typescript // Add data (runs embedding → indexing → storage) -const id = await db.add("Your text data here", { metadata }) +const id = await db.add("Your text data here", {metadata}) // Search (runs embedding → similarity search) const results = await db.searchText("Your query here", 5) // Connect entities (runs graph construction → storage) -await db.addVerb(sourceId, targetId, { verb: VerbType.RelatedTo }) +await db.addVerb(sourceId, targetId, {verb: VerbType.RelatedTo}) ``` Using the CLI: @@ -423,7 +426,8 @@ brainy visualize --root --depth 3 ### Using the CLI in Your Code -The CLI functionality is available as a separate package `@soulcraft/brainy-cli`. If you need CLI functionality in your application, install the CLI package: +The CLI functionality is available as a separate package `@soulcraft/brainy-cli`. If you need CLI functionality in your +application, install the CLI package: ```bash npm install @soulcraft/brainy-cli @@ -481,7 +485,7 @@ const status = await db.status() const backupData = await db.backup() // Restore data into the database -const restoreResult = await db.restore(backupData, { clearExisting: true }) +const restoreResult = await db.restore(backupData, {clearExisting: true}) ``` ### Working with Nouns (Entities) @@ -489,25 +493,25 @@ const restoreResult = await db.restore(backupData, { clearExisting: true }) ```typescript // Add a noun (automatically vectorized) const id = await db.add(textOrVector, { - noun: NounType.Thing, - // other metadata... + noun: NounType.Thing, + // other metadata... }) // Add multiple nouns in parallel (with multithreading and batch embedding) const ids = await db.addBatch([ - { - vectorOrData: "First item to add", - metadata: { noun: NounType.Thing, category: 'example' } - }, - { - vectorOrData: "Second item to add", - metadata: { noun: NounType.Thing, category: 'example' } - }, - // More items... + { + vectorOrData: "First item to add", + metadata: {noun: NounType.Thing, category: 'example'} + }, + { + vectorOrData: "Second item to add", + metadata: {noun: NounType.Thing, category: 'example'} + }, + // More items... ], { - forceEmbed: false, - concurrency: 4, // Control the level of parallelism (default: 4) - batchSize: 50 // Control the number of items to process in a single batch (default: 50) + forceEmbed: false, + concurrency: 4, // Control the level of parallelism (default: 4) + batchSize: 50 // Control the number of items to process in a single batch (default: 50) }) // Retrieve a noun @@ -515,8 +519,8 @@ const noun = await db.get(id) // Update noun metadata await db.updateMetadata(id, { - noun: NounType.Thing, - // updated metadata... + noun: NounType.Thing, + // updated metadata... }) // Delete a noun @@ -535,8 +539,21 @@ const thingNouns = await db.searchByNounTypes([NounType.Thing], numResults) ```typescript // Add a relationship between nouns await db.addVerb(sourceId, targetId, { - verb: VerbType.RelatedTo, - // other metadata... + verb: VerbType.RelatedTo, + // other metadata... +}) + +// Add a relationship with auto-creation of missing nouns +// This is useful when the target noun might not exist yet +await db.addVerb(sourceId, targetId, { + verb: VerbType.RelatedTo, + // Enable auto-creation of missing nouns + autoCreateMissingNouns: true, + // Optional metadata for auto-created nouns + missingNounMetadata: { + noun: NounType.Concept, + description: 'Auto-created noun' + } }) // Get all relationships @@ -564,20 +581,20 @@ await db.deleteVerb(verbId) ```typescript import { - BrainyData, - createTensorFlowEmbeddingFunction, - createThreadedEmbeddingFunction + BrainyData, + createTensorFlowEmbeddingFunction, + createThreadedEmbeddingFunction } from '@soulcraft/brainy' // Use the standard TensorFlow Universal Sentence Encoder embedding function const db = new BrainyData({ - embeddingFunction: createTensorFlowEmbeddingFunction() + embeddingFunction: createTensorFlowEmbeddingFunction() }) await db.init() // Or use the threaded embedding function for better performance const threadedDb = new BrainyData({ - embeddingFunction: createThreadedEmbeddingFunction() + embeddingFunction: createThreadedEmbeddingFunction() }) await threadedDb.init() @@ -618,42 +635,42 @@ Brainy includes comprehensive multithreading support to improve performance acro 7. **Automatic Environment Detection**: Adapts to browser (Web Workers) and Node.js (Worker Threads) environments ```typescript -import { BrainyData, euclideanDistance } from '@soulcraft/brainy' +import {BrainyData, euclideanDistance} from '@soulcraft/brainy' // Configure with custom options const db = new BrainyData({ - // Use Euclidean distance instead of default cosine distance - distanceFunction: euclideanDistance, + // Use Euclidean distance instead of default cosine distance + distanceFunction: euclideanDistance, - // HNSW index configuration for search performance - hnsw: { - M: 16, // Max connections per noun - efConstruction: 200, // Construction candidate list size - efSearch: 50, // Search candidate list size - }, + // HNSW index configuration for search performance + hnsw: { + M: 16, // Max connections per noun + efConstruction: 200, // Construction candidate list size + efSearch: 50, // Search candidate list size + }, - // Performance optimization options - performance: { - useParallelization: true, // Enable multithreaded search operations - }, + // Performance optimization options + performance: { + useParallelization: true, // Enable multithreaded search operations + }, - // Noun and Verb type validation - typeValidation: { - enforceNounTypes: true, // Validate noun types against NounType enum - enforceVerbTypes: true, // Validate verb types against VerbType enum - }, + // Noun and Verb type validation + typeValidation: { + enforceNounTypes: true, // Validate noun types against NounType enum + enforceVerbTypes: true, // Validate verb types against VerbType enum + }, - // Storage configuration - storage: { - requestPersistentStorage: true, - // Example configuration for cloud storage (replace with your own values): - // s3Storage: { - // bucketName: 'your-s3-bucket-name', - // region: 'your-aws-region' - // // Credentials should be provided via environment variables - // // AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - // } - } + // Storage configuration + storage: { + requestPersistentStorage: true, + // Example configuration for cloud storage (replace with your own values): + // s3Storage: { + // bucketName: 'your-s3-bucket-name', + // region: 'your-aws-region' + // // Credentials should be provided via environment variables + // // AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + // } + } }) ``` @@ -667,34 +684,34 @@ hybrid approach: 3. **Memory-Efficient Indexing** - Optimizes memory usage for large-scale vector collections ```typescript -import { BrainyData } from '@soulcraft/brainy' +import {BrainyData} from '@soulcraft/brainy' // Configure with optimized HNSW index for large datasets const db = new BrainyData({ - hnswOptimized: { - // Standard HNSW parameters - M: 16, // Max connections per noun - efConstruction: 200, // Construction candidate list size - efSearch: 50, // Search candidate list size + hnswOptimized: { + // Standard HNSW parameters + M: 16, // Max connections per noun + efConstruction: 200, // Construction candidate list size + efSearch: 50, // Search candidate list size - // Memory threshold in bytes - when exceeded, will use disk-based approach - memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold + // Memory threshold in bytes - when exceeded, will use disk-based approach + memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold - // Product quantization settings for dimensionality reduction - productQuantization: { - enabled: true, // Enable product quantization - numSubvectors: 16, // Number of subvectors to split the vector into - numCentroids: 256 // Number of centroids per subvector + // Product quantization settings for dimensionality reduction + productQuantization: { + enabled: true, // Enable product quantization + numSubvectors: 16, // Number of subvectors to split the vector into + numCentroids: 256 // Number of centroids per subvector + }, + + // Whether to use disk-based storage for the index + useDiskBasedIndex: true // Enable disk-based storage }, - // Whether to use disk-based storage for the index - useDiskBasedIndex: true // Enable disk-based storage - }, - - // Storage configuration (required for disk-based index) - storage: { - requestPersistentStorage: true - } + // Storage configuration (required for disk-based index) + storage: { + requestPersistentStorage: true + } }) // The optimized index automatically adapts based on dataset size: @@ -759,24 +776,24 @@ Brainy's restore functionality can handle: ```typescript // Restore data with all options const restoreResult = await db.restore(backupData, { - clearExisting: true // Whether to clear existing data before restore + clearExisting: true // Whether to clear existing data before restore }) // Import sparse data (without vectors) // Vectors will be automatically created using the embedding function const sparseData = { - nouns: [ - { - id: '123', - // No vector field - will be created during import - metadata: { - noun: 'Thing', - text: 'This text will be used to generate a vector' - } - } - ], - verbs: [], - version: '1.0.0' + nouns: [ + { + id: '123', + // No vector field - will be created during import + metadata: { + noun: 'Thing', + text: 'This text will be used to generate a vector' + } + } + ], + verbs: [], + version: '1.0.0' } const sparseImportResult = await db.importSparseData(sparseData) @@ -823,82 +840,82 @@ boilerplate: ```typescript import { - createMemoryAugmentation, - createConduitAugmentation, - createSenseAugmentation, - addWebSocketSupport, - executeStreamlined, - processStaticData, - processStreamingData, - createPipeline + createMemoryAugmentation, + createConduitAugmentation, + createSenseAugmentation, + addWebSocketSupport, + executeStreamlined, + processStaticData, + processStreamingData, + createPipeline } from '@soulcraft/brainy' // Create a memory augmentation with minimal code const memoryAug = createMemoryAugmentation({ - name: 'simple-memory', - description: 'A simple in-memory storage augmentation', - autoRegister: true, - autoInitialize: true, + name: 'simple-memory', + description: 'A simple in-memory storage augmentation', + autoRegister: true, + autoInitialize: true, - // Implement only the methods you need - storeData: async (key, data) => { - // Your implementation here - return { - success: true, - data: true - } - }, + // Implement only the methods you need + storeData: async (key, data) => { + // Your implementation here + return { + success: true, + data: true + } + }, - retrieveData: async (key) => { - // Your implementation here - return { - success: true, - data: { example: 'data', key } + retrieveData: async (key) => { + // Your implementation here + return { + success: true, + data: {example: 'data', key} + } } - } }) // Add WebSocket support to any augmentation const wsAugmentation = addWebSocketSupport(memoryAug, { - connectWebSocket: async (url) => { - // Your implementation here - return { - connectionId: 'ws-1', - url, - status: 'connected' + connectWebSocket: async (url) => { + // Your implementation here + return { + connectionId: 'ws-1', + url, + status: 'connected' + } } - } }) // Process static data through a pipeline const result = await processStaticData( - 'Input data', - [ - { - augmentation: senseAug, - method: 'processRawData', - transformArgs: (data) => [data, 'text'] - }, - { - augmentation: memoryAug, - method: 'storeData', - transformArgs: (data) => ['processed-data', data] - } - ] + 'Input data', + [ + { + augmentation: senseAug, + method: 'processRawData', + transformArgs: (data) => [data, 'text'] + }, + { + augmentation: memoryAug, + method: 'storeData', + transformArgs: (data) => ['processed-data', data] + } + ] ) // Create a reusable pipeline const pipeline = createPipeline([ - { - augmentation: senseAug, - method: 'processRawData', - transformArgs: (data) => [data, 'text'] - }, - { - augmentation: memoryAug, - method: 'storeData', - transformArgs: (data) => ['processed-data', data] - } + { + augmentation: senseAug, + method: 'processRawData', + transformArgs: (data) => [data, 'text'] + }, + { + augmentation: memoryAug, + method: 'storeData', + transformArgs: (data) => ['processed-data', data] + } ]) // Use the pipeline @@ -906,11 +923,11 @@ const result = await pipeline('New input data') // Dynamically load augmentations at runtime const loadedAugmentations = await loadAugmentationModule( - import('./my-augmentations.js'), - { - autoRegister: true, - autoInitialize: true - } + import('./my-augmentations.js'), + { + autoRegister: true, + autoInitialize: true + } ) ``` @@ -924,60 +941,61 @@ The simplified augmentation system provides: #### WebSocket Augmentation Types -Brainy exports several WebSocket augmentation types that can be used by augmentation creators to add WebSocket capabilities to their augmentations: +Brainy exports several WebSocket augmentation types that can be used by augmentation creators to add WebSocket +capabilities to their augmentations: ```typescript import { - // Base WebSocket support interface - IWebSocketSupport, - - // Combined WebSocket augmentation types - IWebSocketSenseAugmentation, - IWebSocketConduitAugmentation, - IWebSocketCognitionAugmentation, - IWebSocketMemoryAugmentation, - IWebSocketPerceptionAugmentation, - IWebSocketDialogAugmentation, - IWebSocketActivationAugmentation, - - // Function to add WebSocket support to any augmentation - addWebSocketSupport + // Base WebSocket support interface + IWebSocketSupport, + + // Combined WebSocket augmentation types + IWebSocketSenseAugmentation, + IWebSocketConduitAugmentation, + IWebSocketCognitionAugmentation, + IWebSocketMemoryAugmentation, + IWebSocketPerceptionAugmentation, + IWebSocketDialogAugmentation, + IWebSocketActivationAugmentation, + + // Function to add WebSocket support to any augmentation + addWebSocketSupport } from '@soulcraft/brainy' // Example: Creating a typed WebSocket-enabled sense augmentation const mySenseAug = createSenseAugmentation({ - name: 'my-sense', - processRawData: async (data, dataType) => { - // Implementation - return { - success: true, - data: { nouns: [], verbs: [] } + name: 'my-sense', + processRawData: async (data, dataType) => { + // Implementation + return { + success: true, + data: {nouns: [], verbs: []} + } } - } }) as IWebSocketSenseAugmentation // Add WebSocket support addWebSocketSupport(mySenseAug, { - connectWebSocket: async (url) => { - // WebSocket implementation - return { - connectionId: 'ws-1', - url, - status: 'connected' + connectWebSocket: async (url) => { + // WebSocket implementation + return { + connectionId: 'ws-1', + url, + status: 'connected' + } + }, + sendWebSocketMessage: async (connectionId, data) => { + // Send message implementation + }, + onWebSocketMessage: async (connectionId, callback) => { + // Register callback implementation + }, + offWebSocketMessage: async (connectionId, callback) => { + // Remove callback implementation + }, + closeWebSocket: async (connectionId, code, reason) => { + // Close connection implementation } - }, - sendWebSocketMessage: async (connectionId, data) => { - // Send message implementation - }, - onWebSocketMessage: async (connectionId, callback) => { - // Register callback implementation - }, - offWebSocketMessage: async (connectionId, callback) => { - // Remove callback implementation - }, - closeWebSocket: async (connectionId, code, reason) => { - // Close connection implementation - } }) // Now mySenseAug has both sense augmentation methods and WebSocket methods @@ -985,7 +1003,8 @@ await mySenseAug.processRawData('data', 'text') await mySenseAug.connectWebSocket('wss://example.com') ``` -These WebSocket augmentation types combine the base augmentation interfaces with the `IWebSocketSupport` interface, providing type safety and autocompletion for augmentations with WebSocket capabilities. +These WebSocket augmentation types combine the base augmentation interfaces with the `IWebSocketSupport` interface, +providing type safety and autocompletion for augmentations with WebSocket capabilities. ### Model Control Protocol (MCP) @@ -1014,13 +1033,13 @@ everywhere. Brainy automatically detects the environment it's running in: ```typescript -import { environment } from '@soulcraft/brainy' +import {environment} from '@soulcraft/brainy' // Check which environment we're running in console.log(`Running in ${ - environment.isBrowser ? 'browser' : - environment.isNode ? 'Node.js' : - 'serverless/unknown' + environment.isBrowser ? 'browser' : + environment.isNode ? 'Node.js' : + 'serverless/unknown' } environment`) ``` @@ -1049,7 +1068,6 @@ Works in all modern browsers: For browsers without OPFS support, falls back to in-memory storage. - ## Related Projects - **[Cartographer](https://github.com/sodal-project/cartographer)** - A companion project that provides standardized @@ -1094,9 +1112,9 @@ You can use the conduit augmentations to sync Brainy instances: ```typescript import { - BrainyData, - pipeline, - createConduitAugmentation + BrainyData, + pipeline, + createConduitAugmentation } from '@soulcraft/brainy' // Create and initialize the database @@ -1112,36 +1130,36 @@ pipeline.register(wsConduit) // Connect to another Brainy instance (server or browser) // Replace the example URL below with your actual WebSocket server URL const connectionResult = await pipeline.executeConduitPipeline( - 'establishConnection', - ['wss://example-websocket-server.com/brainy-sync', { protocols: 'brainy-sync' }] + 'establishConnection', + ['wss://example-websocket-server.com/brainy-sync', {protocols: 'brainy-sync'}] ) if (connectionResult[0] && (await connectionResult[0]).success) { - const connection = (await connectionResult[0]).data + const connection = (await connectionResult[0]).data - // Read data from the remote instance - const readResult = await pipeline.executeConduitPipeline( - 'readData', - [{ connectionId: connection.connectionId, query: { type: 'getAllNouns' } }] - ) + // Read data from the remote instance + const readResult = await pipeline.executeConduitPipeline( + 'readData', + [{connectionId: connection.connectionId, query: {type: 'getAllNouns'}}] + ) - // Process and add the received data to the local instance - if (readResult[0] && (await readResult[0]).success) { - const remoteNouns = (await readResult[0]).data - for (const noun of remoteNouns) { - await db.add(noun.vector, noun.metadata) + // Process and add the received data to the local instance + if (readResult[0] && (await readResult[0]).success) { + const remoteNouns = (await readResult[0]).data + for (const noun of remoteNouns) { + await db.add(noun.vector, noun.metadata) + } } - } - // Set up real-time sync by monitoring the stream - await wsConduit.monitorStream(connection.connectionId, async (data) => { - // Handle incoming data (e.g., new nouns, verbs, updates) - if (data.type === 'newNoun') { - await db.add(data.vector, data.metadata) - } else if (data.type === 'newVerb') { - await db.addVerb(data.sourceId, data.targetId, data.vector, data.options) - } - }) + // Set up real-time sync by monitoring the stream + await wsConduit.monitorStream(connection.connectionId, async (data) => { + // Handle incoming data (e.g., new nouns, verbs, updates) + if (data.type === 'newNoun') { + await db.add(data.vector, data.metadata) + } else if (data.type === 'newVerb') { + await db.addVerb(data.sourceId, data.targetId, data.vector, data.options) + } + }) } ``` @@ -1149,9 +1167,9 @@ if (connectionResult[0] && (await connectionResult[0]).success) { ```typescript import { - BrainyData, - pipeline, - createConduitAugmentation + BrainyData, + pipeline, + createConduitAugmentation } from '@soulcraft/brainy' // Create and initialize the database @@ -1167,48 +1185,48 @@ pipeline.register(webrtcConduit) // Connect to a peer using a signaling server // Replace the example values below with your actual configuration const connectionResult = await pipeline.executeConduitPipeline( - 'establishConnection', - [ - 'peer-id-to-connect-to', // Replace with actual peer ID - { - signalServerUrl: 'wss://example-signal-server.com', // Replace with your signal server - localPeerId: 'my-local-peer-id', // Replace with your local peer ID - iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] // Public STUN server - } - ] + 'establishConnection', + [ + 'peer-id-to-connect-to', // Replace with actual peer ID + { + signalServerUrl: 'wss://example-signal-server.com', // Replace with your signal server + localPeerId: 'my-local-peer-id', // Replace with your local peer ID + iceServers: [{urls: 'stun:stun.l.google.com:19302'}] // Public STUN server + } + ] ) if (connectionResult[0] && (await connectionResult[0]).success) { - const connection = (await connectionResult[0]).data + const connection = (await connectionResult[0]).data - // Set up real-time sync by monitoring the stream - await webrtcConduit.monitorStream(connection.connectionId, async (data) => { - // Handle incoming data (e.g., new nouns, verbs, updates) - if (data.type === 'newNoun') { - await db.add(data.vector, data.metadata) - } else if (data.type === 'newVerb') { - await db.addVerb(data.sourceId, data.targetId, data.vector, data.options) - } - }) - - // When adding new data locally, also send to the peer - const nounId = await db.add("New data to sync", { noun: "Thing" }) - - // Send the new noun to the peer - await pipeline.executeConduitPipeline( - 'writeData', - [ - { - connectionId: connection.connectionId, - data: { - type: 'newNoun', - id: nounId, - vector: (await db.get(nounId)).vector, - metadata: (await db.get(nounId)).metadata + // Set up real-time sync by monitoring the stream + await webrtcConduit.monitorStream(connection.connectionId, async (data) => { + // Handle incoming data (e.g., new nouns, verbs, updates) + if (data.type === 'newNoun') { + await db.add(data.vector, data.metadata) + } else if (data.type === 'newVerb') { + await db.addVerb(data.sourceId, data.targetId, data.vector, data.options) } - } - ] - ) + }) + + // When adding new data locally, also send to the peer + const nounId = await db.add("New data to sync", {noun: "Thing"}) + + // Send the new noun to the peer + await pipeline.executeConduitPipeline( + 'writeData', + [ + { + connectionId: connection.connectionId, + data: { + type: 'newNoun', + id: nounId, + vector: (await db.get(nounId)).vector, + metadata: (await db.get(nounId)).metadata + } + } + ] + ) } ``` @@ -1218,39 +1236,39 @@ Brainy supports searching a server-hosted instance from a browser, storing resul searches against the local instance: ```typescript -import { BrainyData } from '@soulcraft/brainy' +import {BrainyData} from '@soulcraft/brainy' // Create and initialize the database with remote server configuration // Replace the example URL below with your actual Brainy server URL const db = new BrainyData({ - remoteServer: { - url: 'wss://example-brainy-server.com/ws', // Replace with your server URL - protocols: 'brainy-sync', - autoConnect: true // Connect automatically during initialization - } + remoteServer: { + url: 'wss://example-brainy-server.com/ws', // Replace with your server URL + protocols: 'brainy-sync', + autoConnect: true // Connect automatically during initialization + } }) await db.init() // Or connect manually after initialization if (!db.isConnectedToRemoteServer()) { - // Replace the example URL below with your actual Brainy server URL - await db.connectToRemoteServer('wss://example-brainy-server.com/ws', 'brainy-sync') + // Replace the example URL below with your actual Brainy server URL + await db.connectToRemoteServer('wss://example-brainy-server.com/ws', 'brainy-sync') } // Search the remote server (results are stored locally) -const remoteResults = await db.searchText('machine learning', 5, { searchMode: 'remote' }) +const remoteResults = await db.searchText('machine learning', 5, {searchMode: 'remote'}) // Search the local database (includes previously stored results) -const localResults = await db.searchText('machine learning', 5, { searchMode: 'local' }) +const localResults = await db.searchText('machine learning', 5, {searchMode: 'local'}) // Perform a combined search (local first, then remote if needed) -const combinedResults = await db.searchText('neural networks', 5, { searchMode: 'combined' }) +const combinedResults = await db.searchText('neural networks', 5, {searchMode: 'combined'}) // Add data to both local and remote instances const id = await db.addToBoth('Deep learning is a subset of machine learning', { - noun: 'Concept', - category: 'AI', - tags: ['deep learning', 'neural networks'] + noun: 'Concept', + category: 'AI', + tags: ['deep learning', 'neural networks'] }) // Clean up when done (this also cleans up worker pools) diff --git a/cli-package/package-lock.json b/cli-package/package-lock.json index 682064b7..3126c3af 100644 --- a/cli-package/package-lock.json +++ b/cli-package/package-lock.json @@ -1,16 +1,16 @@ { "name": "@soulcraft/brainy-cli", - "version": "0.17.0", + "version": "0.18.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy-cli", - "version": "0.17.0", + "version": "0.18.0", "hasInstallScript": true, "license": "MIT", "dependencies": { - "@soulcraft/brainy": "0.12.0", + "@soulcraft/brainy": "^0.17.0", "commander": "^14.0.0", "omelette": "^0.4.17" }, @@ -2087,10 +2087,9 @@ } }, "node_modules/@soulcraft/brainy": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.12.0.tgz", - "integrity": "sha512-Wka8OAw0TQTYZqSSFD+t8cRDOr5l6OblPuZspW54theHn9HwohUtpoHzqwcvYfs5qCF6u0zC7YHhBQtlbrdkIg==", - "hasInstallScript": true, + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@soulcraft/brainy/-/brainy-0.17.0.tgz", + "integrity": "sha512-QoTiNnyK7fBuNRDsGOcP89UrFfwxoqkS6lRh6VpIXAWVTtxUBy256PWxQAqtZgpslOZRAA58GlXIQf3IYHiM5g==", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/cli-package/package.json b/cli-package/package.json index 5a0c42e5..049f8597 100644 --- a/cli-package/package.json +++ b/cli-package/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy-cli", - "version": "0.17.0", + "version": "0.18.0", "description": "Command-line interface for the Brainy vector graph database", "type": "module", "bin": { @@ -46,7 +46,7 @@ "url": "git+https://github.com/soulcraft-research/brainy.git" }, "dependencies": { - "@soulcraft/brainy": "^0.15.0", + "@soulcraft/brainy": "^0.17.0", "commander": "^14.0.0", "omelette": "^0.4.17" }, diff --git a/package-lock.json b/package-lock.json index 2b7268e3..cbcab5d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "0.16.0", + "version": "0.17.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "0.16.0", + "version": "0.17.0", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 2e3d3abe..54b53e90 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "0.16.0", + "version": "0.17.0", "description": "A vector graph database using HNSW indexing with Origin Private File System storage", "main": "dist/unified.js", "module": "dist/unified.js", diff --git a/src/brainyData.ts b/src/brainyData.ts index acd8e930..d3d80da0 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -335,13 +335,13 @@ export class BrainyData implements BrainyDataInterface { // Ensure s3Storage has all required fields if it's provided if (storageOptions.s3Storage) { // Only include s3Storage if all required fields are present - if (storageOptions.s3Storage.bucketName && - storageOptions.s3Storage.accessKeyId && + if (storageOptions.s3Storage.bucketName && + storageOptions.s3Storage.accessKeyId && storageOptions.s3Storage.secretAccessKey) { // All required fields are present, keep s3Storage as is } else { // Missing required fields, remove s3Storage to avoid type errors - const { s3Storage, ...rest } = storageOptions + const {s3Storage, ...rest} = storageOptions storageOptions = rest console.warn('Ignoring s3Storage configuration due to missing required fields') } @@ -1253,6 +1253,22 @@ export class BrainyData implements BrainyDataInterface { /** * Add a verb between two nouns * If metadata is provided and vector is not, the metadata will be vectorized using the embedding function + * + * @param sourceId ID of the source noun + * @param targetId ID of the target noun + * @param vector Optional vector for the verb + * @param options Additional options: + * - type: Type of the verb + * - weight: Weight of the verb + * - metadata: Metadata for the verb + * - forceEmbed: Force using the embedding function for metadata even if vector is provided + * - id: Optional ID to use instead of generating a new one + * - autoCreateMissingNouns: Automatically create missing nouns if they don't exist + * - missingNounMetadata: Metadata to use when auto-creating missing nouns + * + * @returns The ID of the added verb + * + * @throws Error if source or target nouns don't exist and autoCreateMissingNouns is false or auto-creation fails */ public async addVerb( sourceId: string, @@ -1264,6 +1280,8 @@ export class BrainyData implements BrainyDataInterface { metadata?: any forceEmbed?: boolean // Force using the embedding function for metadata even if vector is provided id?: string // Optional ID to use instead of generating a new one + autoCreateMissingNouns?: boolean // Automatically create missing nouns + missingNounMetadata?: any // Metadata to use when auto-creating missing nouns } = {} ): Promise { await this.ensureInitialized() @@ -1273,8 +1291,59 @@ export class BrainyData implements BrainyDataInterface { try { // Check if source and target nouns exist - const sourceNoun = this.index.getNouns().get(sourceId) - const targetNoun = this.index.getNouns().get(targetId) + let sourceNoun = this.index.getNouns().get(sourceId) + let targetNoun = this.index.getNouns().get(targetId) + + // Auto-create missing nouns if option is enabled + if (!sourceNoun && options.autoCreateMissingNouns) { + try { + // Create a placeholder vector for the missing noun + const placeholderVector = new Array(this._dimensions).fill(0) + + // Add metadata if provided + const metadata = options.missingNounMetadata || { + autoCreated: true, + createdAt: new Date().toISOString(), + noun: NounType.Concept + } + + // Add the missing noun + await this.add(placeholderVector, metadata, {id: sourceId}) + + // Get the newly created noun + sourceNoun = this.index.getNouns().get(sourceId) + + console.warn(`Auto-created missing source noun with ID ${sourceId}`) + } catch (createError) { + console.error(`Failed to auto-create source noun with ID ${sourceId}:`, createError) + throw new Error(`Failed to auto-create source noun with ID ${sourceId}: ${createError}`) + } + } + + if (!targetNoun && options.autoCreateMissingNouns) { + try { + // Create a placeholder vector for the missing noun + const placeholderVector = new Array(this._dimensions).fill(0) + + // Add metadata if provided + const metadata = options.missingNounMetadata || { + autoCreated: true, + createdAt: new Date().toISOString(), + noun: NounType.Concept + } + + // Add the missing noun + await this.add(placeholderVector, metadata, {id: targetId}) + + // Get the newly created noun + targetNoun = this.index.getNouns().get(targetId) + + console.warn(`Auto-created missing target noun with ID ${targetId}`) + } catch (createError) { + console.error(`Failed to auto-create target noun with ID ${targetId}:`, createError) + throw new Error(`Failed to auto-create target noun with ID ${targetId}: ${createError}`) + } + } if (!sourceNoun) { throw new Error(`Source noun with ID ${sourceId} not found`)