From 5d82db12e203590c1a8d79d9bc4a893d18574627 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 30 Jul 2025 13:32:30 -0700 Subject: [PATCH] **feat: add write-only mode support and example usage** - Introduced `writeOnly` mode in `BrainyData` allowing optimized data ingestion by skipping index loading and disabling search operations. - Enhanced `BrainyDataConfig` to include `writeOnly` support and validate compatibility with `readOnly` mode. - Implemented error handling for search attempts during `writeOnly` mode. - Updated the README with detailed usage examples for database modes, including `writeOnly` and `readOnly`. - Added `examples/write-only-mode.js` to demonstrate practical applications of the `writeOnly` mode. **Purpose**: Optimize memory usage and startup time for data ingestion scenarios by enabling write-only mode with comprehensive documentation and examples. --- README.md | 31 + examples/write-only-mode.js | 68 + src/brainyData.ts | 7023 ++++++++++++++++++----------------- 3 files changed, 3696 insertions(+), 3426 deletions(-) create mode 100644 examples/write-only-mode.js diff --git a/README.md b/README.md index 84571469..75bb2475 100644 --- a/README.md +++ b/README.md @@ -598,6 +598,37 @@ await db.deleteVerb(verbId) ## Advanced Configuration +### Database Modes + +Brainy supports special operational modes that restrict certain operations: + +```typescript +import { BrainyData } from '@soulcraft/brainy' + +// Create and initialize the database +const db = new BrainyData() +await db.init() + +// Set the database to read-only mode (prevents write operations) +db.setReadOnly(true) + +// Check if the database is in read-only mode +const isReadOnly = db.isReadOnly() // Returns true + +// Set the database to write-only mode (prevents search operations) +db.setWriteOnly(true) + +// Check if the database is in write-only mode +const isWriteOnly = db.isWriteOnly() // Returns true + +// Reset to normal mode (allows both read and write operations) +db.setReadOnly(false) +db.setWriteOnly(false) +``` + +- **Read-Only Mode**: When enabled, prevents all write operations (add, update, delete). Useful for deployment scenarios where you want to prevent modifications to the database. +- **Write-Only Mode**: When enabled, prevents all search operations. Useful for initial data loading or when you want to optimize for write performance. + ### Embedding ```typescript diff --git a/examples/write-only-mode.js b/examples/write-only-mode.js new file mode 100644 index 00000000..5a9468d5 --- /dev/null +++ b/examples/write-only-mode.js @@ -0,0 +1,68 @@ +/** + * Example demonstrating the write-only mode in BrainyData + * + * This mode is useful for data ingestion scenarios where you only need to insert data + * and don't need to perform any search operations. It significantly reduces memory usage + * by not loading the index into memory. + */ + +import { BrainyData } from '../dist/index.js' + +// eslint-disable-next-line no-undef +const console = globalThis.console + +async function demonstrateWriteOnlyMode() { + console.log('Demonstrating write-only mode in BrainyData') + + // Create a BrainyData instance in write-only mode + const db = new BrainyData({ + writeOnly: true, + storage: { + // Use file system storage for persistence + forceFileSystemStorage: true + } + }) + + // Initialize the database + // This will skip loading the index into memory, saving memory and startup time + await db.init() + + console.log('Database initialized in write-only mode') + console.log('Is write-only:', db.isWriteOnly()) // Should print true + + // Add some data + // This works normally even in write-only mode + const id1 = await db.add('This is a test document', { title: 'Test Document 1' }) + const id2 = await db.add('Another test document with different content', { title: 'Test Document 2' }) + + console.log(`Added documents with IDs: ${id1}, ${id2}`) + + // Try to search (this will throw an error in write-only mode) + try { + await db.search('test', 5) + } catch (error) { + console.log('Search error (expected):', error.message) + } + + // Switch to normal mode to perform searches + console.log('Switching to normal mode...') + db.setWriteOnly(false) + + // Re-initialize to load the index + await db.init() + + console.log('Database reinitialized in normal mode') + console.log('Is write-only:', db.isWriteOnly()) // Should print false + + // Now search works + const results = await db.search('test', 5) + console.log('Search results:', results.map(r => ({ id: r.id, score: r.score, title: r.metadata?.title }))) + + // Clean up + await db.shutDown() +} + +// Run the example +demonstrateWriteOnlyMode().catch(error => { + console.error('Error in write-only mode example:', error) +}) diff --git a/src/brainyData.ts b/src/brainyData.ts index eb6b9741..b64509ba 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -3,3656 +3,3827 @@ * Main class that provides the vector database functionality */ -import {v4 as uuidv4} from 'uuid' -import {HNSWIndex} from './hnsw/hnswIndex.js' +import { v4 as uuidv4 } from 'uuid' +import { HNSWIndex } from './hnsw/hnswIndex.js' import { - HNSWIndexOptimized, - HNSWOptimizedConfig + HNSWIndexOptimized, + HNSWOptimizedConfig } from './hnsw/hnswIndexOptimized.js' -import {createStorage} from './storage/storageFactory.js' +import { createStorage } from './storage/storageFactory.js' import { - DistanceFunction, - GraphVerb, - EmbeddingFunction, - HNSWConfig, - HNSWNoun, - SearchResult, - StorageAdapter, - Vector, - VectorDocument + DistanceFunction, + GraphVerb, + EmbeddingFunction, + HNSWConfig, + HNSWNoun, + SearchResult, + StorageAdapter, + Vector, + VectorDocument } from './coreTypes.js' import { - cosineDistance, - defaultEmbeddingFunction, - defaultBatchEmbeddingFunction, - getDefaultEmbeddingFunction, - getDefaultBatchEmbeddingFunction, - euclideanDistance, - cleanupWorkerPools + cosineDistance, + defaultEmbeddingFunction, + defaultBatchEmbeddingFunction, + getDefaultEmbeddingFunction, + getDefaultBatchEmbeddingFunction, + euclideanDistance, + cleanupWorkerPools } from './utils/index.js' -import {NounType, VerbType, GraphNoun} from './types/graphTypes.js' +import { NounType, VerbType, GraphNoun } from './types/graphTypes.js' import { - ServerSearchConduitAugmentation, - createServerSearchAugmentations + ServerSearchConduitAugmentation, + createServerSearchAugmentations } from './augmentations/serverSearchAugmentations.js' -import {WebSocketConnection, AugmentationType, IAugmentation} from './types/augmentations.js' -import {BrainyDataInterface} from './types/brainyDataInterface.js' -import {augmentationPipeline} from './augmentationPipeline.js' +import { + WebSocketConnection, + AugmentationType, + IAugmentation +} from './types/augmentations.js' +import { BrainyDataInterface } from './types/brainyDataInterface.js' +import { augmentationPipeline } from './augmentationPipeline.js' export interface BrainyDataConfig { - /** - * HNSW index configuration - * Uses the optimized HNSW implementation which supports large datasets - * through product quantization and disk-based storage - */ - hnsw?: Partial + /** + * HNSW index configuration + * Uses the optimized HNSW implementation which supports large datasets + * through product quantization and disk-based storage + */ + hnsw?: Partial - /** - * Distance function to use for similarity calculations - */ - distanceFunction?: DistanceFunction + /** + * Distance function to use for similarity calculations + */ + distanceFunction?: DistanceFunction - /** - * Custom storage adapter (if not provided, will use OPFS or memory storage) - */ - storageAdapter?: StorageAdapter + /** + * Custom storage adapter (if not provided, will use OPFS or memory storage) + */ + storageAdapter?: StorageAdapter - /** - * Storage configuration options - * These will be passed to createStorage if storageAdapter is not provided - */ - storage?: { - requestPersistentStorage?: boolean - r2Storage?: { - bucketName?: string - accountId?: string - accessKeyId?: string - secretAccessKey?: string - } - s3Storage?: { - bucketName?: string - accessKeyId?: string - secretAccessKey?: string - region?: string - } - gcsStorage?: { - bucketName?: string - accessKeyId?: string - secretAccessKey?: string - endpoint?: string - } - customS3Storage?: { - bucketName?: string - accessKeyId?: string - secretAccessKey?: string - endpoint?: string - region?: string - } - forceFileSystemStorage?: boolean - forceMemoryStorage?: boolean + /** + * Storage configuration options + * These will be passed to createStorage if storageAdapter is not provided + */ + storage?: { + requestPersistentStorage?: boolean + r2Storage?: { + bucketName?: string + accountId?: string + accessKeyId?: string + secretAccessKey?: string } - - /** - * Embedding function to convert data to vectors - */ - embeddingFunction?: EmbeddingFunction - - /** - * Set the database to read-only mode - * When true, all write operations will throw an error - */ - readOnly?: boolean - - /** - * Remote server configuration for search operations - */ - remoteServer?: { - /** - * WebSocket URL of the remote Brainy server - */ - url: string - - /** - * WebSocket protocols to use for the connection - */ - protocols?: string | string[] - - /** - * Whether to automatically connect to the remote server on initialization - */ - autoConnect?: boolean + s3Storage?: { + bucketName?: string + accessKeyId?: string + secretAccessKey?: string + region?: string } + gcsStorage?: { + bucketName?: string + accessKeyId?: string + secretAccessKey?: string + endpoint?: string + } + customS3Storage?: { + bucketName?: string + accessKeyId?: string + secretAccessKey?: string + endpoint?: string + region?: string + } + forceFileSystemStorage?: boolean + forceMemoryStorage?: boolean + } + + /** + * Embedding function to convert data to vectors + */ + embeddingFunction?: EmbeddingFunction + + /** + * Set the database to read-only mode + * When true, all write operations will throw an error + */ + readOnly?: boolean + + /** + * Set the database to write-only mode + * When true, the index is not loaded into memory and search operations will throw an error + * This is useful for data ingestion scenarios where only write operations are needed + */ + writeOnly?: boolean + + /** + * Remote server configuration for search operations + */ + remoteServer?: { + /** + * WebSocket URL of the remote Brainy server + */ + url: string /** - * Logging configuration + * WebSocket protocols to use for the connection */ - logging?: { - /** - * Whether to enable verbose logging - * When false, suppresses non-essential log messages like model loading progress - * Default: true - */ - verbose?: boolean - } + protocols?: string | string[] /** - * Timeout configuration for async operations - * Controls how long operations wait before timing out + * Whether to automatically connect to the remote server on initialization */ - timeouts?: { - /** - * Timeout for get operations in milliseconds - * Default: 30000 (30 seconds) - */ - get?: number + autoConnect?: boolean + } - /** - * Timeout for add operations in milliseconds - * Default: 60000 (60 seconds) - */ - add?: number + /** + * Logging configuration + */ + logging?: { + /** + * Whether to enable verbose logging + * When false, suppresses non-essential log messages like model loading progress + * Default: true + */ + verbose?: boolean + } - /** - * Timeout for delete operations in milliseconds - * Default: 30000 (30 seconds) - */ - delete?: number - } + /** + * Timeout configuration for async operations + * Controls how long operations wait before timing out + */ + timeouts?: { + /** + * Timeout for get operations in milliseconds + * Default: 30000 (30 seconds) + */ + get?: number /** - * Retry policy configuration for failed operations - * Controls how operations are retried on failure + * Timeout for add operations in milliseconds + * Default: 60000 (60 seconds) */ - retryPolicy?: { - /** - * Maximum number of retry attempts - * Default: 3 - */ - maxRetries?: number - - /** - * Initial delay between retries in milliseconds - * Default: 1000 (1 second) - */ - initialDelay?: number - - /** - * Maximum delay between retries in milliseconds - * Default: 10000 (10 seconds) - */ - maxDelay?: number - - /** - * Multiplier for exponential backoff - * Default: 2 - */ - backoffMultiplier?: number - } + add?: number /** - * Real-time update configuration - * Controls how the database handles updates when data is added by external processes + * Timeout for delete operations in milliseconds + * Default: 30000 (30 seconds) */ - realtimeUpdates?: { - /** - * Whether to enable automatic updates of the index and statistics - * When true, the database will periodically check for new data in storage - * Default: false - */ - enabled?: boolean + delete?: number + } - /** - * The interval (in milliseconds) at which to check for updates - * Default: 30000 (30 seconds) - */ - interval?: number + /** + * Retry policy configuration for failed operations + * Controls how operations are retried on failure + */ + retryPolicy?: { + /** + * Maximum number of retry attempts + * Default: 3 + */ + maxRetries?: number - /** - * Whether to update statistics when checking for updates - * Default: true - */ - updateStatistics?: boolean + /** + * Initial delay between retries in milliseconds + * Default: 1000 (1 second) + */ + initialDelay?: number - /** - * Whether to update the index when checking for updates - * Default: true - */ - updateIndex?: boolean - } + /** + * Maximum delay between retries in milliseconds + * Default: 10000 (10 seconds) + */ + maxDelay?: number + + /** + * Multiplier for exponential backoff + * Default: 2 + */ + backoffMultiplier?: number + } + + /** + * Real-time update configuration + * Controls how the database handles updates when data is added by external processes + */ + realtimeUpdates?: { + /** + * Whether to enable automatic updates of the index and statistics + * When true, the database will periodically check for new data in storage + * Default: false + */ + enabled?: boolean + + /** + * The interval (in milliseconds) at which to check for updates + * Default: 30000 (30 seconds) + */ + interval?: number + + /** + * Whether to update statistics when checking for updates + * Default: true + */ + updateStatistics?: boolean + + /** + * Whether to update the index when checking for updates + * Default: true + */ + updateIndex?: boolean + } } export class BrainyData implements BrainyDataInterface { - private index: HNSWIndex | HNSWIndexOptimized - private storage: StorageAdapter | null = null - private isInitialized = false - private isInitializing = false - private embeddingFunction: EmbeddingFunction - private distanceFunction: DistanceFunction - private requestPersistentStorage: boolean - private readOnly: boolean - private storageConfig: BrainyDataConfig['storage'] = {} - private useOptimizedIndex: boolean = false - private _dimensions: number - private loggingConfig: BrainyDataConfig['logging'] = {verbose: true} - - // Timeout and retry configuration - private timeoutConfig: BrainyDataConfig['timeouts'] = {} - private retryConfig: BrainyDataConfig['retryPolicy'] = {} - - // Real-time update properties - private realtimeUpdateConfig: Required> = { - enabled: false, - interval: 30000, // 30 seconds - updateStatistics: true, - updateIndex: true - } - private updateTimerId: NodeJS.Timeout | null = null - private lastUpdateTime = 0 - private lastKnownNounCount = 0 + private index: HNSWIndex | HNSWIndexOptimized + private storage: StorageAdapter | null = null + private isInitialized = false + private isInitializing = false + private embeddingFunction: EmbeddingFunction + private distanceFunction: DistanceFunction + private requestPersistentStorage: boolean + private readOnly: boolean + private writeOnly: boolean + private storageConfig: BrainyDataConfig['storage'] = {} + private useOptimizedIndex: boolean = false + private _dimensions: number + private loggingConfig: BrainyDataConfig['logging'] = { verbose: true } - // Remote server properties - private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null - private serverSearchConduit: ServerSearchConduitAugmentation | null = null - private serverConnection: WebSocketConnection | null = null + // Timeout and retry configuration + private timeoutConfig: BrainyDataConfig['timeouts'] = {} + private retryConfig: BrainyDataConfig['retryPolicy'] = {} - /** - * Get the vector dimensions - */ - public get dimensions(): number { - return this._dimensions + // Real-time update properties + private realtimeUpdateConfig: Required< + NonNullable + > = { + enabled: false, + interval: 30000, // 30 seconds + updateStatistics: true, + updateIndex: true + } + private updateTimerId: NodeJS.Timeout | null = null + private lastUpdateTime = 0 + private lastKnownNounCount = 0 + + // Remote server properties + private remoteServerConfig: BrainyDataConfig['remoteServer'] | null = null + private serverSearchConduit: ServerSearchConduitAugmentation | null = null + private serverConnection: WebSocketConnection | null = null + + /** + * Get the vector dimensions + */ + public get dimensions(): number { + return this._dimensions + } + + /** + * Get the maximum connections parameter from HNSW configuration + */ + public get maxConnections(): number { + const config = this.index.getConfig() + return config.M || 16 + } + + /** + * Get the efConstruction parameter from HNSW configuration + */ + public get efConstruction(): number { + const config = this.index.getConfig() + return config.efConstruction || 200 + } + + /** + * Create a new vector database + */ + constructor(config: BrainyDataConfig = {}) { + // Set dimensions to fixed value of 512 (Universal Sentence Encoder dimension) + this._dimensions = 512 + + // Set distance function + this.distanceFunction = config.distanceFunction || cosineDistance + + // Always use the optimized HNSW index implementation + this.index = new HNSWIndexOptimized( + config.hnsw || {}, + this.distanceFunction, + config.storageAdapter || null + ) + this.useOptimizedIndex = true + + // Set storage if provided, otherwise it will be initialized in init() + this.storage = config.storageAdapter || null + + // Store logging configuration + if (config.logging !== undefined) { + this.loggingConfig = { + ...this.loggingConfig, + ...config.logging + } } - /** - * Get the maximum connections parameter from HNSW configuration - */ - public get maxConnections(): number { - const config = this.index.getConfig() - return config.M || 16 + // Set embedding function if provided, otherwise create one with the appropriate verbose setting + if (config.embeddingFunction) { + this.embeddingFunction = config.embeddingFunction + } else { + this.embeddingFunction = getDefaultEmbeddingFunction({ + verbose: this.loggingConfig?.verbose + }) } - /** - * Get the efConstruction parameter from HNSW configuration - */ - public get efConstruction(): number { - const config = this.index.getConfig() - return config.efConstruction || 200 + // Set persistent storage request flag + this.requestPersistentStorage = + config.storage?.requestPersistentStorage || false + + // Set read-only flag + this.readOnly = config.readOnly || false + + // Set write-only flag + this.writeOnly = config.writeOnly || false + + // Validate that readOnly and writeOnly are not both true + if (this.readOnly && this.writeOnly) { + throw new Error('Database cannot be both read-only and write-only') } - /** - * Create a new vector database - */ - constructor(config: BrainyDataConfig = {}) { - // Set dimensions to fixed value of 512 (Universal Sentence Encoder dimension) - this._dimensions = 512 + // Store storage configuration for later use in init() + this.storageConfig = config.storage || {} - // Set distance function - this.distanceFunction = config.distanceFunction || cosineDistance + // Store timeout and retry configuration + this.timeoutConfig = config.timeouts || {} + this.retryConfig = config.retryPolicy || {} - // Always use the optimized HNSW index implementation - this.index = new HNSWIndexOptimized( - config.hnsw || {}, - this.distanceFunction, - config.storageAdapter || null + // Store remote server configuration if provided + if (config.remoteServer) { + this.remoteServerConfig = config.remoteServer + } + + // Initialize real-time update configuration if provided + if (config.realtimeUpdates) { + this.realtimeUpdateConfig = { + ...this.realtimeUpdateConfig, + ...config.realtimeUpdates + } + } + } + + /** + * Check if the database is in read-only mode and throw an error if it is + * @throws Error if the database is in read-only mode + */ + private checkReadOnly(): void { + if (this.readOnly) { + throw new Error( + 'Cannot perform write operation: database is in read-only mode' + ) + } + } + + /** + * Check if the database is in write-only mode and throw an error if it is + * @throws Error if the database is in write-only mode + */ + private checkWriteOnly(): void { + if (this.writeOnly) { + throw new Error( + 'Cannot perform search operation: database is in write-only mode' + ) + } + } + + /** + * Start real-time updates if enabled in the configuration + * This will periodically check for new data in storage and update the in-memory index and statistics + */ + private startRealtimeUpdates(): void { + // If real-time updates are not enabled, do nothing + if (!this.realtimeUpdateConfig.enabled) { + return + } + + // If the update timer is already running, do nothing + if (this.updateTimerId !== null) { + return + } + + // Set the initial last known noun count + this.getNounCount() + .then((count) => { + this.lastKnownNounCount = count + }) + .catch((error) => { + console.warn( + 'Failed to get initial noun count for real-time updates:', + error ) - this.useOptimizedIndex = true + }) - // Set storage if provided, otherwise it will be initialized in init() - this.storage = config.storageAdapter || null + // Start the update timer + this.updateTimerId = setInterval(() => { + this.checkForUpdates().catch((error) => { + console.warn('Error during real-time update check:', error) + }) + }, this.realtimeUpdateConfig.interval) - // Store logging configuration - if (config.logging !== undefined) { - this.loggingConfig = { - ...this.loggingConfig, - ...config.logging - } - } + if (this.loggingConfig?.verbose) { + console.log( + `Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms` + ) + } + } - // Set embedding function if provided, otherwise create one with the appropriate verbose setting - if (config.embeddingFunction) { - this.embeddingFunction = config.embeddingFunction + /** + * Stop real-time updates + */ + private stopRealtimeUpdates(): void { + // If the update timer is not running, do nothing + if (this.updateTimerId === null) { + return + } + + // Stop the update timer + clearInterval(this.updateTimerId) + this.updateTimerId = null + + if (this.loggingConfig?.verbose) { + console.log('Real-time updates stopped') + } + } + + /** + * Manually check for updates in storage and update the in-memory index and statistics + * This can be called by the user to force an update check even if automatic updates are not enabled + */ + public async checkForUpdatesNow(): Promise { + await this.ensureInitialized() + return this.checkForUpdates() + } + + /** + * Enable real-time updates with the specified configuration + * @param config Configuration for real-time updates + */ + public enableRealtimeUpdates( + config?: Partial + ): void { + // Update configuration if provided + if (config) { + this.realtimeUpdateConfig = { + ...this.realtimeUpdateConfig, + ...config + } + } + + // Enable updates + this.realtimeUpdateConfig.enabled = true + + // Start updates if initialized + if (this.isInitialized) { + this.startRealtimeUpdates() + } + } + + /** + * Disable real-time updates + */ + public disableRealtimeUpdates(): void { + // Disable updates + this.realtimeUpdateConfig.enabled = false + + // Stop updates if running + this.stopRealtimeUpdates() + } + + /** + * Get the current real-time update configuration + * @returns The current real-time update configuration + */ + public getRealtimeUpdateConfig(): Required< + NonNullable + > { + return { ...this.realtimeUpdateConfig } + } + + /** + * Check for updates in storage and update the in-memory index and statistics if needed + * This is called periodically by the update timer when real-time updates are enabled + * Uses change log mechanism for efficient updates instead of full scans + */ + private async checkForUpdates(): Promise { + // If the database is not initialized, do nothing + if (!this.isInitialized || !this.storage) { + return + } + + try { + // Record the current time + const startTime = Date.now() + + // Update statistics if enabled + if (this.realtimeUpdateConfig.updateStatistics) { + await this.storage.flushStatisticsToStorage() + // Clear the statistics cache to force a reload from storage + await this.getStatistics({ forceRefresh: true }) + } + + // Update index if enabled + if (this.realtimeUpdateConfig.updateIndex) { + // Use change log mechanism if available (for S3 and other distributed storage) + if (typeof this.storage.getChangesSince === 'function') { + await this.applyChangesFromLog() } else { - this.embeddingFunction = getDefaultEmbeddingFunction({ - verbose: this.loggingConfig?.verbose - }) + // Fallback to the old method for storage adapters that don't support change logs + await this.applyChangesFromFullScan() } + } - // Set persistent storage request flag - this.requestPersistentStorage = - config.storage?.requestPersistentStorage || false + // Update the last update time + this.lastUpdateTime = Date.now() - // Set read-only flag - this.readOnly = config.readOnly || false + if (this.loggingConfig?.verbose) { + const duration = this.lastUpdateTime - startTime + console.log(`Real-time update completed in ${duration}ms`) + } + } catch (error) { + console.error('Failed to check for updates:', error) + // Don't rethrow the error to avoid disrupting the update timer + } + } - // Store storage configuration for later use in init() - this.storageConfig = config.storage || {} - - // Store timeout and retry configuration - this.timeoutConfig = config.timeouts || {} - this.retryConfig = config.retryPolicy || {} - - // Store remote server configuration if provided - if (config.remoteServer) { - this.remoteServerConfig = config.remoteServer - } - - // Initialize real-time update configuration if provided - if (config.realtimeUpdates) { - this.realtimeUpdateConfig = { - ...this.realtimeUpdateConfig, - ...config.realtimeUpdates - } - } + /** + * Apply changes using the change log mechanism (efficient for distributed storage) + */ + private async applyChangesFromLog(): Promise { + if (!this.storage || typeof this.storage.getChangesSince !== 'function') { + return } - /** - * Check if the database is in read-only mode and throw an error if it is - * @throws Error if the database is in read-only mode - */ - private checkReadOnly(): void { - if (this.readOnly) { - throw new Error( - 'Cannot perform write operation: database is in read-only mode' - ) - } - } - - /** - * Start real-time updates if enabled in the configuration - * This will periodically check for new data in storage and update the in-memory index and statistics - */ - private startRealtimeUpdates(): void { - // If real-time updates are not enabled, do nothing - if (!this.realtimeUpdateConfig.enabled) { - return - } - - // If the update timer is already running, do nothing - if (this.updateTimerId !== null) { - return - } - - // Set the initial last known noun count - this.getNounCount().then(count => { - this.lastKnownNounCount = count - }).catch(error => { - console.warn('Failed to get initial noun count for real-time updates:', error) - }) - - // Start the update timer - this.updateTimerId = setInterval(() => { - this.checkForUpdates().catch(error => { - console.warn('Error during real-time update check:', error) - }) - }, this.realtimeUpdateConfig.interval) - - if (this.loggingConfig?.verbose) { - console.log(`Real-time updates started with interval: ${this.realtimeUpdateConfig.interval}ms`) - } - } - - /** - * Stop real-time updates - */ - private stopRealtimeUpdates(): void { - // If the update timer is not running, do nothing - if (this.updateTimerId === null) { - return - } - - // Stop the update timer - clearInterval(this.updateTimerId) - this.updateTimerId = null - - if (this.loggingConfig?.verbose) { - console.log('Real-time updates stopped') - } - } - - /** - * Manually check for updates in storage and update the in-memory index and statistics - * This can be called by the user to force an update check even if automatic updates are not enabled - */ - public async checkForUpdatesNow(): Promise { - await this.ensureInitialized() - return this.checkForUpdates() - } - - /** - * Enable real-time updates with the specified configuration - * @param config Configuration for real-time updates - */ - public enableRealtimeUpdates(config?: Partial): void { - // Update configuration if provided - if (config) { - this.realtimeUpdateConfig = { - ...this.realtimeUpdateConfig, - ...config - } - } - - // Enable updates - this.realtimeUpdateConfig.enabled = true - - // Start updates if initialized - if (this.isInitialized) { - this.startRealtimeUpdates() - } - } - - /** - * Disable real-time updates - */ - public disableRealtimeUpdates(): void { - // Disable updates - this.realtimeUpdateConfig.enabled = false - - // Stop updates if running - this.stopRealtimeUpdates() - } - - /** - * Get the current real-time update configuration - * @returns The current real-time update configuration - */ - public getRealtimeUpdateConfig(): Required> { - return { ...this.realtimeUpdateConfig } - } - - /** - * Check for updates in storage and update the in-memory index and statistics if needed - * This is called periodically by the update timer when real-time updates are enabled - * Uses change log mechanism for efficient updates instead of full scans - */ - private async checkForUpdates(): Promise { - // If the database is not initialized, do nothing - if (!this.isInitialized || !this.storage) { - return - } - + try { + // Get changes since the last update + const changes = await this.storage.getChangesSince( + this.lastUpdateTime, + 1000 + ) // Limit to 1000 changes per batch + + let addedCount = 0 + let updatedCount = 0 + let deletedCount = 0 + + for (const change of changes) { try { - // Record the current time - const startTime = Date.now() - - // Update statistics if enabled - if (this.realtimeUpdateConfig.updateStatistics) { - await this.storage.flushStatisticsToStorage() - // Clear the statistics cache to force a reload from storage - await this.getStatistics({ forceRefresh: true }) - } - - // Update index if enabled - if (this.realtimeUpdateConfig.updateIndex) { - // Use change log mechanism if available (for S3 and other distributed storage) - if (typeof this.storage.getChangesSince === 'function') { - await this.applyChangesFromLog() - } else { - // Fallback to the old method for storage adapters that don't support change logs - await this.applyChangesFromFullScan() - } - } - - // Update the last update time - this.lastUpdateTime = Date.now() - - if (this.loggingConfig?.verbose) { - const duration = this.lastUpdateTime - startTime - console.log(`Real-time update completed in ${duration}ms`) - } - } catch (error) { - console.error('Failed to check for updates:', error) - // Don't rethrow the error to avoid disrupting the update timer - } - } + switch (change.operation) { + case 'add': + case 'update': + if (change.entityType === 'noun' && change.data) { + const noun = change.data as HNSWNoun - /** - * Apply changes using the change log mechanism (efficient for distributed storage) - */ - private async applyChangesFromLog(): Promise { - if (!this.storage || typeof this.storage.getChangesSince !== 'function') { - return - } - - try { - // Get changes since the last update - const changes = await this.storage.getChangesSince(this.lastUpdateTime, 1000) // Limit to 1000 changes per batch - - let addedCount = 0 - let updatedCount = 0 - let deletedCount = 0 - - for (const change of changes) { - try { - switch (change.operation) { - case 'add': - case 'update': - if (change.entityType === 'noun' && change.data) { - const noun = change.data as HNSWNoun - - // Check if the vector dimensions match the expected dimensions - if (noun.vector.length !== this._dimensions) { - console.warn( - `Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}` - ) - continue - } - - // Add or update in index - await this.index.addItem({ - id: noun.id, - vector: noun.vector - }) - - if (change.operation === 'add') { - addedCount++ - } else { - updatedCount++ - } - - if (this.loggingConfig?.verbose) { - console.log(`${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update`) - } - } - break - - case 'delete': - if (change.entityType === 'noun') { - // Remove from index - await this.index.removeItem(change.entityId) - deletedCount++ - - if (this.loggingConfig?.verbose) { - console.log(`Removed noun ${change.entityId} from index during real-time update`) - } - } - break - } - } catch (changeError) { - console.error(`Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`, changeError) - // Continue with other changes - } - } - - if (this.loggingConfig?.verbose && (addedCount > 0 || updatedCount > 0 || deletedCount > 0)) { - console.log(`Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log`) - } - - // Update the last known noun count - this.lastKnownNounCount = await this.getNounCount() - - } catch (error) { - console.error('Failed to apply changes from log, falling back to full scan:', error) - // Fallback to full scan if change log fails - await this.applyChangesFromFullScan() - } - } - - /** - * Apply changes using full scan method (fallback for storage adapters without change log support) - */ - private async applyChangesFromFullScan(): Promise { - try { - // Get the current noun count - const currentCount = await this.getNounCount() - - // If the noun count has changed, update the index - if (currentCount !== this.lastKnownNounCount) { - // Get all nouns from storage - const nouns = await this.storage!.getAllNouns() - - // Get all nouns currently in the index - const indexNouns = this.index.getNouns() - const indexNounIds = new Set(indexNouns.keys()) - - // Find nouns that are in storage but not in the index - const newNouns = nouns.filter(noun => !indexNounIds.has(noun.id)) - - // Add new nouns to the index - for (const noun of newNouns) { - // Check if the vector dimensions match the expected dimensions - if (noun.vector.length !== this._dimensions) { - console.warn( - `Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}` - ) - continue - } - - // Add to index - await this.index.addItem({ - id: noun.id, - vector: noun.vector - }) - - if (this.loggingConfig?.verbose) { - console.log(`Added new noun ${noun.id} to index during real-time update`) - } - } - - // Update the last known noun count - this.lastKnownNounCount = currentCount - - if (this.loggingConfig?.verbose && newNouns.length > 0) { - console.log(`Real-time update: Added ${newNouns.length} new nouns to index using full scan`) - } - } - } catch (error) { - console.error('Failed to apply changes from full scan:', error) - throw error - } - } - - /** - * Get the current augmentation name if available - * This is used to auto-detect the service performing data operations - * @returns The name of the current augmentation or 'default' if none is detected - */ - private getCurrentAugmentation(): string { - try { - // Get all registered augmentations - const augmentationTypes = augmentationPipeline.getAvailableAugmentationTypes() - - // Check each type of augmentation - for (const type of augmentationTypes) { - const augmentations = augmentationPipeline.getAugmentationsByType(type) - - // Find the first enabled augmentation - for (const augmentation of augmentations) { - if (augmentation.enabled) { - return augmentation.name - } - } - } - - return 'default' - } catch (error) { - // If there's any error in detection, return default - console.warn('Failed to detect current augmentation:', error) - return 'default' - } - } - - /** - * Initialize the database - * Loads existing data from storage if available - */ - public async init(): Promise { - if (this.isInitialized) { - return - } - - // Prevent recursive initialization - if (this.isInitializing) { - return - } - - this.isInitializing = true - - try { - // Pre-load the embedding model early to ensure it's always available - // This helps prevent issues with the Universal Sentence Encoder not being loaded - try { - // Pre-loading Universal Sentence Encoder model - // Call embedding function directly to avoid circular dependency with embed() - await this.embeddingFunction('') - // Universal Sentence Encoder model loaded successfully - } catch (embedError) { - console.warn( - 'Failed to pre-load Universal Sentence Encoder:', - embedError - ) - - // Try again with a retry mechanism - // Retrying Universal Sentence Encoder initialization - try { - // Wait a moment before retrying - await new Promise((resolve) => setTimeout(resolve, 1000)) - - // Try again with a different approach - use the non-threaded version - // This is a fallback in case the threaded version fails - const {createTensorFlowEmbeddingFunction} = await import( - './utils/embedding.js' - ) - const fallbackEmbeddingFunction = createTensorFlowEmbeddingFunction() - - // Test the fallback embedding function - await fallbackEmbeddingFunction('') - - // If successful, replace the embedding function - console.log( - 'Successfully loaded Universal Sentence Encoder with fallback method' - ) - this.embeddingFunction = fallbackEmbeddingFunction - } catch (retryError) { - console.error( - 'All attempts to load Universal Sentence Encoder failed:', - retryError - ) - // Continue initialization even if embedding model fails to load - // The application will need to handle missing embedding functionality - } - } - - // Initialize storage if not provided in constructor - if (!this.storage) { - // Combine storage config with requestPersistentStorage for backward compatibility - let storageOptions = { - ...this.storageConfig, - requestPersistentStorage: this.requestPersistentStorage - } - - // Ensure s3Storage has all required fields if it's provided - if (storageOptions.s3Storage) { - // Only include s3Storage if all required fields are present - if (storageOptions.s3Storage.bucketName && - storageOptions.s3Storage.accessKeyId && - storageOptions.s3Storage.secretAccessKey) { - // All required fields are present, keep s3Storage as is - } else { - // Missing required fields, remove s3Storage to avoid type errors - const {s3Storage, ...rest} = storageOptions - storageOptions = rest - console.warn('Ignoring s3Storage configuration due to missing required fields') - } - } - - // Use type assertion to tell TypeScript that storageOptions conforms to StorageOptions - this.storage = await createStorage(storageOptions as any) - } - - // Initialize storage - await this.storage!.init() - - // If using optimized index, set the storage adapter - if (this.useOptimizedIndex && this.index instanceof HNSWIndexOptimized) { - this.index.setStorage(this.storage!) - } - - // Load all nouns from storage - const nouns: HNSWNoun[] = await this.storage!.getAllNouns() - - // Clear the index and add all nouns - this.index.clear() - for (const noun of nouns) { // Check if the vector dimensions match the expected dimensions if (noun.vector.length !== this._dimensions) { - console.warn( - `Deleting noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}` - ) - // Delete the mismatched noun from storage to prevent future issues - await this.storage!.deleteNoun(noun.id) - continue + console.warn( + `Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}` + ) + continue } - // Add to index + // Add or update in index await this.index.addItem({ - id: noun.id, - vector: noun.vector + id: noun.id, + vector: noun.vector }) - } - // Connect to remote server if configured with autoConnect - if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) { - try { - await this.connectToRemoteServer( - this.remoteServerConfig.url, - this.remoteServerConfig.protocols - ) - } catch (remoteError) { - console.warn('Failed to auto-connect to remote server:', remoteError) - // Continue initialization even if remote connection fails - } - } - - this.isInitialized = true - this.isInitializing = false - - // Start real-time updates if enabled - this.startRealtimeUpdates() - } catch (error) { - console.error('Failed to initialize BrainyData:', error) - this.isInitializing = false - throw new Error(`Failed to initialize BrainyData: ${error}`) - } - } - - /** - * Connect to a remote Brainy server for search operations - * @param serverUrl WebSocket URL of the remote Brainy server - * @param protocols Optional WebSocket protocols to use - * @returns The connection object - */ - public async connectToRemoteServer( - serverUrl: string, - protocols?: string | string[] - ): Promise { - await this.ensureInitialized() - - try { - // Create server search augmentations - const {conduit, connection} = await createServerSearchAugmentations( - serverUrl, - { - protocols, - localDb: this - } - ) - - // Store the conduit and connection - this.serverSearchConduit = conduit - this.serverConnection = connection - - return connection - } catch (error) { - console.error('Failed to connect to remote server:', error) - throw new Error(`Failed to connect to remote server: ${error}`) - } - } - - /** - * Add a vector or data to the database - * If the input is not a vector, it will be converted using the embedding function - * @param vectorOrData Vector or data to add - * @param metadata Optional metadata to associate with the vector - * @param options Additional options - * @returns The ID of the added vector - */ - public async add( - vectorOrData: Vector | any, - metadata?: T, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - addToRemote?: boolean // Whether to also add to the remote server if connected - id?: string // Optional ID to use instead of generating a new one - service?: string // The service that is inserting the data - } = {} - ): Promise { - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() - - // Validate input is not null or undefined - if (vectorOrData === null || vectorOrData === undefined) { - throw new Error('Input cannot be null or undefined') - } - - try { - let vector: Vector - - // First validate if input is an array but contains non-numeric values - if (Array.isArray(vectorOrData)) { - for (let i = 0; i < vectorOrData.length; i++) { - if (typeof vectorOrData[i] !== 'number') { - throw new Error('Vector contains non-numeric values') - } - } - } - - // Check if input is already a vector - if ( - Array.isArray(vectorOrData) && - !options.forceEmbed - ) { - // Input is already a vector (and we've validated it contains only numbers) - vector = vectorOrData - } else { - // Input needs to be vectorized - try { - vector = await this.embeddingFunction(vectorOrData) - } catch (embedError) { - throw new Error(`Failed to vectorize data: ${embedError}`) - } - } - - // Check if vector is defined - if (!vector) { - throw new Error('Vector is undefined or null') - } - - // Validate vector dimensions - if (vector.length !== this._dimensions) { - throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}`) - } - - // Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID - const id = - options.id || - (metadata && typeof metadata === 'object' && 'id' in metadata - ? (metadata as any).id - : uuidv4()) - - // Add to index - await this.index.addItem({id, vector}) - - // Get the noun from the index - const noun = this.index.getNouns().get(id) - - if (!noun) { - throw new Error(`Failed to retrieve newly created noun with ID ${id}`) - } - - // Save noun to storage - await this.storage!.saveNoun(noun) - - // Track noun statistics - const service = options.service || this.getCurrentAugmentation() - await this.storage!.incrementStatistic('noun', service) - - // Save metadata if provided and not empty - if (metadata !== undefined) { - // Skip saving if metadata is an empty object - if (metadata && typeof metadata === 'object' && Object.keys(metadata).length === 0) { - // Don't save empty metadata - // Explicitly save null to ensure no metadata is stored - await this.storage!.saveMetadata(id, null) + if (change.operation === 'add') { + addedCount++ } else { - // Validate noun type if metadata is for a GraphNoun - if (metadata && typeof metadata === 'object' && 'noun' in metadata) { - const nounType = (metadata as unknown as GraphNoun).noun - - // Check if the noun type is valid - const isValidNounType = Object.values(NounType).includes(nounType) - - if (!isValidNounType) { - console.warn( - `Invalid noun type: ${nounType}. Falling back to GraphNoun.` - ) - // Set a default noun type - ;(metadata as unknown as GraphNoun).noun = NounType.Concept - } - - // Ensure createdBy field is populated for GraphNoun - const service = options.service || this.getCurrentAugmentation() - const graphNoun = metadata as unknown as GraphNoun - - // Only set createdBy if it doesn't exist or is being explicitly updated - if (!graphNoun.createdBy || options.service) { - graphNoun.createdBy = { - augmentation: service, - version: '1.0' // TODO: Get actual version from augmentation - } - } - - // Update timestamps - const now = new Date() - const timestamp = { - seconds: Math.floor(now.getTime() / 1000), - nanoseconds: (now.getTime() % 1000) * 1000000 - } - - // Set createdAt if it doesn't exist - if (!graphNoun.createdAt) { - graphNoun.createdAt = timestamp - } - - // Always update updatedAt - graphNoun.updatedAt = timestamp - } - - // Create a copy of the metadata without modifying the original - let metadataToSave = metadata - if (metadata && typeof metadata === 'object') { - // Always make a copy without adding the ID - metadataToSave = {...metadata} - } - - await this.storage!.saveMetadata(id, metadataToSave) - - // Track metadata statistics - const metadataService = options.service || this.getCurrentAugmentation() - await this.storage!.incrementStatistic('metadata', metadataService) + updatedCount++ } - } - // Update HNSW index size (excluding verbs) - await this.storage!.updateHnswIndexSize(await this.getNounCount()) - - // If addToRemote is true and we're connected to a remote server, add to remote as well - if (options.addToRemote && this.isConnectedToRemoteServer()) { - try { - await this.addToRemote(id, vector, metadata) - } catch (remoteError) { - console.warn( - `Failed to add to remote server: ${remoteError}. Continuing with local add.` - ) + if (this.loggingConfig?.verbose) { + console.log( + `${change.operation === 'add' ? 'Added' : 'Updated'} noun ${noun.id} in index during real-time update` + ) } - } + } + break - return id - } catch (error) { - console.error('Failed to add vector:', error) - throw new Error(`Failed to add vector: ${error}`) + case 'delete': + if (change.entityType === 'noun') { + // Remove from index + await this.index.removeItem(change.entityId) + deletedCount++ + + if (this.loggingConfig?.verbose) { + console.log( + `Removed noun ${change.entityId} from index during real-time update` + ) + } + } + break + } + } catch (changeError) { + console.error( + `Failed to apply change ${change.operation} for ${change.entityType} ${change.entityId}:`, + changeError + ) + // Continue with other changes } - } + } - /** - * Add a text item to the database with automatic embedding - * This is a convenience method for adding text data with metadata - * @param text Text data to add - * @param metadata Metadata to associate with the text - * @param options Additional options - * @returns The ID of the added item - */ - public async addItem( - text: string, - metadata?: T, - options: { - addToRemote?: boolean // Whether to also add to the remote server if connected - id?: string // Optional ID to use instead of generating a new one - } = {} - ): Promise { - // Use the existing add method with forceEmbed to ensure text is embedded - return this.add(text, metadata, {...options, forceEmbed: true}) - } + if ( + this.loggingConfig?.verbose && + (addedCount > 0 || updatedCount > 0 || deletedCount > 0) + ) { + console.log( + `Real-time update: Added ${addedCount}, updated ${updatedCount}, deleted ${deletedCount} nouns using change log` + ) + } - /** - * Add data to both local and remote Brainy instances - * @param vectorOrData Vector or data to add - * @param metadata Optional metadata to associate with the vector - * @param options Additional options - * @returns The ID of the added vector - */ - public async addToBoth( - vectorOrData: Vector | any, - metadata?: T, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - } = {} - ): Promise { - // Check if connected to a remote server - if (!this.isConnectedToRemoteServer()) { - throw new Error( - 'Not connected to a remote server. Call connectToRemoteServer() first.' + // Update the last known noun count + this.lastKnownNounCount = await this.getNounCount() + } catch (error) { + console.error( + 'Failed to apply changes from log, falling back to full scan:', + error + ) + // Fallback to full scan if change log fails + await this.applyChangesFromFullScan() + } + } + + /** + * Apply changes using full scan method (fallback for storage adapters without change log support) + */ + private async applyChangesFromFullScan(): Promise { + try { + // Get the current noun count + const currentCount = await this.getNounCount() + + // If the noun count has changed, update the index + if (currentCount !== this.lastKnownNounCount) { + // Get all nouns from storage + const nouns = await this.storage!.getAllNouns() + + // Get all nouns currently in the index + const indexNouns = this.index.getNouns() + const indexNounIds = new Set(indexNouns.keys()) + + // Find nouns that are in storage but not in the index + const newNouns = nouns.filter((noun) => !indexNounIds.has(noun.id)) + + // Add new nouns to the index + for (const noun of newNouns) { + // Check if the vector dimensions match the expected dimensions + if (noun.vector.length !== this._dimensions) { + console.warn( + `Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}` ) + continue + } + + // Add to index + await this.index.addItem({ + id: noun.id, + vector: noun.vector + }) + + if (this.loggingConfig?.verbose) { + console.log( + `Added new noun ${noun.id} to index during real-time update` + ) + } } - // Add to local with addToRemote option - return this.add(vectorOrData, metadata, {...options, addToRemote: true}) + // Update the last known noun count + this.lastKnownNounCount = currentCount + + if (this.loggingConfig?.verbose && newNouns.length > 0) { + console.log( + `Real-time update: Added ${newNouns.length} new nouns to index using full scan` + ) + } + } + } catch (error) { + console.error('Failed to apply changes from full scan:', error) + throw error + } + } + + /** + * Get the current augmentation name if available + * This is used to auto-detect the service performing data operations + * @returns The name of the current augmentation or 'default' if none is detected + */ + private getCurrentAugmentation(): string { + try { + // Get all registered augmentations + const augmentationTypes = + augmentationPipeline.getAvailableAugmentationTypes() + + // Check each type of augmentation + for (const type of augmentationTypes) { + const augmentations = augmentationPipeline.getAugmentationsByType(type) + + // Find the first enabled augmentation + for (const augmentation of augmentations) { + if (augmentation.enabled) { + return augmentation.name + } + } + } + + return 'default' + } catch (error) { + // If there's any error in detection, return default + console.warn('Failed to detect current augmentation:', error) + return 'default' + } + } + + /** + * Initialize the database + * Loads existing data from storage if available + */ + public async init(): Promise { + if (this.isInitialized) { + return } - /** - * Add a vector to the remote server - * @param id ID of the vector to add - * @param vector Vector to add - * @param metadata Optional metadata to associate with the vector - * @returns True if successful, false otherwise - * @private - */ - private async addToRemote( - id: string, - vector: Vector, - metadata?: T - ): Promise { - if (!this.isConnectedToRemoteServer()) { - return false - } + // Prevent recursive initialization + if (this.isInitializing) { + return + } + this.isInitializing = true + + try { + // Pre-load the embedding model early to ensure it's always available + // This helps prevent issues with the Universal Sentence Encoder not being loaded + try { + // Pre-loading Universal Sentence Encoder model + // Call embedding function directly to avoid circular dependency with embed() + await this.embeddingFunction('') + // Universal Sentence Encoder model loaded successfully + } catch (embedError) { + console.warn( + 'Failed to pre-load Universal Sentence Encoder:', + embedError + ) + + // Try again with a retry mechanism + // Retrying Universal Sentence Encoder initialization try { - if (!this.serverSearchConduit || !this.serverConnection) { - throw new Error( - 'Server search conduit or connection is not initialized' - ) - } + // Wait a moment before retrying + await new Promise((resolve) => setTimeout(resolve, 1000)) - // Add to remote server - const addResult = await this.serverSearchConduit.addToBoth( - this.serverConnection.connectionId, - vector, - metadata - ) + // Try again with a different approach - use the non-threaded version + // This is a fallback in case the threaded version fails + const { createTensorFlowEmbeddingFunction } = await import( + './utils/embedding.js' + ) + const fallbackEmbeddingFunction = createTensorFlowEmbeddingFunction() - if (!addResult.success) { - throw new Error(`Remote add failed: ${addResult.error}`) - } + // Test the fallback embedding function + await fallbackEmbeddingFunction('') - return true - } catch (error) { - console.error('Failed to add to remote server:', error) - throw new Error(`Failed to add to remote server: ${error}`) + // If successful, replace the embedding function + console.log( + 'Successfully loaded Universal Sentence Encoder with fallback method' + ) + this.embeddingFunction = fallbackEmbeddingFunction + } catch (retryError) { + console.error( + 'All attempts to load Universal Sentence Encoder failed:', + retryError + ) + // Continue initialization even if embedding model fails to load + // The application will need to handle missing embedding functionality } - } + } - /** - * Add multiple vectors or data items to the database - * @param items Array of items to add - * @param options Additional options - * @returns Array of IDs for the added items - */ - public async addBatch( - items: Array<{ - vectorOrData: Vector | any - metadata?: T - }>, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - addToRemote?: boolean // Whether to also add to the remote server if connected - concurrency?: number // Maximum number of concurrent operations (default: 4) - batchSize?: number // Maximum number of items to process in a single batch (default: 50) - } = {} - ): Promise { - await this.ensureInitialized() + // Initialize storage if not provided in constructor + if (!this.storage) { + // Combine storage config with requestPersistentStorage for backward compatibility + let storageOptions = { + ...this.storageConfig, + requestPersistentStorage: this.requestPersistentStorage + } - // Check if database is in read-only mode - this.checkReadOnly() + // Ensure s3Storage has all required fields if it's provided + if (storageOptions.s3Storage) { + // Only include s3Storage if all required fields are present + if ( + storageOptions.s3Storage.bucketName && + storageOptions.s3Storage.accessKeyId && + storageOptions.s3Storage.secretAccessKey + ) { + // All required fields are present, keep s3Storage as is + } else { + // Missing required fields, remove s3Storage to avoid type errors + const { s3Storage, ...rest } = storageOptions + storageOptions = rest + console.warn( + 'Ignoring s3Storage configuration due to missing required fields' + ) + } + } - // Default concurrency to 4 if not specified - const concurrency = options.concurrency || 4 + // Use type assertion to tell TypeScript that storageOptions conforms to StorageOptions + this.storage = await createStorage(storageOptions as any) + } - // Default batch size to 50 if not specified - const batchSize = options.batchSize || 50 + // Initialize storage + await this.storage!.init() + // If using optimized index, set the storage adapter + if (this.useOptimizedIndex && this.index instanceof HNSWIndexOptimized) { + this.index.setStorage(this.storage!) + } + + // In write-only mode, skip loading the index into memory + if (this.writeOnly) { + if (this.loggingConfig?.verbose) { + console.log('Database is in write-only mode, skipping index loading') + } + } else { + // Load all nouns from storage + const nouns: HNSWNoun[] = await this.storage!.getAllNouns() + + // Clear the index and add all nouns + this.index.clear() + for (const noun of nouns) { + // Check if the vector dimensions match the expected dimensions + if (noun.vector.length !== this._dimensions) { + console.warn( + `Deleting noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}` + ) + // Delete the mismatched noun from storage to prevent future issues + await this.storage!.deleteNoun(noun.id) + continue + } + + // Add to index + await this.index.addItem({ + id: noun.id, + vector: noun.vector + }) + } + } + + // Connect to remote server if configured with autoConnect + if (this.remoteServerConfig && this.remoteServerConfig.autoConnect) { try { - // Process items in batches to control concurrency and memory usage - const ids: string[] = [] - const itemsToProcess = [...items] // Create a copy to avoid modifying the original array - - while (itemsToProcess.length > 0) { - // Take up to 'batchSize' items to process in a batch - const batch = itemsToProcess.splice(0, batchSize) - - // Separate items that are already vectors from those that need embedding - const vectorItems: Array<{ - vectorOrData: Vector - metadata?: T - index: number - }> = [] - - const textItems: Array<{ - text: string - metadata?: T - index: number - }> = [] - - // Categorize items - batch.forEach((item, index) => { - if ( - Array.isArray(item.vectorOrData) && - item.vectorOrData.every((val) => typeof val === 'number') && - !options.forceEmbed - ) { - // Item is already a vector - vectorItems.push({ - vectorOrData: item.vectorOrData, - metadata: item.metadata, - index - }) - } else if (typeof item.vectorOrData === 'string') { - // Item is text that needs embedding - textItems.push({ - text: item.vectorOrData, - metadata: item.metadata, - index - }) - } else { - // For now, treat other types as text - // In a more complete implementation, we might handle other types differently - const textRepresentation = String(item.vectorOrData) - textItems.push({ - text: textRepresentation, - metadata: item.metadata, - index - }) - } - }) - - // Process vector items (already embedded) - const vectorPromises = vectorItems.map((item) => - this.add(item.vectorOrData, item.metadata, options) - ) - - // Process text items in a single batch embedding operation - let textPromises: Promise[] = [] - if (textItems.length > 0) { - // Extract just the text for batch embedding - const texts = textItems.map((item) => item.text) - - // Perform batch embedding - const embeddings = await defaultBatchEmbeddingFunction(texts) - - // Add each item with its embedding - textPromises = textItems.map((item, i) => - this.add(embeddings[i], item.metadata, { - ...options, - forceEmbed: false - }) - ) - } - - // Combine all promises - const batchResults = await Promise.all([ - ...vectorPromises, - ...textPromises - ]) - - // Add the results to our ids array - ids.push(...batchResults) - } - - return ids - } catch (error) { - console.error('Failed to add batch of items:', error) - throw new Error(`Failed to add batch of items: ${error}`) + await this.connectToRemoteServer( + this.remoteServerConfig.url, + this.remoteServerConfig.protocols + ) + } catch (remoteError) { + console.warn('Failed to auto-connect to remote server:', remoteError) + // Continue initialization even if remote connection fails } + } + + this.isInitialized = true + this.isInitializing = false + + // Start real-time updates if enabled + this.startRealtimeUpdates() + } catch (error) { + console.error('Failed to initialize BrainyData:', error) + this.isInitializing = false + throw new Error(`Failed to initialize BrainyData: ${error}`) + } + } + + /** + * Connect to a remote Brainy server for search operations + * @param serverUrl WebSocket URL of the remote Brainy server + * @param protocols Optional WebSocket protocols to use + * @returns The connection object + */ + public async connectToRemoteServer( + serverUrl: string, + protocols?: string | string[] + ): Promise { + await this.ensureInitialized() + + try { + // Create server search augmentations + const { conduit, connection } = await createServerSearchAugmentations( + serverUrl, + { + protocols, + localDb: this + } + ) + + // Store the conduit and connection + this.serverSearchConduit = conduit + this.serverConnection = connection + + return connection + } catch (error) { + console.error('Failed to connect to remote server:', error) + throw new Error(`Failed to connect to remote server: ${error}`) + } + } + + /** + * Add a vector or data to the database + * If the input is not a vector, it will be converted using the embedding function + * @param vectorOrData Vector or data to add + * @param metadata Optional metadata to associate with the vector + * @param options Additional options + * @returns The ID of the added vector + */ + public async add( + vectorOrData: Vector | any, + metadata?: T, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + addToRemote?: boolean // Whether to also add to the remote server if connected + id?: string // Optional ID to use instead of generating a new one + service?: string // The service that is inserting the data + } = {} + ): Promise { + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + // Validate input is not null or undefined + if (vectorOrData === null || vectorOrData === undefined) { + throw new Error('Input cannot be null or undefined') } - /** - * Add multiple vectors or data items to both local and remote databases - * @param items Array of items to add - * @param options Additional options - * @returns Array of IDs for the added items - */ - public async addBatchToBoth( - items: Array<{ - vectorOrData: Vector | any - metadata?: T - }>, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - concurrency?: number // Maximum number of concurrent operations (default: 4) - } = {} - ): Promise { - // Check if connected to a remote server - if (!this.isConnectedToRemoteServer()) { - throw new Error( - 'Not connected to a remote server. Call connectToRemoteServer() first.' - ) - } - - // Add to local with addToRemote option - return this.addBatch(items, {...options, addToRemote: true}) - } - - /** - * Filter search results by service - * @param results Search results to filter - * @param service Service to filter by - * @returns Filtered search results - * @private - */ - private filterResultsByService>( - results: R[], - service?: string - ): R[] { - if (!service) return results - - return results.filter(result => { - if (!result.metadata || typeof result.metadata !== 'object') return false - if (!('createdBy' in result.metadata)) return false - - const createdBy = result.metadata.createdBy as any - if (!createdBy) return false - - return createdBy.augmentation === service - }) - } - - /** - * Search for similar vectors within specific noun types - * @param queryVectorOrData Query vector or data to search for - * @param k Number of results to return - * @param nounTypes Array of noun types to search within, or null to search all - * @param options Additional options - * @returns Array of search results - */ - public async searchByNounTypes( - queryVectorOrData: Vector | any, - k: number = 10, - nounTypes: string[] | null = null, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - service?: string // Filter results by the service that created the data - } = {} - ): Promise[]> { - // Helper function to filter results by service - const filterByService = (metadata: any): boolean => { - if (!options.service) return true // No filter, include all - - // Check if metadata has createdBy field with matching service - if (!metadata || typeof metadata !== 'object') return false - if (!('createdBy' in metadata)) return false - - const createdBy = metadata.createdBy as any - if (!createdBy) return false - - return createdBy.augmentation === options.service - } - if (!this.isInitialized) { - throw new Error('BrainyData must be initialized before searching. Call init() first.') + try { + let vector: Vector + + // First validate if input is an array but contains non-numeric values + if (Array.isArray(vectorOrData)) { + for (let i = 0; i < vectorOrData.length; i++) { + if (typeof vectorOrData[i] !== 'number') { + throw new Error('Vector contains non-numeric values') + } } + } + // Check if input is already a vector + if (Array.isArray(vectorOrData) && !options.forceEmbed) { + // Input is already a vector (and we've validated it contains only numbers) + vector = vectorOrData + } else { + // Input needs to be vectorized try { - let queryVector: Vector - - // Check if input is already a vector - if ( - Array.isArray(queryVectorOrData) && - queryVectorOrData.every((item) => typeof item === 'number') && - !options.forceEmbed - ) { - // Input is already a vector - queryVector = queryVectorOrData - } else { - // Input needs to be vectorized - try { - queryVector = await this.embeddingFunction(queryVectorOrData) - } catch (embedError) { - throw new Error(`Failed to vectorize query data: ${embedError}`) - } - } - - // Check if query vector is defined - if (!queryVector) { - throw new Error('Query vector is undefined or null') - } - - // Check if query vector dimensions match the expected dimensions - if (queryVector.length !== this._dimensions) { - throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}`) - } - - // If no noun types specified, search all nouns - if (!nounTypes || nounTypes.length === 0) { - // Search in the index - const results = await this.index.search(queryVector, k) - - // Get metadata for each result - const searchResults: SearchResult[] = [] - - for (const [id, score] of results) { - const noun = this.index.getNouns().get(id) - if (!noun) { - continue - } - - let metadata = await this.storage!.getMetadata(id) - - // Initialize metadata to an empty object if it's null - if (metadata === null) { - metadata = {} as T - } - - // Ensure metadata has the id field - if (metadata && typeof metadata === 'object') { - metadata = {...metadata, id} as T - } - - searchResults.push({ - id, - score, - vector: noun.vector, - metadata: metadata as T - }) - } - - // Filter results by service if specified - return this.filterResultsByService(searchResults, options.service) - } else { - // Get nouns for each noun type in parallel - const nounPromises = nounTypes.map((nounType) => - this.storage!.getNounsByNounType(nounType) - ) - const nounArrays = await Promise.all(nounPromises) - - // Combine all nouns - const nouns: HNSWNoun[] = [] - for (const nounArray of nounArrays) { - nouns.push(...nounArray) - } - - // Calculate distances for each noun - const results: Array<[string, number]> = [] - for (const noun of nouns) { - const distance = this.index.getDistanceFunction()( - queryVector, - noun.vector - ) - results.push([noun.id, distance]) - } - - // Sort by distance (ascending) - results.sort((a, b) => a[1] - b[1]) - - // Take top k results - const topResults = results.slice(0, k) - - // Get metadata for each result - const searchResults: SearchResult[] = [] - - for (const [id, score] of topResults) { - const noun = nouns.find((n) => n.id === id) - if (!noun) { - continue - } - - let metadata = await this.storage!.getMetadata(id) - - // Initialize metadata to an empty object if it's null - if (metadata === null) { - metadata = {} as T - } - - // Ensure metadata has the id field - if (metadata && typeof metadata === 'object') { - metadata = {...metadata, id} as T - } - - searchResults.push({ - id, - score, - vector: noun.vector, - metadata: metadata as T - }) - } - - // Filter results by service if specified - return this.filterResultsByService(searchResults, options.service) - } - } catch (error) { - console.error('Failed to search vectors by noun types:', error) - throw new Error(`Failed to search vectors by noun types: ${error}`) + vector = await this.embeddingFunction(vectorOrData) + } catch (embedError) { + throw new Error(`Failed to vectorize data: ${embedError}`) } - } + } - /** - * Search for similar vectors - * @param queryVectorOrData Query vector or data to search for - * @param k Number of results to return - * @param options Additional options - * @returns Array of search results - */ - public async search( - queryVectorOrData: Vector | any, - k: number = 10, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - nounTypes?: string[] // Optional array of noun types to search within - includeVerbs?: boolean // Whether to include associated GraphVerbs in the results - searchMode?: 'local' | 'remote' | 'combined' // Where to search: local, remote, or both - searchVerbs?: boolean // Whether to search for verbs directly instead of nouns - verbTypes?: string[] // Optional array of verb types to search within or filter by - searchConnectedNouns?: boolean // Whether to search for nouns connected by verbs - verbDirection?: 'outgoing' | 'incoming' | 'both' // Direction of verbs to consider when searching connected nouns - service?: string // Filter results by the service that created the data - } = {} - ): Promise[]> { - // Validate input is not null or undefined - if (queryVectorOrData === null || queryVectorOrData === undefined) { - throw new Error('Query cannot be null or undefined') - } + // Check if vector is defined + if (!vector) { + throw new Error('Vector is undefined or null') + } - // Validate k parameter first, before any other logic - if (k <= 0 || typeof k !== 'number' || isNaN(k)) { - throw new Error('Parameter k must be a positive number') - } - - if (!this.isInitialized) { - throw new Error('BrainyData must be initialized before searching. Call init() first.') - } - // If searching for verbs directly - if (options.searchVerbs) { - const verbResults = await this.searchVerbs(queryVectorOrData, k, { - forceEmbed: options.forceEmbed, - verbTypes: options.verbTypes - }) + // Validate vector dimensions + if (vector.length !== this._dimensions) { + throw new Error( + `Vector dimension mismatch: expected ${this._dimensions}, got ${vector.length}` + ) + } - // Convert verb results to SearchResult format - return verbResults.map((verb) => ({ - id: verb.id, - score: verb.similarity, - vector: verb.embedding || [], - metadata: { - verb: verb.verb, - source: verb.source, - target: verb.target, - ...verb.data - } as unknown as T - })) - } + // Use ID from options if it exists, otherwise from metadata, otherwise generate a new UUID + const id = + options.id || + (metadata && typeof metadata === 'object' && 'id' in metadata + ? (metadata as any).id + : uuidv4()) - // If searching for nouns connected by verbs - if (options.searchConnectedNouns) { - return this.searchNounsByVerbs(queryVectorOrData, k, { - forceEmbed: options.forceEmbed, - verbTypes: options.verbTypes, - direction: options.verbDirection - }) - } + // Add to index + await this.index.addItem({ id, vector }) - // If a specific search mode is specified, use the appropriate search method - if (options.searchMode === 'local') { - return this.searchLocal(queryVectorOrData, k, options) - } else if (options.searchMode === 'remote') { - return this.searchRemote(queryVectorOrData, k, options) - } else if (options.searchMode === 'combined') { - return this.searchCombined(queryVectorOrData, k, options) - } + // Get the noun from the index + const noun = this.index.getNouns().get(id) - // Default behavior (backward compatible): search locally - return this.searchLocal(queryVectorOrData, k, options) - } + if (!noun) { + throw new Error(`Failed to retrieve newly created noun with ID ${id}`) + } - /** - * Search the local database for similar vectors - * @param queryVectorOrData Query vector or data to search for - * @param k Number of results to return - * @param options Additional options - * @returns Array of search results - */ - public async searchLocal( - queryVectorOrData: Vector | any, - k: number = 10, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - nounTypes?: string[] // Optional array of noun types to search within - includeVerbs?: boolean // Whether to include associated GraphVerbs in the results - service?: string // Filter results by the service that created the data - } = {} - ): Promise[]> { - if (!this.isInitialized) { - throw new Error('BrainyData must be initialized before searching. Call init() first.') - } - // If input is a string and not a vector, automatically vectorize it - let queryToUse = queryVectorOrData - if (typeof queryVectorOrData === 'string' && !options.forceEmbed) { - queryToUse = await this.embed(queryVectorOrData) - options.forceEmbed = false // Already embedded, don't force again - } + // Save noun to storage + await this.storage!.saveNoun(noun) - // If noun types are specified, use searchByNounTypes - let searchResults - if (options.nounTypes && options.nounTypes.length > 0) { - searchResults = await this.searchByNounTypes( - queryToUse, - k, - options.nounTypes, - { - forceEmbed: options.forceEmbed, - service: options.service - } - ) + // Track noun statistics + const service = options.service || this.getCurrentAugmentation() + await this.storage!.incrementStatistic('noun', service) + + // Save metadata if provided and not empty + if (metadata !== undefined) { + // Skip saving if metadata is an empty object + if ( + metadata && + typeof metadata === 'object' && + Object.keys(metadata).length === 0 + ) { + // Don't save empty metadata + // Explicitly save null to ensure no metadata is stored + await this.storage!.saveMetadata(id, null) } else { - // Otherwise, search all GraphNouns - searchResults = await this.searchByNounTypes(queryToUse, k, null, { - forceEmbed: options.forceEmbed, - service: options.service - }) - } + // Validate noun type if metadata is for a GraphNoun + if (metadata && typeof metadata === 'object' && 'noun' in metadata) { + const nounType = (metadata as unknown as GraphNoun).noun - // If includeVerbs is true, retrieve associated GraphVerbs for each result - if (options.includeVerbs && this.storage) { - for (const result of searchResults) { - try { - // Get outgoing verbs for this noun - const outgoingVerbs = await this.storage.getVerbsBySource(result.id) + // Check if the noun type is valid + const isValidNounType = Object.values(NounType).includes(nounType) - // Get incoming verbs for this noun - const incomingVerbs = await this.storage.getVerbsByTarget(result.id) - - // Combine all verbs - const allVerbs = [...outgoingVerbs, ...incomingVerbs] - - // Add verbs to the result metadata - if (!result.metadata) { - result.metadata = {} as T - } - - // Add the verbs to the metadata - ;(result.metadata as Record).associatedVerbs = allVerbs - } catch (error) { - console.warn(`Failed to retrieve verbs for noun ${result.id}:`, error) - } - } - } - - return searchResults - } - - /** - * Find entities similar to a given entity ID - * @param id ID of the entity to find similar entities for - * @param options Additional options - * @returns Array of search results with similarity scores - */ - public async findSimilar( - id: string, - options: { - limit?: number // Number of results to return - nounTypes?: string[] // Optional array of noun types to search within - includeVerbs?: boolean // Whether to include associated GraphVerbs in the results - searchMode?: 'local' | 'remote' | 'combined' // Where to search: local, remote, or both - relationType?: string // Optional relationship type to filter by - } = {} - ): Promise[]> { - await this.ensureInitialized() - - // Get the entity by ID - const entity = await this.get(id) - if (!entity) { - throw new Error(`Entity with ID ${id} not found`) - } - - // If relationType is specified, directly get related entities by that type - if (options.relationType) { - // Get all verbs (relationships) from the source entity - const outgoingVerbs = await this.storage!.getVerbsBySource(id) - - // Filter to only include verbs of the specified type - const verbsOfType = outgoingVerbs.filter(verb => verb.type === options.relationType) - - // Get the target IDs - const targetIds = verbsOfType.map(verb => verb.target) - - // Get the actual entities for these IDs - const results: SearchResult[] = [] - for (const targetId of targetIds) { - // Skip undefined targetIds - if (typeof targetId !== 'string') continue - - const targetEntity = await this.get(targetId) - if (targetEntity) { - results.push({ - id: targetId, - score: 1.0, // Default similarity score - vector: targetEntity.vector, - metadata: targetEntity.metadata - }) - } - } - - // Return the results, limited to the requested number - return results.slice(0, options.limit || 10) - } - - // If no relationType is specified, use the original vector similarity search - const k = (options.limit || 10) + 1 // Add 1 to account for the original entity - const searchResults = await this.search(entity.vector, k, { - forceEmbed: false, - nounTypes: options.nounTypes, - includeVerbs: options.includeVerbs, - searchMode: options.searchMode - }) - - // Filter out the original entity and limit to the requested number - return searchResults - .filter((result) => result.id !== id) - .slice(0, options.limit || 10) - } - - /** - * Get a vector by ID - */ - public async get(id: string): Promise | null> { - // Validate id parameter first, before any other logic - if (id === null || id === undefined) { - throw new Error('ID cannot be null or undefined') - } - - await this.ensureInitialized() - - try { - // Get noun from index - const noun = this.index.getNouns().get(id) - if (!noun) { - return null + if (!isValidNounType) { + console.warn( + `Invalid noun type: ${nounType}. Falling back to GraphNoun.` + ) + // Set a default noun type + ;(metadata as unknown as GraphNoun).noun = NounType.Concept } - // Get metadata - let metadata = await this.storage!.getMetadata(id) - - // Handle special cases for metadata - if (metadata === null) { - metadata = {} - } else if (typeof metadata === 'object') { - // For empty metadata test: if metadata only has an ID, return empty object - if (Object.keys(metadata).length === 1 && 'id' in metadata) { - metadata = {} - } - // Always remove the ID from metadata if present - else if ('id' in metadata) { - const { id: _, ...rest } = metadata - metadata = rest - } - } - - return { - id, - vector: noun.vector, - metadata: metadata as T | undefined - } - } catch (error) { - console.error(`Failed to get vector ${id}:`, error) - throw new Error(`Failed to get vector ${id}: ${error}`) - } - } - - /** - * Get all nouns in the database - * @returns Array of vector documents - */ - public async getAllNouns(): Promise[]> { - await this.ensureInitialized() - - try { - const nouns = this.index.getNouns() - const result: VectorDocument[] = [] - - for (const [id, noun] of nouns.entries()) { - const metadata = await this.storage!.getMetadata(id) - result.push({ - id, - vector: noun.vector, - metadata: metadata as T | undefined - }) - } - - return result - } catch (error) { - console.error('Failed to get all nouns:', error) - throw new Error(`Failed to get all nouns: ${error}`) - } - } - - /** - * Delete a vector by ID - * @param id The ID of the vector to delete - * @param options Additional options - * @returns Promise that resolves to true if the vector was deleted, false otherwise - */ - public async delete( - id: string, - options: { - service?: string // The service that is deleting the data - } = {} - ): Promise { - // Validate id parameter first, before any other logic - if (id === null || id === undefined) { - throw new Error('ID cannot be null or undefined') - } - - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() - - try { - // Remove from index - const removed = this.index.removeItem(id) - if (!removed) { - return false - } - - // Remove from storage - await this.storage!.deleteNoun(id) - - // Track deletion statistics - const service = options.service || 'default' - await this.storage!.decrementStatistic('noun', service) - - // Try to remove metadata (ignore errors) - try { - await this.storage!.saveMetadata(id, null) - await this.storage!.decrementStatistic('metadata', service) - } catch (error) { - // Ignore - } - - return true - } catch (error) { - console.error(`Failed to delete vector ${id}:`, error) - throw new Error(`Failed to delete vector ${id}: ${error}`) - } - } - - /** - * Update metadata for a vector - * @param id The ID of the vector to update metadata for - * @param metadata The new metadata - * @param options Additional options - * @returns Promise that resolves to true if the metadata was updated, false otherwise - */ - public async updateMetadata( - id: string, - metadata: T, - options: { - service?: string // The service that is updating the data - } = {} - ): Promise { - // Validate id parameter first, before any other logic - if (id === null || id === undefined) { - throw new Error('ID cannot be null or undefined') - } - - // Validate that metadata is not null or undefined - if (metadata === null || metadata === undefined) { - throw new Error(`Metadata cannot be null or undefined`) - } - - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() - - try { - // Check if a vector exists - const noun = this.index.getNouns().get(id) - if (!noun) { - throw new Error(`Vector with ID ${id} does not exist`) - } - - // Validate noun type if metadata is for a GraphNoun - if (metadata && typeof metadata === 'object' && 'noun' in metadata) { - const nounType = (metadata as unknown as GraphNoun).noun - - // Check if the noun type is valid - const isValidNounType = Object.values(NounType).includes(nounType) - - if (!isValidNounType) { - console.warn( - `Invalid noun type: ${nounType}. Falling back to GraphNoun.` - ) - // Set a default noun type - ;(metadata as unknown as GraphNoun).noun = NounType.Concept - } - - // Get the service that's updating the metadata - const service = options.service || this.getCurrentAugmentation() - const graphNoun = metadata as unknown as GraphNoun - - // Preserve existing createdBy and createdAt if they exist - const existingMetadata = await this.storage!.getMetadata(id) as any - - if (existingMetadata && - typeof existingMetadata === 'object' && - 'createdBy' in existingMetadata) { - // Preserve the original creator information - graphNoun.createdBy = existingMetadata.createdBy - - // Also preserve creation timestamp if it exists - if ('createdAt' in existingMetadata) { - graphNoun.createdAt = existingMetadata.createdAt - } - } else if (!graphNoun.createdBy) { - // If no existing createdBy and none in the update, set it - graphNoun.createdBy = { - augmentation: service, - version: '1.0' // TODO: Get actual version from augmentation - } - - // Set createdAt if it doesn't exist - if (!graphNoun.createdAt) { - const now = new Date() - graphNoun.createdAt = { - seconds: Math.floor(now.getTime() / 1000), - nanoseconds: (now.getTime() % 1000) * 1000000 - } - } - } - - // Always update the updatedAt timestamp - const now = new Date() - graphNoun.updatedAt = { - seconds: Math.floor(now.getTime() / 1000), - nanoseconds: (now.getTime() % 1000) * 1000000 - } - } - - // Update metadata - await this.storage!.saveMetadata(id, metadata) - - // Track metadata statistics + // Ensure createdBy field is populated for GraphNoun const service = options.service || this.getCurrentAugmentation() - await this.storage!.incrementStatistic('metadata', service) + const graphNoun = metadata as unknown as GraphNoun - return true - } catch (error) { - console.error(`Failed to update metadata for vector ${id}:`, error) - throw new Error(`Failed to update metadata for vector ${id}: ${error}`) - } - } - - /** - * Create a relationship between two entities - * This is a convenience wrapper around addVerb - */ - public async relate( - sourceId: string, - targetId: string, - relationType: string, - metadata?: any - ): Promise { - // Validate inputs are not null or undefined - if (sourceId === null || sourceId === undefined) { - throw new Error('Source ID cannot be null or undefined') - } - if (targetId === null || targetId === undefined) { - throw new Error('Target ID cannot be null or undefined') - } - if (relationType === null || relationType === undefined) { - throw new Error('Relation type cannot be null or undefined') - } - - return this.addVerb(sourceId, targetId, undefined, { - type: relationType, - metadata: metadata - }) - } - - /** - * Create a connection between two entities - * This is an alias for relate() for backward compatibility - */ - public async connect( - sourceId: string, - targetId: string, - relationType: string, - metadata?: any - ): Promise { - return this.relate(sourceId, targetId, relationType, metadata) - } - - /** - * Add a verb between two nouns - * If metadata is provided and vector is not, the metadata will be vectorized using the embedding function - * - * @param sourceId ID of the source noun - * @param targetId ID of the target noun - * @param vector Optional vector for the verb - * @param options Additional options: - * - type: Type of the verb - * - weight: Weight of the verb - * - metadata: Metadata for the verb - * - forceEmbed: Force using the embedding function for metadata even if vector is provided - * - id: Optional ID to use instead of generating a new one - * - autoCreateMissingNouns: Automatically create missing nouns if they don't exist - * - missingNounMetadata: Metadata to use when auto-creating missing nouns - * - * @returns The ID of the added verb - * - * @throws Error if source or target nouns don't exist and autoCreateMissingNouns is false or auto-creation fails - */ - public async addVerb( - sourceId: string, - targetId: string, - vector?: Vector, - options: { - type?: string - weight?: number - metadata?: any - forceEmbed?: boolean // Force using the embedding function for metadata even if vector is provided - id?: string // Optional ID to use instead of generating a new one - autoCreateMissingNouns?: boolean // Automatically create missing nouns - missingNounMetadata?: any // Metadata to use when auto-creating missing nouns - service?: string // The service that is inserting the data - } = {} - ): Promise { - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() - - // Validate inputs are not null or undefined - if (sourceId === null || sourceId === undefined) { - throw new Error('Source ID cannot be null or undefined') - } - if (targetId === null || targetId === undefined) { - throw new Error('Target ID cannot be null or undefined') - } - - try { - // Check if source and target nouns exist - let sourceNoun = this.index.getNouns().get(sourceId) - let targetNoun = this.index.getNouns().get(targetId) - - // Auto-create missing nouns if option is enabled - if (!sourceNoun && options.autoCreateMissingNouns) { - try { - // Create a placeholder vector for the missing noun - const placeholderVector = new Array(this._dimensions).fill(0) - - // Add metadata if provided - const service = options.service || this.getCurrentAugmentation() - const now = new Date() - const timestamp = { - seconds: Math.floor(now.getTime() / 1000), - nanoseconds: (now.getTime() % 1000) * 1000000 - } - - const metadata = options.missingNounMetadata || { - autoCreated: true, - createdAt: timestamp, - updatedAt: timestamp, - noun: NounType.Concept, - createdBy: { - augmentation: service, - version: '1.0' // TODO: Get actual version from augmentation - } - } - - // Add the missing noun - await this.add(placeholderVector, metadata, {id: sourceId}) - - // Get the newly created noun - sourceNoun = this.index.getNouns().get(sourceId) - - console.warn(`Auto-created missing source noun with ID ${sourceId}`) - } catch (createError) { - console.error(`Failed to auto-create source noun with ID ${sourceId}:`, createError) - throw new Error(`Failed to auto-create source noun with ID ${sourceId}: ${createError}`) - } + // Only set createdBy if it doesn't exist or is being explicitly updated + if (!graphNoun.createdBy || options.service) { + graphNoun.createdBy = { + augmentation: service, + version: '1.0' // TODO: Get actual version from augmentation + } } - if (!targetNoun && options.autoCreateMissingNouns) { - try { - // Create a placeholder vector for the missing noun - const placeholderVector = new Array(this._dimensions).fill(0) - - // Add metadata if provided - const service = options.service || this.getCurrentAugmentation() - const now = new Date() - const timestamp = { - seconds: Math.floor(now.getTime() / 1000), - nanoseconds: (now.getTime() % 1000) * 1000000 - } - - const metadata = options.missingNounMetadata || { - autoCreated: true, - createdAt: timestamp, - updatedAt: timestamp, - noun: NounType.Concept, - createdBy: { - augmentation: service, - version: '1.0' // TODO: Get actual version from augmentation - } - } - - // Add the missing noun - await this.add(placeholderVector, metadata, {id: targetId}) - - // Get the newly created noun - targetNoun = this.index.getNouns().get(targetId) - - console.warn(`Auto-created missing target noun with ID ${targetId}`) - } catch (createError) { - console.error(`Failed to auto-create target noun with ID ${targetId}:`, createError) - throw new Error(`Failed to auto-create target noun with ID ${targetId}: ${createError}`) - } - } - - if (!sourceNoun) { - throw new Error(`Source noun with ID ${sourceId} not found`) - } - - if (!targetNoun) { - throw new Error(`Target noun with ID ${targetId} not found`) - } - - // Use provided ID or generate a new one - const id = options.id || uuidv4() - - let verbVector: Vector - - // If metadata is provided and no vector is provided or forceEmbed is true, vectorize the metadata - if (options.metadata && (!vector || options.forceEmbed)) { - try { - // Extract a string representation from metadata for embedding - let textToEmbed: string - if (typeof options.metadata === 'string') { - textToEmbed = options.metadata - } else if ( - options.metadata.description && - typeof options.metadata.description === 'string' - ) { - textToEmbed = options.metadata.description - } else { - // Convert to JSON string as fallback - textToEmbed = JSON.stringify(options.metadata) - } - - // Ensure textToEmbed is a string - if (typeof textToEmbed !== 'string') { - textToEmbed = String(textToEmbed) - } - - verbVector = await this.embeddingFunction(textToEmbed) - } catch (embedError) { - throw new Error(`Failed to vectorize verb metadata: ${embedError}`) - } - } else { - // Use a provided vector or average of source and target vectors - if (vector) { - verbVector = vector - } else { - // Ensure both source and target vectors have the same dimension - if ( - !sourceNoun.vector || - !targetNoun.vector || - sourceNoun.vector.length === 0 || - targetNoun.vector.length === 0 || - sourceNoun.vector.length !== targetNoun.vector.length - ) { - throw new Error( - `Cannot average vectors: source or target vector is invalid or dimensions don't match` - ) - } - - // Average the vectors - verbVector = sourceNoun.vector.map( - (val, i) => (val + targetNoun.vector[i]) / 2 - ) - } - } - - // Validate verb type if provided - let verbType = options.type - if (!verbType) { - // If no verb type is provided, use RelatedTo as default - verbType = VerbType.RelatedTo - } - // Note: We're no longer validating against VerbType enum to allow custom relationship types - - // Get service name from options or current augmentation - const service = options.service || this.getCurrentAugmentation() - - // Create timestamp for creation/update time + // Update timestamps const now = new Date() const timestamp = { - seconds: Math.floor(now.getTime() / 1000), - nanoseconds: (now.getTime() % 1000) * 1000000 + seconds: Math.floor(now.getTime() / 1000), + nanoseconds: (now.getTime() % 1000) * 1000000 } - // Create verb - const verb: GraphVerb = { - id, - vector: verbVector, - connections: new Map(), - sourceId: sourceId, - targetId: targetId, - source: sourceId, - target: targetId, - verb: verbType as VerbType, - type: verbType, // Set the type property to match the verb type - weight: options.weight, - metadata: options.metadata, - createdAt: timestamp, - updatedAt: timestamp, - createdBy: { - augmentation: service, - version: '1.0' // TODO: Get actual version from augmentation - } + // Set createdAt if it doesn't exist + if (!graphNoun.createdAt) { + graphNoun.createdAt = timestamp } - // Add to index - await this.index.addItem({id, vector: verbVector}) + // Always update updatedAt + graphNoun.updatedAt = timestamp + } - // Get the noun from the index - const indexNoun = this.index.getNouns().get(id) + // Create a copy of the metadata without modifying the original + let metadataToSave = metadata + if (metadata && typeof metadata === 'object') { + // Always make a copy without adding the ID + metadataToSave = { ...metadata } + } - if (!indexNoun) { - throw new Error( - `Failed to retrieve newly created verb noun with ID ${id}` - ) + await this.storage!.saveMetadata(id, metadataToSave) + + // Track metadata statistics + const metadataService = + options.service || this.getCurrentAugmentation() + await this.storage!.incrementStatistic('metadata', metadataService) + } + } + + // Update HNSW index size (excluding verbs) + await this.storage!.updateHnswIndexSize(await this.getNounCount()) + + // If addToRemote is true and we're connected to a remote server, add to remote as well + if (options.addToRemote && this.isConnectedToRemoteServer()) { + try { + await this.addToRemote(id, vector, metadata) + } catch (remoteError) { + console.warn( + `Failed to add to remote server: ${remoteError}. Continuing with local add.` + ) + } + } + + return id + } catch (error) { + console.error('Failed to add vector:', error) + throw new Error(`Failed to add vector: ${error}`) + } + } + + /** + * Add a text item to the database with automatic embedding + * This is a convenience method for adding text data with metadata + * @param text Text data to add + * @param metadata Metadata to associate with the text + * @param options Additional options + * @returns The ID of the added item + */ + public async addItem( + text: string, + metadata?: T, + options: { + addToRemote?: boolean // Whether to also add to the remote server if connected + id?: string // Optional ID to use instead of generating a new one + } = {} + ): Promise { + // Use the existing add method with forceEmbed to ensure text is embedded + return this.add(text, metadata, { ...options, forceEmbed: true }) + } + + /** + * Add data to both local and remote Brainy instances + * @param vectorOrData Vector or data to add + * @param metadata Optional metadata to associate with the vector + * @param options Additional options + * @returns The ID of the added vector + */ + public async addToBoth( + vectorOrData: Vector | any, + metadata?: T, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + } = {} + ): Promise { + // Check if connected to a remote server + if (!this.isConnectedToRemoteServer()) { + throw new Error( + 'Not connected to a remote server. Call connectToRemoteServer() first.' + ) + } + + // Add to local with addToRemote option + return this.add(vectorOrData, metadata, { ...options, addToRemote: true }) + } + + /** + * Add a vector to the remote server + * @param id ID of the vector to add + * @param vector Vector to add + * @param metadata Optional metadata to associate with the vector + * @returns True if successful, false otherwise + * @private + */ + private async addToRemote( + id: string, + vector: Vector, + metadata?: T + ): Promise { + if (!this.isConnectedToRemoteServer()) { + return false + } + + try { + if (!this.serverSearchConduit || !this.serverConnection) { + throw new Error( + 'Server search conduit or connection is not initialized' + ) + } + + // Add to remote server + const addResult = await this.serverSearchConduit.addToBoth( + this.serverConnection.connectionId, + vector, + metadata + ) + + if (!addResult.success) { + throw new Error(`Remote add failed: ${addResult.error}`) + } + + return true + } catch (error) { + console.error('Failed to add to remote server:', error) + throw new Error(`Failed to add to remote server: ${error}`) + } + } + + /** + * Add multiple vectors or data items to the database + * @param items Array of items to add + * @param options Additional options + * @returns Array of IDs for the added items + */ + public async addBatch( + items: Array<{ + vectorOrData: Vector | any + metadata?: T + }>, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + addToRemote?: boolean // Whether to also add to the remote server if connected + concurrency?: number // Maximum number of concurrent operations (default: 4) + batchSize?: number // Maximum number of items to process in a single batch (default: 50) + } = {} + ): Promise { + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + // Default concurrency to 4 if not specified + const concurrency = options.concurrency || 4 + + // Default batch size to 50 if not specified + const batchSize = options.batchSize || 50 + + try { + // Process items in batches to control concurrency and memory usage + const ids: string[] = [] + const itemsToProcess = [...items] // Create a copy to avoid modifying the original array + + while (itemsToProcess.length > 0) { + // Take up to 'batchSize' items to process in a batch + const batch = itemsToProcess.splice(0, batchSize) + + // Separate items that are already vectors from those that need embedding + const vectorItems: Array<{ + vectorOrData: Vector + metadata?: T + index: number + }> = [] + + const textItems: Array<{ + text: string + metadata?: T + index: number + }> = [] + + // Categorize items + batch.forEach((item, index) => { + if ( + Array.isArray(item.vectorOrData) && + item.vectorOrData.every((val) => typeof val === 'number') && + !options.forceEmbed + ) { + // Item is already a vector + vectorItems.push({ + vectorOrData: item.vectorOrData, + metadata: item.metadata, + index + }) + } else if (typeof item.vectorOrData === 'string') { + // Item is text that needs embedding + textItems.push({ + text: item.vectorOrData, + metadata: item.metadata, + index + }) + } else { + // For now, treat other types as text + // In a more complete implementation, we might handle other types differently + const textRepresentation = String(item.vectorOrData) + textItems.push({ + text: textRepresentation, + metadata: item.metadata, + index + }) + } + }) + + // Process vector items (already embedded) + const vectorPromises = vectorItems.map((item) => + this.add(item.vectorOrData, item.metadata, options) + ) + + // Process text items in a single batch embedding operation + let textPromises: Promise[] = [] + if (textItems.length > 0) { + // Extract just the text for batch embedding + const texts = textItems.map((item) => item.text) + + // Perform batch embedding + const embeddings = await defaultBatchEmbeddingFunction(texts) + + // Add each item with its embedding + textPromises = textItems.map((item, i) => + this.add(embeddings[i], item.metadata, { + ...options, + forceEmbed: false + }) + ) + } + + // Combine all promises + const batchResults = await Promise.all([ + ...vectorPromises, + ...textPromises + ]) + + // Add the results to our ids array + ids.push(...batchResults) + } + + return ids + } catch (error) { + console.error('Failed to add batch of items:', error) + throw new Error(`Failed to add batch of items: ${error}`) + } + } + + /** + * Add multiple vectors or data items to both local and remote databases + * @param items Array of items to add + * @param options Additional options + * @returns Array of IDs for the added items + */ + public async addBatchToBoth( + items: Array<{ + vectorOrData: Vector | any + metadata?: T + }>, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + concurrency?: number // Maximum number of concurrent operations (default: 4) + } = {} + ): Promise { + // Check if connected to a remote server + if (!this.isConnectedToRemoteServer()) { + throw new Error( + 'Not connected to a remote server. Call connectToRemoteServer() first.' + ) + } + + // Add to local with addToRemote option + return this.addBatch(items, { ...options, addToRemote: true }) + } + + /** + * Filter search results by service + * @param results Search results to filter + * @param service Service to filter by + * @returns Filtered search results + * @private + */ + private filterResultsByService>( + results: R[], + service?: string + ): R[] { + if (!service) return results + + return results.filter((result) => { + if (!result.metadata || typeof result.metadata !== 'object') return false + if (!('createdBy' in result.metadata)) return false + + const createdBy = result.metadata.createdBy as any + if (!createdBy) return false + + return createdBy.augmentation === service + }) + } + + /** + * Search for similar vectors within specific noun types + * @param queryVectorOrData Query vector or data to search for + * @param k Number of results to return + * @param nounTypes Array of noun types to search within, or null to search all + * @param options Additional options + * @returns Array of search results + */ + public async searchByNounTypes( + queryVectorOrData: Vector | any, + k: number = 10, + nounTypes: string[] | null = null, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + service?: string // Filter results by the service that created the data + } = {} + ): Promise[]> { + // Helper function to filter results by service + const filterByService = (metadata: any): boolean => { + if (!options.service) return true // No filter, include all + + // Check if metadata has createdBy field with matching service + if (!metadata || typeof metadata !== 'object') return false + if (!('createdBy' in metadata)) return false + + const createdBy = metadata.createdBy as any + if (!createdBy) return false + + return createdBy.augmentation === options.service + } + if (!this.isInitialized) { + throw new Error( + 'BrainyData must be initialized before searching. Call init() first.' + ) + } + + // Check if database is in write-only mode + this.checkWriteOnly() + + try { + let queryVector: Vector + + // Check if input is already a vector + if ( + Array.isArray(queryVectorOrData) && + queryVectorOrData.every((item) => typeof item === 'number') && + !options.forceEmbed + ) { + // Input is already a vector + queryVector = queryVectorOrData + } else { + // Input needs to be vectorized + try { + queryVector = await this.embeddingFunction(queryVectorOrData) + } catch (embedError) { + throw new Error(`Failed to vectorize query data: ${embedError}`) + } + } + + // Check if query vector is defined + if (!queryVector) { + throw new Error('Query vector is undefined or null') + } + + // Check if query vector dimensions match the expected dimensions + if (queryVector.length !== this._dimensions) { + throw new Error( + `Query vector dimension mismatch: expected ${this._dimensions}, got ${queryVector.length}` + ) + } + + // If no noun types specified, search all nouns + if (!nounTypes || nounTypes.length === 0) { + // Search in the index + const results = await this.index.search(queryVector, k) + + // Get metadata for each result + const searchResults: SearchResult[] = [] + + for (const [id, score] of results) { + const noun = this.index.getNouns().get(id) + if (!noun) { + continue + } + + let metadata = await this.storage!.getMetadata(id) + + // Initialize metadata to an empty object if it's null + if (metadata === null) { + metadata = {} as T + } + + // Ensure metadata has the id field + if (metadata && typeof metadata === 'object') { + metadata = { ...metadata, id } as T + } + + searchResults.push({ + id, + score, + vector: noun.vector, + metadata: metadata as T + }) + } + + // Filter results by service if specified + return this.filterResultsByService(searchResults, options.service) + } else { + // Get nouns for each noun type in parallel + const nounPromises = nounTypes.map((nounType) => + this.storage!.getNounsByNounType(nounType) + ) + const nounArrays = await Promise.all(nounPromises) + + // Combine all nouns + const nouns: HNSWNoun[] = [] + for (const nounArray of nounArrays) { + nouns.push(...nounArray) + } + + // Calculate distances for each noun + const results: Array<[string, number]> = [] + for (const noun of nouns) { + const distance = this.index.getDistanceFunction()( + queryVector, + noun.vector + ) + results.push([noun.id, distance]) + } + + // Sort by distance (ascending) + results.sort((a, b) => a[1] - b[1]) + + // Take top k results + const topResults = results.slice(0, k) + + // Get metadata for each result + const searchResults: SearchResult[] = [] + + for (const [id, score] of topResults) { + const noun = nouns.find((n) => n.id === id) + if (!noun) { + continue + } + + let metadata = await this.storage!.getMetadata(id) + + // Initialize metadata to an empty object if it's null + if (metadata === null) { + metadata = {} as T + } + + // Ensure metadata has the id field + if (metadata && typeof metadata === 'object') { + metadata = { ...metadata, id } as T + } + + searchResults.push({ + id, + score, + vector: noun.vector, + metadata: metadata as T + }) + } + + // Filter results by service if specified + return this.filterResultsByService(searchResults, options.service) + } + } catch (error) { + console.error('Failed to search vectors by noun types:', error) + throw new Error(`Failed to search vectors by noun types: ${error}`) + } + } + + /** + * Search for similar vectors + * @param queryVectorOrData Query vector or data to search for + * @param k Number of results to return + * @param options Additional options + * @returns Array of search results + */ + public async search( + queryVectorOrData: Vector | any, + k: number = 10, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + nounTypes?: string[] // Optional array of noun types to search within + includeVerbs?: boolean // Whether to include associated GraphVerbs in the results + searchMode?: 'local' | 'remote' | 'combined' // Where to search: local, remote, or both + searchVerbs?: boolean // Whether to search for verbs directly instead of nouns + verbTypes?: string[] // Optional array of verb types to search within or filter by + searchConnectedNouns?: boolean // Whether to search for nouns connected by verbs + verbDirection?: 'outgoing' | 'incoming' | 'both' // Direction of verbs to consider when searching connected nouns + service?: string // Filter results by the service that created the data + } = {} + ): Promise[]> { + // Validate input is not null or undefined + if (queryVectorOrData === null || queryVectorOrData === undefined) { + throw new Error('Query cannot be null or undefined') + } + + // Validate k parameter first, before any other logic + if (k <= 0 || typeof k !== 'number' || isNaN(k)) { + throw new Error('Parameter k must be a positive number') + } + + if (!this.isInitialized) { + throw new Error( + 'BrainyData must be initialized before searching. Call init() first.' + ) + } + + // Check if database is in write-only mode + this.checkWriteOnly() + // If searching for verbs directly + if (options.searchVerbs) { + const verbResults = await this.searchVerbs(queryVectorOrData, k, { + forceEmbed: options.forceEmbed, + verbTypes: options.verbTypes + }) + + // Convert verb results to SearchResult format + return verbResults.map((verb) => ({ + id: verb.id, + score: verb.similarity, + vector: verb.embedding || [], + metadata: { + verb: verb.verb, + source: verb.source, + target: verb.target, + ...verb.data + } as unknown as T + })) + } + + // If searching for nouns connected by verbs + if (options.searchConnectedNouns) { + return this.searchNounsByVerbs(queryVectorOrData, k, { + forceEmbed: options.forceEmbed, + verbTypes: options.verbTypes, + direction: options.verbDirection + }) + } + + // If a specific search mode is specified, use the appropriate search method + if (options.searchMode === 'local') { + return this.searchLocal(queryVectorOrData, k, options) + } else if (options.searchMode === 'remote') { + return this.searchRemote(queryVectorOrData, k, options) + } else if (options.searchMode === 'combined') { + return this.searchCombined(queryVectorOrData, k, options) + } + + // Default behavior (backward compatible): search locally + return this.searchLocal(queryVectorOrData, k, options) + } + + /** + * Search the local database for similar vectors + * @param queryVectorOrData Query vector or data to search for + * @param k Number of results to return + * @param options Additional options + * @returns Array of search results + */ + public async searchLocal( + queryVectorOrData: Vector | any, + k: number = 10, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + nounTypes?: string[] // Optional array of noun types to search within + includeVerbs?: boolean // Whether to include associated GraphVerbs in the results + service?: string // Filter results by the service that created the data + } = {} + ): Promise[]> { + if (!this.isInitialized) { + throw new Error( + 'BrainyData must be initialized before searching. Call init() first.' + ) + } + + // Check if database is in write-only mode + this.checkWriteOnly() + // If input is a string and not a vector, automatically vectorize it + let queryToUse = queryVectorOrData + if (typeof queryVectorOrData === 'string' && !options.forceEmbed) { + queryToUse = await this.embed(queryVectorOrData) + options.forceEmbed = false // Already embedded, don't force again + } + + // If noun types are specified, use searchByNounTypes + let searchResults + if (options.nounTypes && options.nounTypes.length > 0) { + searchResults = await this.searchByNounTypes( + queryToUse, + k, + options.nounTypes, + { + forceEmbed: options.forceEmbed, + service: options.service + } + ) + } else { + // Otherwise, search all GraphNouns + searchResults = await this.searchByNounTypes(queryToUse, k, null, { + forceEmbed: options.forceEmbed, + service: options.service + }) + } + + // If includeVerbs is true, retrieve associated GraphVerbs for each result + if (options.includeVerbs && this.storage) { + for (const result of searchResults) { + try { + // Get outgoing verbs for this noun + const outgoingVerbs = await this.storage.getVerbsBySource(result.id) + + // Get incoming verbs for this noun + const incomingVerbs = await this.storage.getVerbsByTarget(result.id) + + // Combine all verbs + const allVerbs = [...outgoingVerbs, ...incomingVerbs] + + // Add verbs to the result metadata + if (!result.metadata) { + result.metadata = {} as T + } + + // Add the verbs to the metadata + ;(result.metadata as Record).associatedVerbs = allVerbs + } catch (error) { + console.warn(`Failed to retrieve verbs for noun ${result.id}:`, error) + } + } + } + + return searchResults + } + + /** + * Find entities similar to a given entity ID + * @param id ID of the entity to find similar entities for + * @param options Additional options + * @returns Array of search results with similarity scores + */ + public async findSimilar( + id: string, + options: { + limit?: number // Number of results to return + nounTypes?: string[] // Optional array of noun types to search within + includeVerbs?: boolean // Whether to include associated GraphVerbs in the results + searchMode?: 'local' | 'remote' | 'combined' // Where to search: local, remote, or both + relationType?: string // Optional relationship type to filter by + } = {} + ): Promise[]> { + await this.ensureInitialized() + + // Get the entity by ID + const entity = await this.get(id) + if (!entity) { + throw new Error(`Entity with ID ${id} not found`) + } + + // If relationType is specified, directly get related entities by that type + if (options.relationType) { + // Get all verbs (relationships) from the source entity + const outgoingVerbs = await this.storage!.getVerbsBySource(id) + + // Filter to only include verbs of the specified type + const verbsOfType = outgoingVerbs.filter( + (verb) => verb.type === options.relationType + ) + + // Get the target IDs + const targetIds = verbsOfType.map((verb) => verb.target) + + // Get the actual entities for these IDs + const results: SearchResult[] = [] + for (const targetId of targetIds) { + // Skip undefined targetIds + if (typeof targetId !== 'string') continue + + const targetEntity = await this.get(targetId) + if (targetEntity) { + results.push({ + id: targetId, + score: 1.0, // Default similarity score + vector: targetEntity.vector, + metadata: targetEntity.metadata + }) + } + } + + // Return the results, limited to the requested number + return results.slice(0, options.limit || 10) + } + + // If no relationType is specified, use the original vector similarity search + const k = (options.limit || 10) + 1 // Add 1 to account for the original entity + const searchResults = await this.search(entity.vector, k, { + forceEmbed: false, + nounTypes: options.nounTypes, + includeVerbs: options.includeVerbs, + searchMode: options.searchMode + }) + + // Filter out the original entity and limit to the requested number + return searchResults + .filter((result) => result.id !== id) + .slice(0, options.limit || 10) + } + + /** + * Get a vector by ID + */ + public async get(id: string): Promise | null> { + // Validate id parameter first, before any other logic + if (id === null || id === undefined) { + throw new Error('ID cannot be null or undefined') + } + + await this.ensureInitialized() + + try { + // Get noun from index + const noun = this.index.getNouns().get(id) + if (!noun) { + return null + } + + // Get metadata + let metadata = await this.storage!.getMetadata(id) + + // Handle special cases for metadata + if (metadata === null) { + metadata = {} + } else if (typeof metadata === 'object') { + // For empty metadata test: if metadata only has an ID, return empty object + if (Object.keys(metadata).length === 1 && 'id' in metadata) { + metadata = {} + } + // Always remove the ID from metadata if present + else if ('id' in metadata) { + const { id: _, ...rest } = metadata + metadata = rest + } + } + + return { + id, + vector: noun.vector, + metadata: metadata as T | undefined + } + } catch (error) { + console.error(`Failed to get vector ${id}:`, error) + throw new Error(`Failed to get vector ${id}: ${error}`) + } + } + + /** + * Get all nouns in the database + * @returns Array of vector documents + */ + public async getAllNouns(): Promise[]> { + await this.ensureInitialized() + + try { + const nouns = this.index.getNouns() + const result: VectorDocument[] = [] + + for (const [id, noun] of nouns.entries()) { + const metadata = await this.storage!.getMetadata(id) + result.push({ + id, + vector: noun.vector, + metadata: metadata as T | undefined + }) + } + + return result + } catch (error) { + console.error('Failed to get all nouns:', error) + throw new Error(`Failed to get all nouns: ${error}`) + } + } + + /** + * Delete a vector by ID + * @param id The ID of the vector to delete + * @param options Additional options + * @returns Promise that resolves to true if the vector was deleted, false otherwise + */ + public async delete( + id: string, + options: { + service?: string // The service that is deleting the data + } = {} + ): Promise { + // Validate id parameter first, before any other logic + if (id === null || id === undefined) { + throw new Error('ID cannot be null or undefined') + } + + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + try { + // Remove from index + const removed = this.index.removeItem(id) + if (!removed) { + return false + } + + // Remove from storage + await this.storage!.deleteNoun(id) + + // Track deletion statistics + const service = options.service || 'default' + await this.storage!.decrementStatistic('noun', service) + + // Try to remove metadata (ignore errors) + try { + await this.storage!.saveMetadata(id, null) + await this.storage!.decrementStatistic('metadata', service) + } catch (error) { + // Ignore + } + + return true + } catch (error) { + console.error(`Failed to delete vector ${id}:`, error) + throw new Error(`Failed to delete vector ${id}: ${error}`) + } + } + + /** + * Update metadata for a vector + * @param id The ID of the vector to update metadata for + * @param metadata The new metadata + * @param options Additional options + * @returns Promise that resolves to true if the metadata was updated, false otherwise + */ + public async updateMetadata( + id: string, + metadata: T, + options: { + service?: string // The service that is updating the data + } = {} + ): Promise { + // Validate id parameter first, before any other logic + if (id === null || id === undefined) { + throw new Error('ID cannot be null or undefined') + } + + // Validate that metadata is not null or undefined + if (metadata === null || metadata === undefined) { + throw new Error(`Metadata cannot be null or undefined`) + } + + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + try { + // Check if a vector exists + const noun = this.index.getNouns().get(id) + if (!noun) { + throw new Error(`Vector with ID ${id} does not exist`) + } + + // Validate noun type if metadata is for a GraphNoun + if (metadata && typeof metadata === 'object' && 'noun' in metadata) { + const nounType = (metadata as unknown as GraphNoun).noun + + // Check if the noun type is valid + const isValidNounType = Object.values(NounType).includes(nounType) + + if (!isValidNounType) { + console.warn( + `Invalid noun type: ${nounType}. Falling back to GraphNoun.` + ) + // Set a default noun type + ;(metadata as unknown as GraphNoun).noun = NounType.Concept + } + + // Get the service that's updating the metadata + const service = options.service || this.getCurrentAugmentation() + const graphNoun = metadata as unknown as GraphNoun + + // Preserve existing createdBy and createdAt if they exist + const existingMetadata = (await this.storage!.getMetadata(id)) as any + + if ( + existingMetadata && + typeof existingMetadata === 'object' && + 'createdBy' in existingMetadata + ) { + // Preserve the original creator information + graphNoun.createdBy = existingMetadata.createdBy + + // Also preserve creation timestamp if it exists + if ('createdAt' in existingMetadata) { + graphNoun.createdAt = existingMetadata.createdAt + } + } else if (!graphNoun.createdBy) { + // If no existing createdBy and none in the update, set it + graphNoun.createdBy = { + augmentation: service, + version: '1.0' // TODO: Get actual version from augmentation + } + + // Set createdAt if it doesn't exist + if (!graphNoun.createdAt) { + const now = new Date() + graphNoun.createdAt = { + seconds: Math.floor(now.getTime() / 1000), + nanoseconds: (now.getTime() % 1000) * 1000000 } - - // Update verb connections from index - verb.connections = indexNoun.connections - - // Save verb to storage - await this.storage!.saveVerb(verb) - - // Track verb statistics - const serviceForStats = options.service || 'default' - await this.storage!.incrementStatistic('verb', serviceForStats) - - // Update HNSW index size (excluding verbs) - await this.storage!.updateHnswIndexSize(await this.getNounCount()) - - return id - } catch (error) { - console.error('Failed to add verb:', error) - throw new Error(`Failed to add verb: ${error}`) + } } + + // Always update the updatedAt timestamp + const now = new Date() + graphNoun.updatedAt = { + seconds: Math.floor(now.getTime() / 1000), + nanoseconds: (now.getTime() % 1000) * 1000000 + } + } + + // Update metadata + await this.storage!.saveMetadata(id, metadata) + + // Track metadata statistics + const service = options.service || this.getCurrentAugmentation() + await this.storage!.incrementStatistic('metadata', service) + + return true + } catch (error) { + console.error(`Failed to update metadata for vector ${id}:`, error) + throw new Error(`Failed to update metadata for vector ${id}: ${error}`) + } + } + + /** + * Create a relationship between two entities + * This is a convenience wrapper around addVerb + */ + public async relate( + sourceId: string, + targetId: string, + relationType: string, + metadata?: any + ): Promise { + // Validate inputs are not null or undefined + if (sourceId === null || sourceId === undefined) { + throw new Error('Source ID cannot be null or undefined') + } + if (targetId === null || targetId === undefined) { + throw new Error('Target ID cannot be null or undefined') + } + if (relationType === null || relationType === undefined) { + throw new Error('Relation type cannot be null or undefined') } - /** - * Get a verb by ID - */ - public async getVerb(id: string): Promise { - await this.ensureInitialized() + return this.addVerb(sourceId, targetId, undefined, { + type: relationType, + metadata: metadata + }) + } - try { - return await this.storage!.getVerb(id) - } catch (error) { - console.error(`Failed to get verb ${id}:`, error) - throw new Error(`Failed to get verb ${id}: ${error}`) - } + /** + * Create a connection between two entities + * This is an alias for relate() for backward compatibility + */ + public async connect( + sourceId: string, + targetId: string, + relationType: string, + metadata?: any + ): Promise { + return this.relate(sourceId, targetId, relationType, metadata) + } + + /** + * Add a verb between two nouns + * If metadata is provided and vector is not, the metadata will be vectorized using the embedding function + * + * @param sourceId ID of the source noun + * @param targetId ID of the target noun + * @param vector Optional vector for the verb + * @param options Additional options: + * - type: Type of the verb + * - weight: Weight of the verb + * - metadata: Metadata for the verb + * - forceEmbed: Force using the embedding function for metadata even if vector is provided + * - id: Optional ID to use instead of generating a new one + * - autoCreateMissingNouns: Automatically create missing nouns if they don't exist + * - missingNounMetadata: Metadata to use when auto-creating missing nouns + * + * @returns The ID of the added verb + * + * @throws Error if source or target nouns don't exist and autoCreateMissingNouns is false or auto-creation fails + */ + public async addVerb( + sourceId: string, + targetId: string, + vector?: Vector, + options: { + type?: string + weight?: number + metadata?: any + forceEmbed?: boolean // Force using the embedding function for metadata even if vector is provided + id?: string // Optional ID to use instead of generating a new one + autoCreateMissingNouns?: boolean // Automatically create missing nouns + missingNounMetadata?: any // Metadata to use when auto-creating missing nouns + service?: string // The service that is inserting the data + } = {} + ): Promise { + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + // Validate inputs are not null or undefined + if (sourceId === null || sourceId === undefined) { + throw new Error('Source ID cannot be null or undefined') + } + if (targetId === null || targetId === undefined) { + throw new Error('Target ID cannot be null or undefined') } - /** - * Get all verbs - */ - public async getAllVerbs(): Promise { - await this.ensureInitialized() + try { + // Check if source and target nouns exist + let sourceNoun = this.index.getNouns().get(sourceId) + let targetNoun = this.index.getNouns().get(targetId) + // Auto-create missing nouns if option is enabled + if (!sourceNoun && options.autoCreateMissingNouns) { try { - return await this.storage!.getAllVerbs() - } catch (error) { - console.error('Failed to get all verbs:', error) - throw new Error(`Failed to get all verbs: ${error}`) - } - } + // Create a placeholder vector for the missing noun + const placeholderVector = new Array(this._dimensions).fill(0) - /** - * Get verbs by source noun ID - */ - public async getVerbsBySource(sourceId: string): Promise { - await this.ensureInitialized() + // Add metadata if provided + const service = options.service || this.getCurrentAugmentation() + const now = new Date() + const timestamp = { + seconds: Math.floor(now.getTime() / 1000), + nanoseconds: (now.getTime() % 1000) * 1000000 + } - try { - return await this.storage!.getVerbsBySource(sourceId) - } catch (error) { - console.error(`Failed to get verbs by source ${sourceId}:`, error) - throw new Error(`Failed to get verbs by source ${sourceId}: ${error}`) - } - } - - /** - * Get verbs by target noun ID - */ - public async getVerbsByTarget(targetId: string): Promise { - await this.ensureInitialized() - - try { - return await this.storage!.getVerbsByTarget(targetId) - } catch (error) { - console.error(`Failed to get verbs by target ${targetId}:`, error) - throw new Error(`Failed to get verbs by target ${targetId}: ${error}`) - } - } - - /** - * Get verbs by type - */ - public async getVerbsByType(type: string): Promise { - await this.ensureInitialized() - - try { - return await this.storage!.getVerbsByType(type) - } catch (error) { - console.error(`Failed to get verbs by type ${type}:`, error) - throw new Error(`Failed to get verbs by type ${type}: ${error}`) - } - } - - /** - * Delete a verb - * @param id The ID of the verb to delete - * @param options Additional options - * @returns Promise that resolves to true if the verb was deleted, false otherwise - */ - public async deleteVerb( - id: string, - options: { - service?: string // The service that is deleting the data - } = {} - ): Promise { - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() - - try { - // Remove from index - const removed = this.index.removeItem(id) - if (!removed) { - return false + const metadata = options.missingNounMetadata || { + autoCreated: true, + createdAt: timestamp, + updatedAt: timestamp, + noun: NounType.Concept, + createdBy: { + augmentation: service, + version: '1.0' // TODO: Get actual version from augmentation } + } - // Remove from storage - await this.storage!.deleteVerb(id) + // Add the missing noun + await this.add(placeholderVector, metadata, { id: sourceId }) - // Track deletion statistics - const service = options.service || 'default' - await this.storage!.decrementStatistic('verb', service) + // Get the newly created noun + sourceNoun = this.index.getNouns().get(sourceId) - return true - } catch (error) { - console.error(`Failed to delete verb ${id}:`, error) - throw new Error(`Failed to delete verb ${id}: ${error}`) + console.warn(`Auto-created missing source noun with ID ${sourceId}`) + } catch (createError) { + console.error( + `Failed to auto-create source noun with ID ${sourceId}:`, + createError + ) + throw new Error( + `Failed to auto-create source noun with ID ${sourceId}: ${createError}` + ) } - } - - /** - * Clear the database - */ - public async clear(): Promise { - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() + } + if (!targetNoun && options.autoCreateMissingNouns) { try { - // Clear index - await this.index.clear() + // Create a placeholder vector for the missing noun + const placeholderVector = new Array(this._dimensions).fill(0) - // Clear storage - await this.storage!.clear() - } catch (error) { - console.error('Failed to clear vector database:', error) - throw new Error(`Failed to clear vector database: ${error}`) - } - } + // Add metadata if provided + const service = options.service || this.getCurrentAugmentation() + const now = new Date() + const timestamp = { + seconds: Math.floor(now.getTime() / 1000), + nanoseconds: (now.getTime() % 1000) * 1000000 + } - /** - * Get the number of vectors in the database - */ - public size(): number { - return this.index.size() - } - - /** - * Get the number of nouns in the database (excluding verbs) - * This is used for statistics reporting to match the expected behavior in tests - * @private - */ - private async getNounCount(): Promise { - // Get all verbs from storage - const allVerbs = await this.storage!.getAllVerbs() - - // Create a set of verb IDs for faster lookup - const verbIds = new Set(allVerbs.map(verb => verb.id)) - - // Get all nouns from the index - const nouns = this.index.getNouns() - - // Count nouns that are not verbs - let nounCount = 0 - for (const [id] of nouns.entries()) { - if (!verbIds.has(id)) { - nounCount++ + const metadata = options.missingNounMetadata || { + autoCreated: true, + createdAt: timestamp, + updatedAt: timestamp, + noun: NounType.Concept, + createdBy: { + augmentation: service, + version: '1.0' // TODO: Get actual version from augmentation } - } + } - return nounCount + // Add the missing noun + await this.add(placeholderVector, metadata, { id: targetId }) + + // Get the newly created noun + targetNoun = this.index.getNouns().get(targetId) + + console.warn(`Auto-created missing target noun with ID ${targetId}`) + } catch (createError) { + console.error( + `Failed to auto-create target noun with ID ${targetId}:`, + createError + ) + throw new Error( + `Failed to auto-create target noun with ID ${targetId}: ${createError}` + ) + } + } + + if (!sourceNoun) { + throw new Error(`Source noun with ID ${sourceId} not found`) + } + + if (!targetNoun) { + throw new Error(`Target noun with ID ${targetId} not found`) + } + + // Use provided ID or generate a new one + const id = options.id || uuidv4() + + let verbVector: Vector + + // If metadata is provided and no vector is provided or forceEmbed is true, vectorize the metadata + if (options.metadata && (!vector || options.forceEmbed)) { + try { + // Extract a string representation from metadata for embedding + let textToEmbed: string + if (typeof options.metadata === 'string') { + textToEmbed = options.metadata + } else if ( + options.metadata.description && + typeof options.metadata.description === 'string' + ) { + textToEmbed = options.metadata.description + } else { + // Convert to JSON string as fallback + textToEmbed = JSON.stringify(options.metadata) + } + + // Ensure textToEmbed is a string + if (typeof textToEmbed !== 'string') { + textToEmbed = String(textToEmbed) + } + + verbVector = await this.embeddingFunction(textToEmbed) + } catch (embedError) { + throw new Error(`Failed to vectorize verb metadata: ${embedError}`) + } + } else { + // Use a provided vector or average of source and target vectors + if (vector) { + verbVector = vector + } else { + // Ensure both source and target vectors have the same dimension + if ( + !sourceNoun.vector || + !targetNoun.vector || + sourceNoun.vector.length === 0 || + targetNoun.vector.length === 0 || + sourceNoun.vector.length !== targetNoun.vector.length + ) { + throw new Error( + `Cannot average vectors: source or target vector is invalid or dimensions don't match` + ) + } + + // Average the vectors + verbVector = sourceNoun.vector.map( + (val, i) => (val + targetNoun.vector[i]) / 2 + ) + } + } + + // Validate verb type if provided + let verbType = options.type + if (!verbType) { + // If no verb type is provided, use RelatedTo as default + verbType = VerbType.RelatedTo + } + // Note: We're no longer validating against VerbType enum to allow custom relationship types + + // Get service name from options or current augmentation + const service = options.service || this.getCurrentAugmentation() + + // Create timestamp for creation/update time + const now = new Date() + const timestamp = { + seconds: Math.floor(now.getTime() / 1000), + nanoseconds: (now.getTime() % 1000) * 1000000 + } + + // Create verb + const verb: GraphVerb = { + id, + vector: verbVector, + connections: new Map(), + sourceId: sourceId, + targetId: targetId, + source: sourceId, + target: targetId, + verb: verbType as VerbType, + type: verbType, // Set the type property to match the verb type + weight: options.weight, + metadata: options.metadata, + createdAt: timestamp, + updatedAt: timestamp, + createdBy: { + augmentation: service, + version: '1.0' // TODO: Get actual version from augmentation + } + } + + // Add to index + await this.index.addItem({ id, vector: verbVector }) + + // Get the noun from the index + const indexNoun = this.index.getNouns().get(id) + + if (!indexNoun) { + throw new Error( + `Failed to retrieve newly created verb noun with ID ${id}` + ) + } + + // Update verb connections from index + verb.connections = indexNoun.connections + + // Save verb to storage + await this.storage!.saveVerb(verb) + + // Track verb statistics + const serviceForStats = options.service || 'default' + await this.storage!.incrementStatistic('verb', serviceForStats) + + // Update HNSW index size (excluding verbs) + await this.storage!.updateHnswIndexSize(await this.getNounCount()) + + return id + } catch (error) { + console.error('Failed to add verb:', error) + throw new Error(`Failed to add verb: ${error}`) + } + } + + /** + * Get a verb by ID + */ + public async getVerb(id: string): Promise { + await this.ensureInitialized() + + try { + return await this.storage!.getVerb(id) + } catch (error) { + console.error(`Failed to get verb ${id}:`, error) + throw new Error(`Failed to get verb ${id}: ${error}`) + } + } + + /** + * Get all verbs + */ + public async getAllVerbs(): Promise { + await this.ensureInitialized() + + try { + return await this.storage!.getAllVerbs() + } catch (error) { + console.error('Failed to get all verbs:', error) + throw new Error(`Failed to get all verbs: ${error}`) + } + } + + /** + * Get verbs by source noun ID + */ + public async getVerbsBySource(sourceId: string): Promise { + await this.ensureInitialized() + + try { + return await this.storage!.getVerbsBySource(sourceId) + } catch (error) { + console.error(`Failed to get verbs by source ${sourceId}:`, error) + throw new Error(`Failed to get verbs by source ${sourceId}: ${error}`) + } + } + + /** + * Get verbs by target noun ID + */ + public async getVerbsByTarget(targetId: string): Promise { + await this.ensureInitialized() + + try { + return await this.storage!.getVerbsByTarget(targetId) + } catch (error) { + console.error(`Failed to get verbs by target ${targetId}:`, error) + throw new Error(`Failed to get verbs by target ${targetId}: ${error}`) + } + } + + /** + * Get verbs by type + */ + public async getVerbsByType(type: string): Promise { + await this.ensureInitialized() + + try { + return await this.storage!.getVerbsByType(type) + } catch (error) { + console.error(`Failed to get verbs by type ${type}:`, error) + throw new Error(`Failed to get verbs by type ${type}: ${error}`) + } + } + + /** + * Delete a verb + * @param id The ID of the verb to delete + * @param options Additional options + * @returns Promise that resolves to true if the verb was deleted, false otherwise + */ + public async deleteVerb( + id: string, + options: { + service?: string // The service that is deleting the data + } = {} + ): Promise { + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + try { + // Remove from index + const removed = this.index.removeItem(id) + if (!removed) { + return false + } + + // Remove from storage + await this.storage!.deleteVerb(id) + + // Track deletion statistics + const service = options.service || 'default' + await this.storage!.decrementStatistic('verb', service) + + return true + } catch (error) { + console.error(`Failed to delete verb ${id}:`, error) + throw new Error(`Failed to delete verb ${id}: ${error}`) + } + } + + /** + * Clear the database + */ + public async clear(): Promise { + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + try { + // Clear index + await this.index.clear() + + // Clear storage + await this.storage!.clear() + } catch (error) { + console.error('Failed to clear vector database:', error) + throw new Error(`Failed to clear vector database: ${error}`) + } + } + + /** + * Get the number of vectors in the database + */ + public size(): number { + return this.index.size() + } + + /** + * Get the number of nouns in the database (excluding verbs) + * This is used for statistics reporting to match the expected behavior in tests + * @private + */ + private async getNounCount(): Promise { + // Get all verbs from storage + const allVerbs = await this.storage!.getAllVerbs() + + // Create a set of verb IDs for faster lookup + const verbIds = new Set(allVerbs.map((verb) => verb.id)) + + // Get all nouns from the index + const nouns = this.index.getNouns() + + // Count nouns that are not verbs + let nounCount = 0 + for (const [id] of nouns.entries()) { + if (!verbIds.has(id)) { + nounCount++ + } } - /** - * Force an immediate flush of statistics to storage - * This ensures that any pending statistics updates are written to persistent storage - * @returns Promise that resolves when the statistics have been flushed - */ - public async flushStatistics(): Promise { - await this.ensureInitialized() + return nounCount + } - if (!this.storage) { - throw new Error('Storage not initialized') - } + /** + * Force an immediate flush of statistics to storage + * This ensures that any pending statistics updates are written to persistent storage + * @returns Promise that resolves when the statistics have been flushed + */ + public async flushStatistics(): Promise { + await this.ensureInitialized() - // Call the flushStatisticsToStorage method on the storage adapter - await this.storage.flushStatisticsToStorage() + if (!this.storage) { + throw new Error('Storage not initialized') } - /** - * Get statistics about the current state of the database - * @param options Additional options for retrieving statistics - * @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size - */ - public async getStatistics(options: { - service?: string | string[] // Filter statistics by service(s) - forceRefresh?: boolean // Force a refresh of statistics from storage - } = {}): Promise<{ + // Call the flushStatisticsToStorage method on the storage adapter + await this.storage.flushStatisticsToStorage() + } + + /** + * Get statistics about the current state of the database + * @param options Additional options for retrieving statistics + * @returns Object containing counts of nouns, verbs, metadata entries, and HNSW index size + */ + public async getStatistics( + options: { + service?: string | string[] // Filter statistics by service(s) + forceRefresh?: boolean // Force a refresh of statistics from storage + } = {} + ): Promise<{ + nounCount: number + verbCount: number + metadataCount: number + hnswIndexSize: number + nouns?: { count: number } + verbs?: { count: number } + metadata?: { count: number } + operations?: { + add: number + search: number + delete: number + update: number + relate: number + total: number + } + serviceBreakdown?: { + [service: string]: { nounCount: number verbCount: number metadataCount: number - hnswIndexSize: number - nouns?: { count: number } - verbs?: { count: number } - metadata?: { count: number } - operations?: { - add: number - search: number - delete: number - update: number - relate: number - total: number - } - serviceBreakdown?: { + } + } + }> { + await this.ensureInitialized() + + try { + // If forceRefresh is true, flush statistics to storage first + if (options.forceRefresh && this.storage) { + await this.storage.flushStatisticsToStorage() + } + + // Get statistics from storage + const stats = await this.storage!.getStatistics() + + // If statistics are available, use them + if (stats) { + // Initialize result + const result = { + nounCount: 0, + verbCount: 0, + metadataCount: 0, + hnswIndexSize: stats.hnswIndexSize, + nouns: { count: 0 }, + verbs: { count: 0 }, + metadata: { count: 0 }, + operations: { + add: 0, + search: 0, + delete: 0, + update: 0, + relate: 0, + total: 0 + }, + serviceBreakdown: {} as { [service: string]: { - nounCount: number - verbCount: number - metadataCount: number + nounCount: number + verbCount: number + metadataCount: number } + } } - }> { - await this.ensureInitialized() - try { - // If forceRefresh is true, flush statistics to storage first - if (options.forceRefresh && this.storage) { - await this.storage.flushStatisticsToStorage() - } - - // Get statistics from storage - const stats = await this.storage!.getStatistics() - - // If statistics are available, use them - if (stats) { - // Initialize result - const result = { - nounCount: 0, - verbCount: 0, - metadataCount: 0, - hnswIndexSize: stats.hnswIndexSize, - nouns: { count: 0 }, - verbs: { count: 0 }, - metadata: { count: 0 }, - operations: { - add: 0, - search: 0, - delete: 0, - update: 0, - relate: 0, - total: 0 - }, - serviceBreakdown: {} as { - [service: string]: { - nounCount: number - verbCount: number - metadataCount: number - } - } - } - - // Filter by service if specified - const services = options.service - ? (Array.isArray(options.service) ? options.service : [options.service]) - : Object.keys({...stats.nounCount, ...stats.verbCount, ...stats.metadataCount}) - - // Calculate totals and service breakdown - for (const service of services) { - const nounCount = stats.nounCount[service] || 0 - const verbCount = stats.verbCount[service] || 0 - const metadataCount = stats.metadataCount[service] || 0 - - // Add to totals - result.nounCount += nounCount - result.verbCount += verbCount - result.metadataCount += metadataCount - - // Add to service breakdown - result.serviceBreakdown[service] = { - nounCount, - verbCount, - metadataCount - } - } - - // Update the alternative format properties - result.nouns.count = result.nounCount - result.verbs.count = result.verbCount - result.metadata.count = result.metadataCount - - // Add operations tracking - result.operations = { - add: result.nounCount, - search: 0, - delete: 0, - update: result.metadataCount, - relate: result.verbCount, - total: result.nounCount + result.verbCount + result.metadataCount - } - - return result - } - - // If statistics are not available, fall back to calculating them on-demand - console.warn('Persistent statistics not available, calculating on-demand') - - // Get all verbs from storage - const allVerbs = await this.storage!.getAllVerbs() - const verbCount = allVerbs.length - - // Get the noun count using the helper method - const nounCount = await this.getNounCount() - - // Count metadata entries by checking each noun for metadata - let metadataCount = 0 - const nouns = this.index.getNouns() - for (const [id] of nouns.entries()) { - try { - const metadata = await this.storage!.getMetadata(id) - if (metadata !== null && metadata !== undefined) { - metadataCount++ - } - } catch (error) { - // Ignore errors when checking individual metadata entries - // This could happen if metadata is corrupted or missing - } - } - - // Get HNSW index size (excluding verbs) - // The HNSW index includes both nouns and verbs, but for statistics we want to report - // only the number of actual nouns (excluding verbs) to match the expected behavior in tests - const hnswIndexSize = nounCount - - // Create default statistics - const defaultStats = { - nounCount, - verbCount, - metadataCount, - hnswIndexSize, - nouns: { count: nounCount }, - verbs: { count: verbCount }, - metadata: { count: metadataCount }, - operations: { - add: nounCount, - search: 0, - delete: 0, - update: metadataCount, - relate: verbCount, - total: nounCount + verbCount + metadataCount - } - } - - // Initialize persistent statistics - const service = 'default' - await this.storage!.saveStatistics({ - nounCount: {[service]: nounCount}, - verbCount: {[service]: verbCount}, - metadataCount: {[service]: metadataCount}, - hnswIndexSize, - lastUpdated: new Date().toISOString() + // Filter by service if specified + const services = options.service + ? Array.isArray(options.service) + ? options.service + : [options.service] + : Object.keys({ + ...stats.nounCount, + ...stats.verbCount, + ...stats.metadataCount }) - return defaultStats - } catch (error) { - console.error('Failed to get statistics:', error) - throw new Error(`Failed to get statistics: ${error}`) + // Calculate totals and service breakdown + for (const service of services) { + const nounCount = stats.nounCount[service] || 0 + const verbCount = stats.verbCount[service] || 0 + const metadataCount = stats.metadataCount[service] || 0 + + // Add to totals + result.nounCount += nounCount + result.verbCount += verbCount + result.metadataCount += metadataCount + + // Add to service breakdown + result.serviceBreakdown[service] = { + nounCount, + verbCount, + metadataCount + } } - } - /** - * Check if the database is in read-only mode - * @returns True if the database is in read-only mode, false otherwise - */ - public isReadOnly(): boolean { - return this.readOnly - } + // Update the alternative format properties + result.nouns.count = result.nounCount + result.verbs.count = result.verbCount + result.metadata.count = result.metadataCount - /** - * Set the database to read-only mode - * @param readOnly True to set the database to read-only mode, false to allow writes - */ - public setReadOnly(readOnly: boolean): void { - this.readOnly = readOnly - } + // Add operations tracking + result.operations = { + add: result.nounCount, + search: 0, + delete: 0, + update: result.metadataCount, + relate: result.verbCount, + total: result.nounCount + result.verbCount + result.metadataCount + } - /** - * Embed text or data into a vector using the same embedding function used by this instance - * This allows clients to use the same TensorFlow Universal Sentence Encoder throughout their application - * - * @param data Text or data to embed - * @returns A promise that resolves to the embedded vector - */ - public async embed(data: string | string[]): Promise { - await this.ensureInitialized() + return result + } + // If statistics are not available, fall back to calculating them on-demand + console.warn('Persistent statistics not available, calculating on-demand') + + // Get all verbs from storage + const allVerbs = await this.storage!.getAllVerbs() + const verbCount = allVerbs.length + + // Get the noun count using the helper method + const nounCount = await this.getNounCount() + + // Count metadata entries by checking each noun for metadata + let metadataCount = 0 + const nouns = this.index.getNouns() + for (const [id] of nouns.entries()) { try { - return await this.embeddingFunction(data) + const metadata = await this.storage!.getMetadata(id) + if (metadata !== null && metadata !== undefined) { + metadataCount++ + } } catch (error) { - console.error('Failed to embed data:', error) - throw new Error(`Failed to embed data: ${error}`) + // Ignore errors when checking individual metadata entries + // This could happen if metadata is corrupted or missing } + } + + // Get HNSW index size (excluding verbs) + // The HNSW index includes both nouns and verbs, but for statistics we want to report + // only the number of actual nouns (excluding verbs) to match the expected behavior in tests + const hnswIndexSize = nounCount + + // Create default statistics + const defaultStats = { + nounCount, + verbCount, + metadataCount, + hnswIndexSize, + nouns: { count: nounCount }, + verbs: { count: verbCount }, + metadata: { count: metadataCount }, + operations: { + add: nounCount, + search: 0, + delete: 0, + update: metadataCount, + relate: verbCount, + total: nounCount + verbCount + metadataCount + } + } + + // Initialize persistent statistics + const service = 'default' + await this.storage!.saveStatistics({ + nounCount: { [service]: nounCount }, + verbCount: { [service]: verbCount }, + metadataCount: { [service]: metadataCount }, + hnswIndexSize, + lastUpdated: new Date().toISOString() + }) + + return defaultStats + } catch (error) { + console.error('Failed to get statistics:', error) + throw new Error(`Failed to get statistics: ${error}`) } + } - /** - * Search for verbs by type and/or vector similarity - * @param queryVectorOrData Query vector or data to search for - * @param k Number of results to return - * @param options Additional options - * @returns Array of verbs with similarity scores - */ - public async searchVerbs( - queryVectorOrData: Vector | any, - k: number = 10, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - verbTypes?: string[] // Optional array of verb types to search within - service?: string // Filter results by the service that created the data - } = {} - ): Promise> { - await this.ensureInitialized() + /** + * Check if the database is in read-only mode + * @returns True if the database is in read-only mode, false otherwise + */ + public isReadOnly(): boolean { + return this.readOnly + } + /** + * Set the database to read-only mode + * @param readOnly True to set the database to read-only mode, false to allow writes + */ + public setReadOnly(readOnly: boolean): void { + this.readOnly = readOnly + + // Ensure readOnly and writeOnly are not both true + if (readOnly && this.writeOnly) { + this.writeOnly = false + } + } + + /** + * Check if the database is in write-only mode + * @returns True if the database is in write-only mode, false otherwise + */ + public isWriteOnly(): boolean { + return this.writeOnly + } + + /** + * Set the database to write-only mode + * @param writeOnly True to set the database to write-only mode, false to allow searches + */ + public setWriteOnly(writeOnly: boolean): void { + this.writeOnly = writeOnly + + // Ensure readOnly and writeOnly are not both true + if (writeOnly && this.readOnly) { + this.readOnly = false + } + } + + /** + * Embed text or data into a vector using the same embedding function used by this instance + * This allows clients to use the same TensorFlow Universal Sentence Encoder throughout their application + * + * @param data Text or data to embed + * @returns A promise that resolves to the embedded vector + */ + public async embed(data: string | string[]): Promise { + await this.ensureInitialized() + + try { + return await this.embeddingFunction(data) + } catch (error) { + console.error('Failed to embed data:', error) + throw new Error(`Failed to embed data: ${error}`) + } + } + + /** + * Search for verbs by type and/or vector similarity + * @param queryVectorOrData Query vector or data to search for + * @param k Number of results to return + * @param options Additional options + * @returns Array of verbs with similarity scores + */ + public async searchVerbs( + queryVectorOrData: Vector | any, + k: number = 10, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + verbTypes?: string[] // Optional array of verb types to search within + service?: string // Filter results by the service that created the data + } = {} + ): Promise> { + await this.ensureInitialized() + + // Check if database is in write-only mode + this.checkWriteOnly() + + try { + let queryVector: Vector + + // Check if input is already a vector + if ( + Array.isArray(queryVectorOrData) && + queryVectorOrData.every((item) => typeof item === 'number') && + !options.forceEmbed + ) { + // Input is already a vector + queryVector = queryVectorOrData + } else { + // Input needs to be vectorized try { - let queryVector: Vector - - // Check if input is already a vector - if ( - Array.isArray(queryVectorOrData) && - queryVectorOrData.every((item) => typeof item === 'number') && - !options.forceEmbed - ) { - // Input is already a vector - queryVector = queryVectorOrData - } else { - // Input needs to be vectorized - try { - queryVector = await this.embeddingFunction(queryVectorOrData) - } catch (embedError) { - throw new Error(`Failed to vectorize query data: ${embedError}`) - } - } - - // First use the HNSW index to find similar vectors efficiently - const searchResults = await this.index.search(queryVector, k * 2) - - // Get all verbs for filtering - const allVerbs = await this.storage!.getAllVerbs() - - // Create a map of verb IDs for faster lookup - const verbMap = new Map() - for (const verb of allVerbs) { - verbMap.set(verb.id, verb) - } - - // Filter search results to only include verbs - const verbResults: Array = [] - - for (const result of searchResults) { - // Search results are [id, distance] tuples - const [id, distance] = result - const verb = verbMap.get(id) - if (verb) { - // If verb types are specified, check if this verb matches - if (options.verbTypes && options.verbTypes.length > 0) { - if (!verb.type || !options.verbTypes.includes(verb.type)) { - continue - } - } - - verbResults.push({ - ...verb, - similarity: distance - }) - } - } - - // If we didn't get enough results from the index, fall back to the old method - if (verbResults.length < k) { - console.warn('Not enough verb results from HNSW index, falling back to manual search') - - // Get verbs to search through - let verbs: GraphVerb[] = [] - - // If verb types are specified, get verbs of those types - if (options.verbTypes && options.verbTypes.length > 0) { - // Get verbs for each verb type in parallel - const verbPromises = options.verbTypes.map((verbType) => - this.getVerbsByType(verbType) - ) - const verbArrays = await Promise.all(verbPromises) - - // Combine all verbs - for (const verbArray of verbArrays) { - verbs.push(...verbArray) - } - } else { - // Use all verbs - verbs = allVerbs - } - - // Calculate similarity for each verb not already in results - const existingIds = new Set(verbResults.map(v => v.id)) - for (const verb of verbs) { - if (!existingIds.has(verb.id) && verb.vector && verb.vector.length > 0) { - const distance = this.index.getDistanceFunction()( - queryVector, - verb.vector - ) - verbResults.push({ - ...verb, - similarity: distance - }) - } - } - } - - // Sort by similarity (ascending distance) - verbResults.sort((a, b) => a.similarity - b.similarity) - - // Take top k results - return verbResults.slice(0, k) - } catch (error) { - console.error('Failed to search verbs:', error) - throw new Error(`Failed to search verbs: ${error}`) + queryVector = await this.embeddingFunction(queryVectorOrData) + } catch (embedError) { + throw new Error(`Failed to vectorize query data: ${embedError}`) } - } + } - /** - * Search for nouns connected by specific verb types - * @param queryVectorOrData Query vector or data to search for - * @param k Number of results to return - * @param options Additional options - * @returns Array of search results - */ - public async searchNounsByVerbs( - queryVectorOrData: Vector | any, - k: number = 10, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - verbTypes?: string[] // Optional array of verb types to filter by - direction?: 'outgoing' | 'incoming' | 'both' // Direction of verbs to consider - } = {} - ): Promise[]> { - await this.ensureInitialized() + // First use the HNSW index to find similar vectors efficiently + const searchResults = await this.index.search(queryVector, k * 2) - try { - // First, search for nouns - const nounResults = await this.searchByNounTypes( - queryVectorOrData, - k * 2, // Get more results initially to account for filtering - null, - {forceEmbed: options.forceEmbed} - ) + // Get all verbs for filtering + const allVerbs = await this.storage!.getAllVerbs() - // If no verb types specified, return the noun results directly - if (!options.verbTypes || options.verbTypes.length === 0) { - return nounResults.slice(0, k) + // Create a map of verb IDs for faster lookup + const verbMap = new Map() + for (const verb of allVerbs) { + verbMap.set(verb.id, verb) + } + + // Filter search results to only include verbs + const verbResults: Array = [] + + for (const result of searchResults) { + // Search results are [id, distance] tuples + const [id, distance] = result + const verb = verbMap.get(id) + if (verb) { + // If verb types are specified, check if this verb matches + if (options.verbTypes && options.verbTypes.length > 0) { + if (!verb.type || !options.verbTypes.includes(verb.type)) { + continue } + } - // For each noun, get connected nouns through specified verb types - const connectedNounIds = new Set() - const direction = options.direction || 'both' - - for (const result of nounResults) { - // Get verbs connected to this noun - let connectedVerbs: GraphVerb[] = [] - - if (direction === 'outgoing' || direction === 'both') { - // Get outgoing verbs - const outgoingVerbs = await this.storage!.getVerbsBySource(result.id) - connectedVerbs.push(...outgoingVerbs) - } - - if (direction === 'incoming' || direction === 'both') { - // Get incoming verbs - const incomingVerbs = await this.storage!.getVerbsByTarget(result.id) - connectedVerbs.push(...incomingVerbs) - } - - // Filter by verb types if specified - if (options.verbTypes && options.verbTypes.length > 0) { - connectedVerbs = connectedVerbs.filter( - (verb) => verb.verb && options.verbTypes!.includes(verb.verb) - ) - } - - // Add connected noun IDs to the set - for (const verb of connectedVerbs) { - if (verb.source && verb.source !== result.id) { - connectedNounIds.add(verb.source) - } - if (verb.target && verb.target !== result.id) { - connectedNounIds.add(verb.target) - } - } - } - - // Get the connected nouns - const connectedNouns: SearchResult[] = [] - for (const id of connectedNounIds) { - try { - const noun = this.index.getNouns().get(id) - if (noun) { - const metadata = await this.storage!.getMetadata(id) - - // Calculate similarity score - let queryVector: Vector - if ( - Array.isArray(queryVectorOrData) && - queryVectorOrData.every((item) => typeof item === 'number') && - !options.forceEmbed - ) { - queryVector = queryVectorOrData - } else { - queryVector = await this.embeddingFunction(queryVectorOrData) - } - - const distance = this.index.getDistanceFunction()( - queryVector, - noun.vector - ) - - connectedNouns.push({ - id, - score: distance, - vector: noun.vector, - metadata: metadata as T | undefined - }) - } - } catch (error) { - console.warn(`Failed to retrieve noun ${id}:`, error) - } - } - - // Sort by similarity score - connectedNouns.sort((a, b) => a.score - b.score) - - // Return top k results - return connectedNouns.slice(0, k) - } catch (error) { - console.error('Failed to search nouns by verbs:', error) - throw new Error(`Failed to search nouns by verbs: ${error}`) + verbResults.push({ + ...verb, + similarity: distance + }) } - } + } - /** - * Search for similar documents using a text query - * This is a convenience method that embeds the query text and performs a search - * - * @param query Text query to search for - * @param k Number of results to return - * @param options Additional options - * @returns Array of search results - */ - public async searchText( - query: string, - k: number = 10, - options: { - nounTypes?: string[] - includeVerbs?: boolean - searchMode?: 'local' | 'remote' | 'combined' - } = {} - ): Promise[]> { - await this.ensureInitialized() + // If we didn't get enough results from the index, fall back to the old method + if (verbResults.length < k) { + console.warn( + 'Not enough verb results from HNSW index, falling back to manual search' + ) - try { - // Embed the query text - const queryVector = await this.embed(query) + // Get verbs to search through + let verbs: GraphVerb[] = [] - // Search using the embedded vector - return await this.search(queryVector, k, { - nounTypes: options.nounTypes, - includeVerbs: options.includeVerbs, - searchMode: options.searchMode - }) - } catch (error) { - console.error('Failed to search with text query:', error) - throw new Error(`Failed to search with text query: ${error}`) - } - } + // If verb types are specified, get verbs of those types + if (options.verbTypes && options.verbTypes.length > 0) { + // Get verbs for each verb type in parallel + const verbPromises = options.verbTypes.map((verbType) => + this.getVerbsByType(verbType) + ) + const verbArrays = await Promise.all(verbPromises) - /** - * Search a remote Brainy server for similar vectors - * @param queryVectorOrData Query vector or data to search for - * @param k Number of results to return - * @param options Additional options - * @returns Array of search results - */ - public async searchRemote( - queryVectorOrData: Vector | any, - k: number = 10, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - nounTypes?: string[] // Optional array of noun types to search within - includeVerbs?: boolean // Whether to include associated GraphVerbs in the results - storeResults?: boolean // Whether to store the results in the local database (default: true) - service?: string // Filter results by the service that created the data - } = {} - ): Promise[]> { - await this.ensureInitialized() - - // Check if connected to a remote server - if (!this.isConnectedToRemoteServer()) { - throw new Error( - 'Not connected to a remote server. Call connectToRemoteServer() first.' - ) - } - - try { - // If input is a string, convert it to a query string for the server - let query: string - if (typeof queryVectorOrData === 'string') { - query = queryVectorOrData - } else { - // For vectors, we need to embed them as a string query - // This is a simplification - ideally we would send the vector directly - query = 'vector-query' // Placeholder, would need a better approach for vector queries - } - - if (!this.serverSearchConduit || !this.serverConnection) { - throw new Error( - 'Server search conduit or connection is not initialized' - ) - } - - // Search the remote server - const searchResult = await this.serverSearchConduit.searchServer( - this.serverConnection.connectionId, - query, - k - ) - - if (!searchResult.success) { - throw new Error(`Remote search failed: ${searchResult.error}`) - } - - return searchResult.data as SearchResult[] - } catch (error) { - console.error('Failed to search remote server:', error) - throw new Error(`Failed to search remote server: ${error}`) - } - } - - /** - * Search both local and remote Brainy instances, combining the results - * @param queryVectorOrData Query vector or data to search for - * @param k Number of results to return - * @param options Additional options - * @returns Array of search results - */ - public async searchCombined( - queryVectorOrData: Vector | any, - k: number = 10, - options: { - forceEmbed?: boolean // Force using the embedding function even if input is a vector - nounTypes?: string[] // Optional array of noun types to search within - includeVerbs?: boolean // Whether to include associated GraphVerbs in the results - localFirst?: boolean // Whether to search local first (default: true) - service?: string // Filter results by the service that created the data - } = {} - ): Promise[]> { - await this.ensureInitialized() - - // Check if connected to a remote server - if (!this.isConnectedToRemoteServer()) { - // If not connected to a remote server, just search locally - return this.searchLocal(queryVectorOrData, k, options) - } - - try { - // Default to searching local first - const localFirst = options.localFirst !== false - - if (localFirst) { - // Search local first - const localResults = await this.searchLocal( - queryVectorOrData, - k, - options - ) - - // If we have enough local results, return them - if (localResults.length >= k) { - return localResults - } - - // Otherwise, search remote for additional results - const remoteResults = await this.searchRemote( - queryVectorOrData, - k - localResults.length, - {...options, storeResults: true} - ) - - // Combine results, removing duplicates - const combinedResults = [...localResults] - const localIds = new Set(localResults.map((r) => r.id)) - - for (const result of remoteResults) { - if (!localIds.has(result.id)) { - combinedResults.push(result) - } - } - - return combinedResults - } else { - // Search remote first - const remoteResults = await this.searchRemote(queryVectorOrData, k, { - ...options, - storeResults: true - }) - - // If we have enough remote results, return them - if (remoteResults.length >= k) { - return remoteResults - } - - // Otherwise, search local for additional results - const localResults = await this.searchLocal( - queryVectorOrData, - k - remoteResults.length, - options - ) - - // Combine results, removing duplicates - const combinedResults = [...remoteResults] - const remoteIds = new Set(remoteResults.map((r) => r.id)) - - for (const result of localResults) { - if (!remoteIds.has(result.id)) { - combinedResults.push(result) - } - } - - return combinedResults - } - } catch (error) { - console.error('Failed to perform combined search:', error) - throw new Error(`Failed to perform combined search: ${error}`) - } - } - - /** - * Check if the instance is connected to a remote server - * @returns True if connected to a remote server, false otherwise - */ - public isConnectedToRemoteServer(): boolean { - return !!(this.serverSearchConduit && this.serverConnection) - } - - /** - * Disconnect from the remote server - * @returns True if successfully disconnected, false if not connected - */ - public async disconnectFromRemoteServer(): Promise { - if (!this.isConnectedToRemoteServer()) { - return false - } - - try { - if (!this.serverSearchConduit || !this.serverConnection) { - throw new Error( - 'Server search conduit or connection is not initialized' - ) - } - - // Close the WebSocket connection - await this.serverSearchConduit.closeWebSocket( - this.serverConnection.connectionId - ) - - // Clear the connection information - this.serverSearchConduit = null - this.serverConnection = null - - return true - } catch (error) { - console.error('Failed to disconnect from remote server:', error) - throw new Error(`Failed to disconnect from remote server: ${error}`) - } - } - - /** - * Ensure the database is initialized - */ - private async ensureInitialized(): Promise { - if (this.isInitialized) { - return - } - - if (this.isInitializing) { - // If initialization is already in progress, wait for it to complete - // by polling the isInitialized flag - let attempts = 0 - const maxAttempts = 100 // Prevent infinite loop - const delay = 50 // ms - - while ( - this.isInitializing && - !this.isInitialized && - attempts < maxAttempts - ) { - await new Promise((resolve) => setTimeout(resolve, delay)) - attempts++ - } - - if (!this.isInitialized) { - // If still not initialized after waiting, try to initialize again - await this.init() - } + // Combine all verbs + for (const verbArray of verbArrays) { + verbs.push(...verbArray) + } } else { - // Normal case - not initialized and not initializing - await this.init() + // Use all verbs + verbs = allVerbs } + + // Calculate similarity for each verb not already in results + const existingIds = new Set(verbResults.map((v) => v.id)) + for (const verb of verbs) { + if ( + !existingIds.has(verb.id) && + verb.vector && + verb.vector.length > 0 + ) { + const distance = this.index.getDistanceFunction()( + queryVector, + verb.vector + ) + verbResults.push({ + ...verb, + similarity: distance + }) + } + } + } + + // Sort by similarity (ascending distance) + verbResults.sort((a, b) => a.similarity - b.similarity) + + // Take top k results + return verbResults.slice(0, k) + } catch (error) { + console.error('Failed to search verbs:', error) + throw new Error(`Failed to search verbs: ${error}`) } + } - /** - * Get information about the current storage usage and capacity - * @returns Object containing the storage type, used space, quota, and additional details - */ - public async status(): Promise<{ - type: string - used: number - quota: number | null - details?: Record - }> { - await this.ensureInitialized() + /** + * Search for nouns connected by specific verb types + * @param queryVectorOrData Query vector or data to search for + * @param k Number of results to return + * @param options Additional options + * @returns Array of search results + */ + public async searchNounsByVerbs( + queryVectorOrData: Vector | any, + k: number = 10, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + verbTypes?: string[] // Optional array of verb types to filter by + direction?: 'outgoing' | 'incoming' | 'both' // Direction of verbs to consider + } = {} + ): Promise[]> { + await this.ensureInitialized() - if (!this.storage) { - return { - type: 'any', - used: 0, - quota: null, - details: {error: 'Storage not initialized'} - } + // Check if database is in write-only mode + this.checkWriteOnly() + + try { + // First, search for nouns + const nounResults = await this.searchByNounTypes( + queryVectorOrData, + k * 2, // Get more results initially to account for filtering + null, + { forceEmbed: options.forceEmbed } + ) + + // If no verb types specified, return the noun results directly + if (!options.verbTypes || options.verbTypes.length === 0) { + return nounResults.slice(0, k) + } + + // For each noun, get connected nouns through specified verb types + const connectedNounIds = new Set() + const direction = options.direction || 'both' + + for (const result of nounResults) { + // Get verbs connected to this noun + let connectedVerbs: GraphVerb[] = [] + + if (direction === 'outgoing' || direction === 'both') { + // Get outgoing verbs + const outgoingVerbs = await this.storage!.getVerbsBySource(result.id) + connectedVerbs.push(...outgoingVerbs) } + if (direction === 'incoming' || direction === 'both') { + // Get incoming verbs + const incomingVerbs = await this.storage!.getVerbsByTarget(result.id) + connectedVerbs.push(...incomingVerbs) + } + + // Filter by verb types if specified + if (options.verbTypes && options.verbTypes.length > 0) { + connectedVerbs = connectedVerbs.filter( + (verb) => verb.verb && options.verbTypes!.includes(verb.verb) + ) + } + + // Add connected noun IDs to the set + for (const verb of connectedVerbs) { + if (verb.source && verb.source !== result.id) { + connectedNounIds.add(verb.source) + } + if (verb.target && verb.target !== result.id) { + connectedNounIds.add(verb.target) + } + } + } + + // Get the connected nouns + const connectedNouns: SearchResult[] = [] + for (const id of connectedNounIds) { try { - // Check if the storage adapter has a getStorageStatus method - if (typeof this.storage.getStorageStatus !== 'function') { - // If not, determine the storage type based on the constructor name - const storageType = this.storage.constructor.name - .toLowerCase() - .replace('storage', '') - return { - type: storageType || 'any', - used: 0, - quota: null, - details: { - error: 'Storage adapter does not implement getStorageStatus method', - storageAdapter: this.storage.constructor.name, - indexSize: this.size() - } - } - } + const noun = this.index.getNouns().get(id) + if (noun) { + const metadata = await this.storage!.getMetadata(id) - // Get storage status from the storage adapter - const storageStatus = await this.storage.getStorageStatus() - - // Add index information to the details - let indexInfo: Record = { - indexSize: this.size() - } - - // Add optimized index information if using optimized index - if (this.useOptimizedIndex && this.index instanceof HNSWIndexOptimized) { - const optimizedIndex = this.index as HNSWIndexOptimized - indexInfo = { - ...indexInfo, - optimized: true, - memoryUsage: optimizedIndex.getMemoryUsage(), - productQuantization: optimizedIndex.getUseProductQuantization(), - diskBasedIndex: optimizedIndex.getUseDiskBasedIndex() - } + // Calculate similarity score + let queryVector: Vector + if ( + Array.isArray(queryVectorOrData) && + queryVectorOrData.every((item) => typeof item === 'number') && + !options.forceEmbed + ) { + queryVector = queryVectorOrData } else { - indexInfo.optimized = false + queryVector = await this.embeddingFunction(queryVectorOrData) } - // Ensure all required fields are present - return { - type: storageStatus.type || 'any', - used: storageStatus.used || 0, - quota: storageStatus.quota || null, - details: { - ...(storageStatus.details || {}), - index: indexInfo - } - } + const distance = this.index.getDistanceFunction()( + queryVector, + noun.vector + ) + + connectedNouns.push({ + id, + score: distance, + vector: noun.vector, + metadata: metadata as T | undefined + }) + } } catch (error) { - console.error('Failed to get storage status:', error) - - // Determine the storage type based on the constructor name - const storageType = this.storage.constructor.name - .toLowerCase() - .replace('storage', '') - - return { - type: storageType || 'any', - used: 0, - quota: null, - details: { - error: String(error), - storageAdapter: this.storage.constructor.name, - indexSize: this.size() - } - } + console.warn(`Failed to retrieve noun ${id}:`, error) } + } + + // Sort by similarity score + connectedNouns.sort((a, b) => a.score - b.score) + + // Return top k results + return connectedNouns.slice(0, k) + } catch (error) { + console.error('Failed to search nouns by verbs:', error) + throw new Error(`Failed to search nouns by verbs: ${error}`) + } + } + + /** + * Search for similar documents using a text query + * This is a convenience method that embeds the query text and performs a search + * + * @param query Text query to search for + * @param k Number of results to return + * @param options Additional options + * @returns Array of search results + */ + public async searchText( + query: string, + k: number = 10, + options: { + nounTypes?: string[] + includeVerbs?: boolean + searchMode?: 'local' | 'remote' | 'combined' + } = {} + ): Promise[]> { + await this.ensureInitialized() + + // Check if database is in write-only mode + this.checkWriteOnly() + + try { + // Embed the query text + const queryVector = await this.embed(query) + + // Search using the embedded vector + return await this.search(queryVector, k, { + nounTypes: options.nounTypes, + includeVerbs: options.includeVerbs, + searchMode: options.searchMode + }) + } catch (error) { + console.error('Failed to search with text query:', error) + throw new Error(`Failed to search with text query: ${error}`) + } + } + + /** + * Search a remote Brainy server for similar vectors + * @param queryVectorOrData Query vector or data to search for + * @param k Number of results to return + * @param options Additional options + * @returns Array of search results + */ + public async searchRemote( + queryVectorOrData: Vector | any, + k: number = 10, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + nounTypes?: string[] // Optional array of noun types to search within + includeVerbs?: boolean // Whether to include associated GraphVerbs in the results + storeResults?: boolean // Whether to store the results in the local database (default: true) + service?: string // Filter results by the service that created the data + } = {} + ): Promise[]> { + await this.ensureInitialized() + + // Check if database is in write-only mode + this.checkWriteOnly() + + // Check if connected to a remote server + if (!this.isConnectedToRemoteServer()) { + throw new Error( + 'Not connected to a remote server. Call connectToRemoteServer() first.' + ) } - /** - * Shut down the database and clean up resources - * This should be called when the database is no longer needed - */ - public async shutDown(): Promise { + try { + // If input is a string, convert it to a query string for the server + let query: string + if (typeof queryVectorOrData === 'string') { + query = queryVectorOrData + } else { + // For vectors, we need to embed them as a string query + // This is a simplification - ideally we would send the vector directly + query = 'vector-query' // Placeholder, would need a better approach for vector queries + } + + if (!this.serverSearchConduit || !this.serverConnection) { + throw new Error( + 'Server search conduit or connection is not initialized' + ) + } + + // Search the remote server + const searchResult = await this.serverSearchConduit.searchServer( + this.serverConnection.connectionId, + query, + k + ) + + if (!searchResult.success) { + throw new Error(`Remote search failed: ${searchResult.error}`) + } + + return searchResult.data as SearchResult[] + } catch (error) { + console.error('Failed to search remote server:', error) + throw new Error(`Failed to search remote server: ${error}`) + } + } + + /** + * Search both local and remote Brainy instances, combining the results + * @param queryVectorOrData Query vector or data to search for + * @param k Number of results to return + * @param options Additional options + * @returns Array of search results + */ + public async searchCombined( + queryVectorOrData: Vector | any, + k: number = 10, + options: { + forceEmbed?: boolean // Force using the embedding function even if input is a vector + nounTypes?: string[] // Optional array of noun types to search within + includeVerbs?: boolean // Whether to include associated GraphVerbs in the results + localFirst?: boolean // Whether to search local first (default: true) + service?: string // Filter results by the service that created the data + } = {} + ): Promise[]> { + await this.ensureInitialized() + + // Check if database is in write-only mode + this.checkWriteOnly() + + // Check if connected to a remote server + if (!this.isConnectedToRemoteServer()) { + // If not connected to a remote server, just search locally + return this.searchLocal(queryVectorOrData, k, options) + } + + try { + // Default to searching local first + const localFirst = options.localFirst !== false + + if (localFirst) { + // Search local first + const localResults = await this.searchLocal( + queryVectorOrData, + k, + options + ) + + // If we have enough local results, return them + if (localResults.length >= k) { + return localResults + } + + // Otherwise, search remote for additional results + const remoteResults = await this.searchRemote( + queryVectorOrData, + k - localResults.length, + { ...options, storeResults: true } + ) + + // Combine results, removing duplicates + const combinedResults = [...localResults] + const localIds = new Set(localResults.map((r) => r.id)) + + for (const result of remoteResults) { + if (!localIds.has(result.id)) { + combinedResults.push(result) + } + } + + return combinedResults + } else { + // Search remote first + const remoteResults = await this.searchRemote(queryVectorOrData, k, { + ...options, + storeResults: true + }) + + // If we have enough remote results, return them + if (remoteResults.length >= k) { + return remoteResults + } + + // Otherwise, search local for additional results + const localResults = await this.searchLocal( + queryVectorOrData, + k - remoteResults.length, + options + ) + + // Combine results, removing duplicates + const combinedResults = [...remoteResults] + const remoteIds = new Set(remoteResults.map((r) => r.id)) + + for (const result of localResults) { + if (!remoteIds.has(result.id)) { + combinedResults.push(result) + } + } + + return combinedResults + } + } catch (error) { + console.error('Failed to perform combined search:', error) + throw new Error(`Failed to perform combined search: ${error}`) + } + } + + /** + * Check if the instance is connected to a remote server + * @returns True if connected to a remote server, false otherwise + */ + public isConnectedToRemoteServer(): boolean { + return !!(this.serverSearchConduit && this.serverConnection) + } + + /** + * Disconnect from the remote server + * @returns True if successfully disconnected, false if not connected + */ + public async disconnectFromRemoteServer(): Promise { + if (!this.isConnectedToRemoteServer()) { + return false + } + + try { + if (!this.serverSearchConduit || !this.serverConnection) { + throw new Error( + 'Server search conduit or connection is not initialized' + ) + } + + // Close the WebSocket connection + await this.serverSearchConduit.closeWebSocket( + this.serverConnection.connectionId + ) + + // Clear the connection information + this.serverSearchConduit = null + this.serverConnection = null + + return true + } catch (error) { + console.error('Failed to disconnect from remote server:', error) + throw new Error(`Failed to disconnect from remote server: ${error}`) + } + } + + /** + * Ensure the database is initialized + */ + private async ensureInitialized(): Promise { + if (this.isInitialized) { + return + } + + if (this.isInitializing) { + // If initialization is already in progress, wait for it to complete + // by polling the isInitialized flag + let attempts = 0 + const maxAttempts = 100 // Prevent infinite loop + const delay = 50 // ms + + while ( + this.isInitializing && + !this.isInitialized && + attempts < maxAttempts + ) { + await new Promise((resolve) => setTimeout(resolve, delay)) + attempts++ + } + + if (!this.isInitialized) { + // If still not initialized after waiting, try to initialize again + await this.init() + } + } else { + // Normal case - not initialized and not initializing + await this.init() + } + } + + /** + * Get information about the current storage usage and capacity + * @returns Object containing the storage type, used space, quota, and additional details + */ + public async status(): Promise<{ + type: string + used: number + quota: number | null + details?: Record + }> { + await this.ensureInitialized() + + if (!this.storage) { + return { + type: 'any', + used: 0, + quota: null, + details: { error: 'Storage not initialized' } + } + } + + try { + // Check if the storage adapter has a getStorageStatus method + if (typeof this.storage.getStorageStatus !== 'function') { + // If not, determine the storage type based on the constructor name + const storageType = this.storage.constructor.name + .toLowerCase() + .replace('storage', '') + return { + type: storageType || 'any', + used: 0, + quota: null, + details: { + error: 'Storage adapter does not implement getStorageStatus method', + storageAdapter: this.storage.constructor.name, + indexSize: this.size() + } + } + } + + // Get storage status from the storage adapter + const storageStatus = await this.storage.getStorageStatus() + + // Add index information to the details + let indexInfo: Record = { + indexSize: this.size() + } + + // Add optimized index information if using optimized index + if (this.useOptimizedIndex && this.index instanceof HNSWIndexOptimized) { + const optimizedIndex = this.index as HNSWIndexOptimized + indexInfo = { + ...indexInfo, + optimized: true, + memoryUsage: optimizedIndex.getMemoryUsage(), + productQuantization: optimizedIndex.getUseProductQuantization(), + diskBasedIndex: optimizedIndex.getUseDiskBasedIndex() + } + } else { + indexInfo.optimized = false + } + + // Ensure all required fields are present + return { + type: storageStatus.type || 'any', + used: storageStatus.used || 0, + quota: storageStatus.quota || null, + details: { + ...(storageStatus.details || {}), + index: indexInfo + } + } + } catch (error) { + console.error('Failed to get storage status:', error) + + // Determine the storage type based on the constructor name + const storageType = this.storage.constructor.name + .toLowerCase() + .replace('storage', '') + + return { + type: storageType || 'any', + used: 0, + quota: null, + details: { + error: String(error), + storageAdapter: this.storage.constructor.name, + indexSize: this.size() + } + } + } + } + + /** + * Shut down the database and clean up resources + * This should be called when the database is no longer needed + */ + public async shutDown(): Promise { + try { + // Stop real-time updates if they're running + this.stopRealtimeUpdates() + + // Flush statistics to ensure they're saved before shutting down + if (this.storage && this.isInitialized) { try { - // Stop real-time updates if they're running - this.stopRealtimeUpdates() - - // Flush statistics to ensure they're saved before shutting down - if (this.storage && this.isInitialized) { - try { - await this.flushStatistics() - } catch (statsError) { - console.warn('Failed to flush statistics during shutdown:', statsError) - // Continue with shutdown even if statistics flush fails - } - } - - // Disconnect from remote server if connected - if (this.isConnectedToRemoteServer()) { - await this.disconnectFromRemoteServer() - } - - // Clean up worker pools to release resources - cleanupWorkerPools() - - // Additional cleanup could be added here in the future - - this.isInitialized = false - } catch (error) { - console.error('Failed to shut down BrainyData:', error) - throw new Error(`Failed to shut down BrainyData: ${error}`) + await this.flushStatistics() + } catch (statsError) { + console.warn( + 'Failed to flush statistics during shutdown:', + statsError + ) + // Continue with shutdown even if statistics flush fails } + } + + // Disconnect from remote server if connected + if (this.isConnectedToRemoteServer()) { + await this.disconnectFromRemoteServer() + } + + // Clean up worker pools to release resources + cleanupWorkerPools() + + // Additional cleanup could be added here in the future + + this.isInitialized = false + } catch (error) { + console.error('Failed to shut down BrainyData:', error) + throw new Error(`Failed to shut down BrainyData: ${error}`) } + } - /** - * Backup all data from the database to a JSON-serializable format - * @returns Object containing all nouns, verbs, noun types, verb types, HNSW index, and other related data - * - * The HNSW index data includes: - * - entryPointId: The ID of the entry point for the graph - * - maxLevel: The maximum level in the hierarchical structure - * - dimension: The dimension of the vectors - * - config: Configuration parameters for the HNSW algorithm - * - connections: A serialized representation of the connections between nouns - */ - public async backup(): Promise<{ - nouns: VectorDocument[] - verbs: GraphVerb[] - nounTypes: string[] - verbTypes: string[] - version: string - hnswIndex?: { - entryPointId: string | null - maxLevel: number - dimension: number | null - config: HNSWConfig - connections: Record> + /** + * Backup all data from the database to a JSON-serializable format + * @returns Object containing all nouns, verbs, noun types, verb types, HNSW index, and other related data + * + * The HNSW index data includes: + * - entryPointId: The ID of the entry point for the graph + * - maxLevel: The maximum level in the hierarchical structure + * - dimension: The dimension of the vectors + * - config: Configuration parameters for the HNSW algorithm + * - connections: A serialized representation of the connections between nouns + */ + public async backup(): Promise<{ + nouns: VectorDocument[] + verbs: GraphVerb[] + nounTypes: string[] + verbTypes: string[] + version: string + hnswIndex?: { + entryPointId: string | null + maxLevel: number + dimension: number | null + config: HNSWConfig + connections: Record> + } + }> { + await this.ensureInitialized() + + try { + // Get all nouns + const nouns = await this.getAllNouns() + + // Get all verbs + const verbs = await this.getAllVerbs() + + // Get all noun types + const nounTypes = Object.values(NounType) + + // Get all verb types + const verbTypes = Object.values(VerbType) + + // Get HNSW index data + const hnswIndexData = { + entryPointId: this.index.getEntryPointId(), + maxLevel: this.index.getMaxLevel(), + dimension: this.index.getDimension(), + config: this.index.getConfig(), + connections: {} as Record> + } + + // Convert Map> to a serializable format + const indexNouns = this.index.getNouns() + for (const [id, noun] of indexNouns.entries()) { + hnswIndexData.connections[id] = {} + for (const [level, connections] of noun.connections.entries()) { + hnswIndexData.connections[id][level] = Array.from(connections) } - }> { - await this.ensureInitialized() + } + // Return the data with version information + return { + nouns, + verbs, + nounTypes, + verbTypes, + hnswIndex: hnswIndexData, + version: '1.0.0' // Version of the backup format + } + } catch (error) { + console.error('Failed to backup data:', error) + throw new Error(`Failed to backup data: ${error}`) + } + } + + /** + * Import sparse data into the database + * @param data The sparse data to import + * If vectors are not present for nouns, they will be created using the embedding function + * @param options Import options + * @returns Object containing counts of imported items + */ + public async importSparseData( + data: { + nouns: VectorDocument[] + verbs: GraphVerb[] + nounTypes?: string[] + verbTypes?: string[] + hnswIndex?: { + entryPointId: string | null + maxLevel: number + dimension: number | null + config: HNSWConfig + connections: Record> + } + version: string + }, + options: { + clearExisting?: boolean + } = {} + ): Promise<{ + nounsRestored: number + verbsRestored: number + }> { + return this.restore(data, options) + } + + /** + * Restore data into the database from a previously backed up format + * @param data The data to restore, in the format returned by backup() + * This can include HNSW index data if it was included in the backup + * If vectors are not present for nouns, they will be created using the embedding function + * @param options Restore options + * @returns Object containing counts of restored items + */ + public async restore( + data: { + nouns: VectorDocument[] + verbs: GraphVerb[] + nounTypes?: string[] + verbTypes?: string[] + hnswIndex?: { + entryPointId: string | null + maxLevel: number + dimension: number | null + config: HNSWConfig + connections: Record> + } + version: string + }, + options: { + clearExisting?: boolean + } = {} + ): Promise<{ + nounsRestored: number + verbsRestored: number + }> { + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + try { + // Clear existing data if requested + if (options.clearExisting) { + await this.clear() + } + + // Validate the data format + if (!data || !data.nouns || !data.verbs || !data.version) { + throw new Error('Invalid restore data format') + } + + // Log additional data if present + if (data.nounTypes) { + console.log(`Found ${data.nounTypes.length} noun types in restore data`) + } + + if (data.verbTypes) { + console.log(`Found ${data.verbTypes.length} verb types in restore data`) + } + + if (data.hnswIndex) { + console.log('Found HNSW index data in backup') + } + + // Restore nouns + let nounsRestored = 0 + for (const noun of data.nouns) { try { - // Get all nouns - const nouns = await this.getAllNouns() - - // Get all verbs - const verbs = await this.getAllVerbs() - - // Get all noun types - const nounTypes = Object.values(NounType) - - // Get all verb types - const verbTypes = Object.values(VerbType) - - // Get HNSW index data - const hnswIndexData = { - entryPointId: this.index.getEntryPointId(), - maxLevel: this.index.getMaxLevel(), - dimension: this.index.getDimension(), - config: this.index.getConfig(), - connections: {} as Record> + // Check if the noun has a vector + if (!noun.vector || noun.vector.length === 0) { + // If no vector, create one using the embedding function + if ( + noun.metadata && + typeof noun.metadata === 'object' && + 'text' in noun.metadata + ) { + // If the metadata has a text field, use it for embedding + noun.vector = await this.embeddingFunction(noun.metadata.text) + } else { + // Otherwise, use the entire metadata for embedding + noun.vector = await this.embeddingFunction(noun.metadata) } + } - // Convert Map> to a serializable format - const indexNouns = this.index.getNouns() - for (const [id, noun] of indexNouns.entries()) { - hnswIndexData.connections[id] = {} - for (const [level, connections] of noun.connections.entries()) { - hnswIndexData.connections[id][level] = Array.from(connections) - } - } - - // Return the data with version information - return { - nouns, - verbs, - nounTypes, - verbTypes, - hnswIndex: hnswIndexData, - version: '1.0.0' // Version of the backup format - } + // Add the noun with its vector and metadata + await this.add(noun.vector, noun.metadata, { id: noun.id }) + nounsRestored++ } catch (error) { - console.error('Failed to backup data:', error) - throw new Error(`Failed to backup data: ${error}`) + console.error(`Failed to restore noun ${noun.id}:`, error) + // Continue with other nouns } - } - - /** - * Import sparse data into the database - * @param data The sparse data to import - * If vectors are not present for nouns, they will be created using the embedding function - * @param options Import options - * @returns Object containing counts of imported items - */ - public async importSparseData( - data: { - nouns: VectorDocument[] - verbs: GraphVerb[] - nounTypes?: string[] - verbTypes?: string[] - hnswIndex?: { - entryPointId: string | null - maxLevel: number - dimension: number | null - config: HNSWConfig - connections: Record> - } - version: string - }, - options: { - clearExisting?: boolean - } = {} - ): Promise<{ - nounsRestored: number - verbsRestored: number - }> { - return this.restore(data, options) - } - - /** - * Restore data into the database from a previously backed up format - * @param data The data to restore, in the format returned by backup() - * This can include HNSW index data if it was included in the backup - * If vectors are not present for nouns, they will be created using the embedding function - * @param options Restore options - * @returns Object containing counts of restored items - */ - public async restore( - data: { - nouns: VectorDocument[] - verbs: GraphVerb[] - nounTypes?: string[] - verbTypes?: string[] - hnswIndex?: { - entryPointId: string | null - maxLevel: number - dimension: number | null - config: HNSWConfig - connections: Record> - } - version: string - }, - options: { - clearExisting?: boolean - } = {} - ): Promise<{ - nounsRestored: number - verbsRestored: number - }> { - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() + } + // Restore verbs + let verbsRestored = 0 + for (const verb of data.verbs) { try { - // Clear existing data if requested - if (options.clearExisting) { - await this.clear() + // Check if the verb has a vector + if (!verb.vector || verb.vector.length === 0) { + // If no vector, create one using the embedding function + if ( + verb.metadata && + typeof verb.metadata === 'object' && + 'text' in verb.metadata + ) { + // If the metadata has a text field, use it for embedding + verb.vector = await this.embeddingFunction(verb.metadata.text) + } else { + // Otherwise, use the entire metadata for embedding + verb.vector = await this.embeddingFunction(verb.metadata) } + } - // Validate the data format - if (!data || !data.nouns || !data.verbs || !data.version) { - throw new Error('Invalid restore data format') - } - - // Log additional data if present - if (data.nounTypes) { - console.log(`Found ${data.nounTypes.length} noun types in restore data`) - } - - if (data.verbTypes) { - console.log(`Found ${data.verbTypes.length} verb types in restore data`) - } - - if (data.hnswIndex) { - console.log('Found HNSW index data in backup') - } - - // Restore nouns - let nounsRestored = 0 - for (const noun of data.nouns) { - try { - // Check if the noun has a vector - if (!noun.vector || noun.vector.length === 0) { - // If no vector, create one using the embedding function - if ( - noun.metadata && - typeof noun.metadata === 'object' && - 'text' in noun.metadata - ) { - // If the metadata has a text field, use it for embedding - noun.vector = await this.embeddingFunction(noun.metadata.text) - } else { - // Otherwise, use the entire metadata for embedding - noun.vector = await this.embeddingFunction(noun.metadata) - } - } - - // Add the noun with its vector and metadata - await this.add(noun.vector, noun.metadata, {id: noun.id}) - nounsRestored++ - } catch (error) { - console.error(`Failed to restore noun ${noun.id}:`, error) - // Continue with other nouns - } - } - - // Restore verbs - let verbsRestored = 0 - for (const verb of data.verbs) { - try { - // Check if the verb has a vector - if (!verb.vector || verb.vector.length === 0) { - // If no vector, create one using the embedding function - if ( - verb.metadata && - typeof verb.metadata === 'object' && - 'text' in verb.metadata - ) { - // If the metadata has a text field, use it for embedding - verb.vector = await this.embeddingFunction(verb.metadata.text) - } else { - // Otherwise, use the entire metadata for embedding - verb.vector = await this.embeddingFunction(verb.metadata) - } - } - - // Add the verb - await this.addVerb(verb.sourceId, verb.targetId, verb.vector, { - id: verb.id, - type: verb.metadata?.verb || VerbType.RelatedTo, - metadata: verb.metadata - }) - verbsRestored++ - } catch (error) { - console.error(`Failed to restore verb ${verb.id}:`, error) - // Continue with other verbs - } - } - - // If HNSW index data is provided and we've restored nouns, reconstruct the index - if (data.hnswIndex && nounsRestored > 0) { - try { - console.log('Reconstructing HNSW index from backup data...') - - // Create a new index with the restored configuration - // Always use the optimized implementation for consistency - this.index = new HNSWIndexOptimized( - data.hnswIndex.config, - this.distanceFunction, - this.storage - ) - this.useOptimizedIndex = true - - // Re-add all nouns to the index - for (const noun of data.nouns) { - if (noun.vector && noun.vector.length > 0) { - await this.index.addItem({id: noun.id, vector: noun.vector}) - } - } - - console.log('HNSW index reconstruction complete') - } catch (error) { - console.error('Failed to reconstruct HNSW index:', error) - console.log('Continuing with standard restore process...') - } - } - - return { - nounsRestored, - verbsRestored - } + // Add the verb + await this.addVerb(verb.sourceId, verb.targetId, verb.vector, { + id: verb.id, + type: verb.metadata?.verb || VerbType.RelatedTo, + metadata: verb.metadata + }) + verbsRestored++ } catch (error) { - console.error('Failed to restore data:', error) - throw new Error(`Failed to restore data: ${error}`) - } - } - - /** - * Generate a random graph of data with typed nouns and verbs for testing and experimentation - * @param options Configuration options for the random graph - * @returns Object containing the IDs of the generated nouns and verbs - */ - public async generateRandomGraph( - options: { - nounCount?: number // Number of nouns to generate (default: 10) - verbCount?: number // Number of verbs to generate (default: 20) - nounTypes?: NounType[] // Types of nouns to generate (default: all types) - verbTypes?: VerbType[] // Types of verbs to generate (default: all types) - clearExisting?: boolean // Whether to clear existing data before generating (default: false) - seed?: string // Seed for random generation (default: random) - } = {} - ): Promise<{ - nounIds: string[] - verbIds: string[] - }> { - await this.ensureInitialized() - - // Check if database is in read-only mode - this.checkReadOnly() - - // Set default options - const nounCount = options.nounCount || 10 - const verbCount = options.verbCount || 20 - const nounTypes = options.nounTypes || Object.values(NounType) - const verbTypes = options.verbTypes || Object.values(VerbType) - const clearExisting = options.clearExisting || false - - // Clear existing data if requested - if (clearExisting) { - await this.clear() + console.error(`Failed to restore verb ${verb.id}:`, error) + // Continue with other verbs } + } + // If HNSW index data is provided and we've restored nouns, reconstruct the index + if (data.hnswIndex && nounsRestored > 0) { try { - // Generate random nouns - const nounIds: string[] = [] - const nounDescriptions: Record = { - [NounType.Person]: 'A person with unique characteristics', - [NounType.Place]: 'A location with specific attributes', - [NounType.Thing]: 'An object with distinct properties', - [NounType.Event]: 'An occurrence with temporal aspects', - [NounType.Concept]: 'An abstract idea or notion', - [NounType.Content]: 'A piece of content or information', - [NounType.Group]: 'A collection of related entities', - [NounType.List]: 'An ordered sequence of items', - [NounType.Category]: 'A classification or grouping' + console.log('Reconstructing HNSW index from backup data...') + + // Create a new index with the restored configuration + // Always use the optimized implementation for consistency + this.index = new HNSWIndexOptimized( + data.hnswIndex.config, + this.distanceFunction, + this.storage + ) + this.useOptimizedIndex = true + + // Re-add all nouns to the index + for (const noun of data.nouns) { + if (noun.vector && noun.vector.length > 0) { + await this.index.addItem({ id: noun.id, vector: noun.vector }) } + } - for (let i = 0; i < nounCount; i++) { - // Select a random noun type - const nounType = nounTypes[Math.floor(Math.random() * nounTypes.length)] - - // Generate a random label - const label = `Random ${nounType} ${i + 1}` - - // Create metadata - const metadata = { - noun: nounType, - label, - description: nounDescriptions[nounType] || `A random ${nounType}`, - randomAttributes: { - value: Math.random() * 100, - priority: Math.floor(Math.random() * 5) + 1, - tags: [`tag-${i % 5}`, `category-${i % 3}`] - } - } - - // Add the noun - const id = await this.add(metadata.description, metadata as T) - nounIds.push(id) - } - - // Generate random verbs between nouns - const verbIds: string[] = [] - const verbDescriptions: Record = { - [VerbType.AttributedTo]: 'Attribution relationship', - [VerbType.Controls]: 'Control relationship', - [VerbType.Created]: 'Creation relationship', - [VerbType.Earned]: 'Achievement relationship', - [VerbType.Owns]: 'Ownership relationship', - [VerbType.MemberOf]: 'Membership relationship', - [VerbType.RelatedTo]: 'General relationship', - [VerbType.WorksWith]: 'Collaboration relationship', - [VerbType.FriendOf]: 'Friendship relationship', - [VerbType.ReportsTo]: 'Reporting relationship', - [VerbType.Supervises]: 'Supervision relationship', - [VerbType.Mentors]: 'Mentorship relationship' - } - - for (let i = 0; i < verbCount; i++) { - // Select random source and target nouns - const sourceIndex = Math.floor(Math.random() * nounIds.length) - let targetIndex = Math.floor(Math.random() * nounIds.length) - - // Ensure source and target are different - while (targetIndex === sourceIndex && nounIds.length > 1) { - targetIndex = Math.floor(Math.random() * nounIds.length) - } - - const sourceId = nounIds[sourceIndex] - const targetId = nounIds[targetIndex] - - // Select a random verb type - const verbType = verbTypes[Math.floor(Math.random() * verbTypes.length)] - - // Create metadata - const metadata = { - verb: verbType, - description: - verbDescriptions[verbType] || `A random ${verbType} relationship`, - weight: Math.random(), - confidence: Math.random(), - randomAttributes: { - strength: Math.random() * 100, - duration: Math.floor(Math.random() * 365) + 1, - tags: [`relation-${i % 5}`, `strength-${i % 3}`] - } - } - - // Add the verb - const id = await this.addVerb(sourceId, targetId, undefined, { - type: verbType, - weight: metadata.weight, - metadata - }) - - verbIds.push(id) - } - - return { - nounIds, - verbIds - } + console.log('HNSW index reconstruction complete') } catch (error) { - console.error('Failed to generate random graph:', error) - throw new Error(`Failed to generate random graph: ${error}`) + console.error('Failed to reconstruct HNSW index:', error) + console.log('Continuing with standard restore process...') } + } + + return { + nounsRestored, + verbsRestored + } + } catch (error) { + console.error('Failed to restore data:', error) + throw new Error(`Failed to restore data: ${error}`) } + } + + /** + * Generate a random graph of data with typed nouns and verbs for testing and experimentation + * @param options Configuration options for the random graph + * @returns Object containing the IDs of the generated nouns and verbs + */ + public async generateRandomGraph( + options: { + nounCount?: number // Number of nouns to generate (default: 10) + verbCount?: number // Number of verbs to generate (default: 20) + nounTypes?: NounType[] // Types of nouns to generate (default: all types) + verbTypes?: VerbType[] // Types of verbs to generate (default: all types) + clearExisting?: boolean // Whether to clear existing data before generating (default: false) + seed?: string // Seed for random generation (default: random) + } = {} + ): Promise<{ + nounIds: string[] + verbIds: string[] + }> { + await this.ensureInitialized() + + // Check if database is in read-only mode + this.checkReadOnly() + + // Set default options + const nounCount = options.nounCount || 10 + const verbCount = options.verbCount || 20 + const nounTypes = options.nounTypes || Object.values(NounType) + const verbTypes = options.verbTypes || Object.values(VerbType) + const clearExisting = options.clearExisting || false + + // Clear existing data if requested + if (clearExisting) { + await this.clear() + } + + try { + // Generate random nouns + const nounIds: string[] = [] + const nounDescriptions: Record = { + [NounType.Person]: 'A person with unique characteristics', + [NounType.Location]: 'A location with specific attributes', + [NounType.Thing]: 'An object with distinct properties', + [NounType.Event]: 'An occurrence with temporal aspects', + [NounType.Concept]: 'An abstract idea or notion', + [NounType.Content]: 'A piece of content or information', + [NounType.Collection]: 'A collection of related entities', + [NounType.Organization]: 'An organization or institution', + [NounType.Document]: 'A document or text-based file' + } + + for (let i = 0; i < nounCount; i++) { + // Select a random noun type + const nounType = nounTypes[Math.floor(Math.random() * nounTypes.length)] + + // Generate a random label + const label = `Random ${nounType} ${i + 1}` + + // Create metadata + const metadata = { + noun: nounType, + label, + description: nounDescriptions[nounType] || `A random ${nounType}`, + randomAttributes: { + value: Math.random() * 100, + priority: Math.floor(Math.random() * 5) + 1, + tags: [`tag-${i % 5}`, `category-${i % 3}`] + } + } + + // Add the noun + const id = await this.add(metadata.description, metadata as T) + nounIds.push(id) + } + + // Generate random verbs between nouns + const verbIds: string[] = [] + const verbDescriptions: Record = { + [VerbType.AttributedTo]: 'Attribution relationship', + [VerbType.Owns]: 'Ownership relationship', + [VerbType.Creates]: 'Creation relationship', + [VerbType.Uses]: 'Utilization relationship', + [VerbType.BelongsTo]: 'Belonging relationship', + [VerbType.MemberOf]: 'Membership relationship', + [VerbType.RelatedTo]: 'General relationship', + [VerbType.WorksWith]: 'Collaboration relationship', + [VerbType.FriendOf]: 'Friendship relationship', + [VerbType.ReportsTo]: 'Reporting relationship', + [VerbType.Supervises]: 'Supervision relationship', + [VerbType.Mentors]: 'Mentorship relationship' + } + + for (let i = 0; i < verbCount; i++) { + // Select random source and target nouns + const sourceIndex = Math.floor(Math.random() * nounIds.length) + let targetIndex = Math.floor(Math.random() * nounIds.length) + + // Ensure source and target are different + while (targetIndex === sourceIndex && nounIds.length > 1) { + targetIndex = Math.floor(Math.random() * nounIds.length) + } + + const sourceId = nounIds[sourceIndex] + const targetId = nounIds[targetIndex] + + // Select a random verb type + const verbType = verbTypes[Math.floor(Math.random() * verbTypes.length)] + + // Create metadata + const metadata = { + verb: verbType, + description: + verbDescriptions[verbType] || `A random ${verbType} relationship`, + weight: Math.random(), + confidence: Math.random(), + randomAttributes: { + strength: Math.random() * 100, + duration: Math.floor(Math.random() * 365) + 1, + tags: [`relation-${i % 5}`, `strength-${i % 3}`] + } + } + + // Add the verb + const id = await this.addVerb(sourceId, targetId, undefined, { + type: verbType, + weight: metadata.weight, + metadata + }) + + verbIds.push(id) + } + + return { + nounIds, + verbIds + } + } catch (error) { + console.error('Failed to generate random graph:', error) + throw new Error(`Failed to generate random graph: ${error}`) + } + } } // Export distance functions for convenience export { - euclideanDistance, - cosineDistance, - manhattanDistance, - dotProductDistance + euclideanDistance, + cosineDistance, + manhattanDistance, + dotProductDistance } from './utils/index.js'