diff --git a/examples/brainy-backup.ts b/examples/brainy-backup.ts deleted file mode 100644 index 660e75e4..00000000 --- a/examples/brainy-backup.ts +++ /dev/null @@ -1,903 +0,0 @@ -/** - * 🧠 Brainy 3.0 - The Future of Neural Databases - * - * Beautiful, Professional, Planet-Scale, Fun to Use - * NO STUBS, NO MOCKS, REAL IMPLEMENTATION - */ - -import { v4 as uuidv4 } from './universal/uuid.js' -import { HNSWIndex } from './hnsw/hnswIndex.js' -import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js' -import { createStorage } from './storage/storageFactory.js' -import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb } from './coreTypes.js' -import { - defaultEmbeddingFunction, - cosineDistance -} from './utils/index.js' -import { validateNounType, validateVerbType } from './utils/typeValidation.js' -import { matchesMetadataFilter } from './utils/metadataFilter.js' -import { AugmentationRegistry, AugmentationContext } from './augmentations/brainyAugmentation.js' -import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js' -import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js' -import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js' -import { - Entity, - Relation, - Result, - AddParams, - UpdateParams, - RelateParams, - FindParams, - SimilarParams, - GetRelationsParams, - AddManyParams, - DeleteManyParams, - BatchResult, - BrainyConfig -} from './types/brainy.types.js' -import { NounType, VerbType } from './types/graphTypes.js' - -/** - * The main Brainy class - Clean, Beautiful, Powerful - * REAL IMPLEMENTATION - No stubs, no mocks - */ -export class Brainy { - // Core components - private index!: HNSWIndex | HNSWIndexOptimized - private storage!: StorageAdapter - private embedder: EmbeddingFunction - private distance: DistanceFunction - private augmentationRegistry: AugmentationRegistry - private config: Required - - // Sub-APIs (lazy-loaded) - private _neural?: ImprovedNeuralAPI - private _nlp?: NaturalLanguageProcessor - - // State - private initialized = false - private dimensions?: number - - constructor(config?: BrainyConfig) { - // Normalize configuration with defaults - this.config = this.normalizeConfig(config) - - // Setup core components - this.distance = cosineDistance - this.embedder = this.setupEmbedder() - this.augmentationRegistry = this.setupAugmentations() - - // Index and storage are initialized in init() because they may need each other - } - - /** - * Initialize Brainy - MUST be called before use - */ - async init(): Promise { - if (this.initialized) { - return - } - - try { - // Setup and initialize storage - this.storage = await this.setupStorage() - await this.storage.init() - - // Setup index now that we have storage - this.index = this.setupIndex() - - // Initialize augmentations - await this.augmentationRegistry.initializeAll({ - brain: this, - storage: this.storage, - config: this.config, - log: (message: string, level = 'info') => { - // Simple logging for now - if (level === 'error') { - console.error(message) - } else if (level === 'warn') { - console.warn(message) - } else { - console.log(message) - } - } - }) - - // Warm up if configured - if (this.config.warmup) { - await this.warmup() - } - - this.initialized = true - } catch (error) { - throw new Error(`Failed to initialize Brainy: ${error}`) - } - } - - /** - * Ensure Brainy is initialized - */ - private async ensureInitialized(): Promise { - if (!this.initialized) { - throw new Error('Brainy not initialized. Call init() first.') - } - } - - // ============= CORE CRUD OPERATIONS ============= - - /** - * Add an entity to the database - */ - async add(params: AddParams): Promise { - await this.ensureInitialized() - - // Validate parameters - if (!params.data && !params.vector) { - throw new Error('Either data or vector is required') - } - validateNounType(params.type) - - // Generate ID if not provided - const id = params.id || uuidv4() - - // Get or compute vector - const vector = params.vector || (await this.embed(params.data)) - - // Ensure dimensions are set - if (!this.dimensions) { - this.dimensions = vector.length - } else if (vector.length !== this.dimensions) { - throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` - ) - } - - // Execute through augmentation pipeline - return this.augmentationRegistry.execute('add', params, async () => { - // Add to index - await this.index.addItem({ id, vector }) - - // Save to storage - await this.storage.saveNoun({ - id, - vector, - connections: new Map(), - level: 0, - metadata: { - ...params.metadata, - noun: params.type, - service: params.service, - createdAt: Date.now() - } - }) - - return id - }) - } - - /** - * Get an entity by ID - */ - async get(id: string): Promise | null> { - await this.ensureInitialized() - - return this.augmentationRegistry.execute('get', { id }, async () => { - // Get from storage - const noun = await this.storage.getNoun(id) - if (!noun) { - return null - } - - // Extract metadata - separate user metadata from system metadata - const { noun: nounType, service, createdAt, updatedAt, ...userMetadata } = noun.metadata || {} - - // Construct entity from noun - const entity: Entity = { - id: noun.id, - vector: noun.vector, - type: (nounType as NounType) || NounType.Thing, - metadata: userMetadata as T, - service: service as string, - createdAt: (createdAt as number) || Date.now(), - updatedAt: updatedAt as number - } - - return entity - }) - } - - /** - * Update an entity - */ - async update(params: UpdateParams): Promise { - await this.ensureInitialized() - - if (!params.id) { - throw new Error('ID is required for update') - } - - return this.augmentationRegistry.execute('update', params, async () => { - // Get existing entity - const existing = await this.get(params.id) - if (!existing) { - throw new Error(`Entity ${params.id} not found`) - } - - // Update vector if data changed - let vector = existing.vector - if (params.data) { - vector = params.vector || (await this.embed(params.data)) - // Update in index (remove and re-add since no update method) - await this.index.removeItem(params.id) - await this.index.addItem({ id: params.id, vector }) - } - - // Always update the noun with new metadata - const newMetadata = params.merge !== false - ? { ...existing.metadata, ...params.metadata } - : params.metadata || existing.metadata - - await this.storage.saveNoun({ - id: params.id, - vector, - connections: new Map(), - level: 0, - metadata: { - ...newMetadata, - noun: params.type || existing.type, - service: existing.service, - createdAt: existing.createdAt, - updatedAt: Date.now() - } - }) - }) - } - - /** - * Delete an entity - */ - async delete(id: string): Promise { - await this.ensureInitialized() - - return this.augmentationRegistry.execute('delete', { id }, async () => { - // Remove from index - await this.index.removeItem(id) - - // Delete from storage - await this.storage.deleteNoun(id) - - // Delete metadata (if it exists as separate) - try { - await this.storage.saveMetadata(id, null as any) // Clear metadata - } catch { - // Ignore if not supported - } - - // Delete related verbs - const verbs = await this.storage.getVerbsBySource(id) - const targetVerbs = await this.storage.getVerbsByTarget(id) - const allVerbs = [...verbs, ...targetVerbs] - - for (const verb of allVerbs) { - await this.storage.deleteVerb(verb.id) - } - }) - } - - // ============= RELATIONSHIP OPERATIONS ============= - - /** - * Create a relationship between entities - */ - async relate(params: RelateParams): Promise { - await this.ensureInitialized() - - // Validate parameters - if (!params.from || !params.to) { - throw new Error('Both from and to are required') - } - validateVerbType(params.type) - - // Verify entities exist - const fromEntity = await this.get(params.from) - const toEntity = await this.get(params.to) - - if (!fromEntity) { - throw new Error(`Source entity ${params.from} not found`) - } - if (!toEntity) { - throw new Error(`Target entity ${params.to} not found`) - } - - // Generate ID - const id = uuidv4() - - // Compute relationship vector (average of entities) - const relationVector = fromEntity.vector.map( - (v, i) => (v + toEntity.vector[i]) / 2 - ) - - return this.augmentationRegistry.execute('relate', params, async () => { - // Save to storage - const verb: GraphVerb = { - id, - vector: relationVector, - sourceId: params.from, - targetId: params.to, - source: fromEntity.type, - target: toEntity.type, - verb: params.type, - type: params.type, - weight: params.weight ?? 1.0, - metadata: params.metadata as any, - createdAt: Date.now() - } as any - - await this.storage.saveVerb(verb) - - // Create bidirectional if requested - if (params.bidirectional) { - const reverseId = uuidv4() - const reverseVerb: GraphVerb = { - ...verb, - id: reverseId, - sourceId: params.to, - targetId: params.from, - source: toEntity.type, - target: fromEntity.type - } as any - - await this.storage.saveVerb(reverseVerb) - } - - return id - }) - } - - /** - * Delete a relationship - */ - async unrelate(id: string): Promise { - await this.ensureInitialized() - - return this.augmentationRegistry.execute('unrelate', { id }, async () => { - await this.storage.deleteVerb(id) - }) - } - - /** - * Get relationships - */ - async getRelations( - params: GetRelationsParams = {} - ): Promise[]> { - await this.ensureInitialized() - - const relations: Relation[] = [] - - if (params.from) { - const verbs = await this.storage.getVerbsBySource(params.from) - relations.push(...this.verbsToRelations(verbs)) - } - - if (params.to) { - const verbs = await this.storage.getVerbsByTarget(params.to) - relations.push(...this.verbsToRelations(verbs)) - } - - // Filter by type - let filtered = relations - if (params.type) { - const types = Array.isArray(params.type) ? params.type : [params.type] - filtered = relations.filter((r) => types.includes(r.type)) - } - - // Filter by service - if (params.service) { - filtered = filtered.filter((r) => r.service === params.service) - } - - // Apply pagination - const limit = params.limit || 100 - const offset = params.offset || 0 - return filtered.slice(offset, offset + limit) - } - - // ============= SEARCH & DISCOVERY ============= - - /** - * Unified find method - supports natural language and structured queries - */ - async find(query: string | FindParams): Promise[]> { - await this.ensureInitialized() - - // Parse natural language queries - const params: FindParams = - typeof query === 'string' ? await this.parseNaturalQuery(query) : query - - return this.augmentationRegistry.execute('find', params, async () => { - let results: Result[] = [] - - // Vector search - if (params.query || params.vector) { - const vector = params.vector || (await this.embed(params.query!)) - const limit = params.limit || 10 - - // Search index - returns array of [id, score] tuples - const searchResults = await this.index.search(vector, limit * 2) // Get extra for filtering - - // Hydrate results - for (const [id, score] of searchResults) { - const entity = await this.get(id) - if (entity) { - results.push({ - id, - score, - entity - }) - } - } - } - - // Proximity search - if (params.near) { - const nearEntity = await this.get(params.near.id) - if (nearEntity) { - const nearResults = await this.index.search( - nearEntity.vector, - params.limit || 10 - ) - - for (const [id, score] of nearResults) { - if (score >= (params.near.threshold || 0.7)) { - const entity = await this.get(id) - if (entity) { - results.push({ - id, - score, - entity - }) - } - } - } - } - } - - // Apply filters - if (params.where) { - results = results.filter((r) => - matchesMetadataFilter(r.entity.metadata, params.where!) - ) - } - - if (params.type) { - const types = Array.isArray(params.type) ? params.type : [params.type] - results = results.filter((r) => types.includes(r.entity.type)) - } - - if (params.service) { - results = results.filter((r) => r.entity.service === params.service) - } - - // Graph constraints - if (params.connected) { - results = await this.applyGraphConstraints(results, params.connected) - } - - // Sort by score and limit - results.sort((a, b) => b.score - a.score) - const limit = params.limit || 10 - const offset = params.offset || 0 - - return results.slice(offset, offset + limit) - }) - } - - /** - * Find similar entities - */ - async similar(params: SimilarParams): Promise[]> { - await this.ensureInitialized() - - // Get target vector - let targetVector: Vector - - if (typeof params.to === 'string') { - const entity = await this.get(params.to) - if (!entity) { - throw new Error(`Entity ${params.to} not found`) - } - targetVector = entity.vector - } else if (Array.isArray(params.to)) { - targetVector = params.to as Vector - } else { - targetVector = (params.to as Entity).vector - } - - // Use find with vector - return this.find({ - vector: targetVector, - limit: params.limit, - type: params.type, - where: params.where, - service: params.service - }) - } - - // ============= BATCH OPERATIONS ============= - - /** - * Add multiple entities - */ - async addMany(params: AddManyParams): Promise> { - await this.ensureInitialized() - - const result: BatchResult = { - successful: [], - failed: [], - total: params.items.length, - duration: 0 - } - - const startTime = Date.now() - const chunkSize = params.chunkSize || 100 - - // Process in chunks - for (let i = 0; i < params.items.length; i += chunkSize) { - const chunk = params.items.slice(i, i + chunkSize) - - const promises = chunk.map(async (item) => { - try { - const id = await this.add(item) - result.successful.push(id) - } catch (error) { - result.failed.push({ - item, - error: (error as Error).message - }) - if (!params.continueOnError) { - throw error - } - } - }) - - if (params.parallel !== false) { - await Promise.allSettled(promises) - } else { - for (const promise of promises) { - await promise - } - } - - // Report progress - if (params.onProgress) { - params.onProgress( - result.successful.length + result.failed.length, - result.total - ) - } - } - - result.duration = Date.now() - startTime - return result - } - - /** - * Delete multiple entities - */ - async deleteMany(params: DeleteManyParams): Promise> { - await this.ensureInitialized() - - // Determine what to delete - let idsToDelete: string[] = [] - - if (params.ids) { - idsToDelete = params.ids - } else if (params.type || params.where) { - // Find entities to delete - const entities = await this.find({ - type: params.type, - where: params.where, - limit: params.limit || 1000 - }) - idsToDelete = entities.map((e) => e.id) - } - - const result: BatchResult = { - successful: [], - failed: [], - total: idsToDelete.length, - duration: 0 - } - - const startTime = Date.now() - - for (const id of idsToDelete) { - try { - await this.delete(id) - result.successful.push(id) - } catch (error) { - result.failed.push({ - item: id, - error: (error as Error).message - }) - } - - if (params.onProgress) { - params.onProgress( - result.successful.length + result.failed.length, - result.total - ) - } - } - - result.duration = Date.now() - startTime - return result - } - - // ============= SUB-APIS ============= - - /** - * Neural API - Advanced AI operations - */ - neural() { - if (!this._neural) { - this._neural = new ImprovedNeuralAPI(this as any) - } - return this._neural - } - return this._neural - } - - /** - * Neural API - Advanced AI operations - */ - neural() { - if (!this._neural) { - this._neural = new ImprovedNeuralAPI(this as any) - } - return this._neural - } - - /** - * Natural Language Processing API - */ - nlp() { - if (!this._nlp) { - this._nlp = new NaturalLanguageProcessor(this as any) - } - return this._nlp - } - - /** - * Create a streaming pipeline - */ - stream() { - const { Pipeline } = require('./streaming/pipeline.js') - return new Pipeline(this) - } - - /** - * Get insights about the data - */ - async insights(): Promise<{ - entities: number - relationships: number - types: Record - services: string[] - density: number - }> { - await this.ensureInitialized() - - // Get all entities count - const allEntities = await this.storage.getAllNouns() - const entities = allEntities.length - - // Get relationships count - const allVerbs = await this.storage.getAllVerbs() - const relationships = allVerbs.length - - // Count by type - const types: Record = {} - for (const entity of allEntities) { - const type = entity.metadata?.noun || 'unknown' - types[type] = (types[type] || 0) + 1 - } - - // Get unique services - const services = [...new Set(allEntities.map(e => e.metadata?.service).filter(Boolean))] - - // Calculate density (relationships per entity) - const density = entities > 0 ? relationships / entities : 0 - - return { - entities, - relationships, - types, - services, - density - } - } - - /** - * Augmentations API - Clean and simple - */ - get augmentations() { - return { - list: () => this.augmentationRegistry.getAll().map(a => a.name), - get: (name: string) => this.augmentationRegistry.getAll().find(a => a.name === name), - has: (name: string) => this.augmentationRegistry.getAll().some(a => a.name === name) - } - } - - // ============= HELPER METHODS ============= - - /** - * Parse natural language query - */ - private async parseNaturalQuery(query: string): Promise> { - if (!this._nlp) { - this._nlp = new NaturalLanguageProcessor(this as any) - } - const parsed = await this._nlp.processNaturalQuery(query) - return parsed as FindParams - } - - /** - * Apply graph constraints to results - */ - private async applyGraphConstraints( - results: Result[], - constraints: any - ): Promise[]> { - // Filter by graph connections - if (constraints.to || constraints.from) { - const filtered: Result[] = [] - - for (const result of results) { - if (constraints.to) { - const verbs = await this.storage.getVerbsBySource(result.id) - const hasConnection = verbs.some(v => v.targetId === constraints.to) - if (hasConnection) { - filtered.push(result) - } - } - - if (constraints.from) { - const verbs = await this.storage.getVerbsByTarget(result.id) - const hasConnection = verbs.some(v => v.sourceId === constraints.from) - if (hasConnection) { - filtered.push(result) - } - } - } - - return filtered - } - - return results - } - - /** - * Convert verbs to relations - */ - private verbsToRelations(verbs: GraphVerb[]): Relation[] { - return verbs.map((v) => ({ - id: v.id, - from: v.sourceId, - to: v.targetId, - type: (v.verb || v.type) as VerbType, - weight: v.weight, - metadata: v.metadata, - service: v.metadata?.service as string, - createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now() - })) - } - - /** - * Embed data into vector - */ - private async embed(data: any): Promise { - return this.embedder(data) - } - - /** - * Warm up the system - */ - private async warmup(): Promise { - // Warm up embedder - await this.embed('warmup') - } - - /** - * Setup embedder - */ - private setupEmbedder(): EmbeddingFunction { - if (this.config.model?.type === 'custom' && this.config.model.name) { - // TODO: Load custom model - return defaultEmbeddingFunction - } - return defaultEmbeddingFunction - } - - /** - * Setup storage - */ - private async setupStorage(): Promise { - const storage = await createStorage({ - type: this.config.storage?.type || 'memory', - ...this.config.storage?.options - }) - return storage - } - - /** - * Setup index - */ - private setupIndex(): HNSWIndex | HNSWIndexOptimized { - const indexConfig = { - ...this.config.index, - distanceFunction: this.distance - } - - // Use optimized index for larger datasets - if (this.config.storage?.type !== 'memory') { - return new HNSWIndexOptimized(indexConfig, this.distance, this.storage) - } - - return new HNSWIndex(indexConfig as any) - } - - /** - * Setup augmentations - */ - private setupAugmentations(): AugmentationRegistry { - const registry = new AugmentationRegistry() - - // Register default augmentations - const defaults = createDefaultAugmentations(this.config.augmentations) - for (const aug of defaults) { - registry.register(aug) - } - - return registry - } - - /** - * Normalize configuration - */ - private normalizeConfig(config?: BrainyConfig): Required { - return { - storage: config?.storage || { type: 'memory' }, - model: config?.model || { type: 'fast' }, - index: config?.index || {}, - cache: config?.cache ?? true, - augmentations: config?.augmentations || {}, - warmup: config?.warmup ?? false, - realtime: config?.realtime ?? false, - multiTenancy: config?.multiTenancy ?? false, - telemetry: config?.telemetry ?? false - } - } - - /** - * Close and cleanup - */ - async close(): Promise { - // Shutdown augmentations - const augs = this.augmentationRegistry.getAll() - for (const aug of augs) { - if ('shutdown' in aug && typeof aug.shutdown === 'function') { - await aug.shutdown() - } - } - - // Storage doesn't have close in current interface - // We'll just mark as not initialized - this.initialized = false - } -} - -// Re-export types for convenience -export * from './types/brainy.types.js' -export { NounType, VerbType } from './types/graphTypes.js' \ No newline at end of file diff --git a/src/brainy.ts b/src/brainy.ts index bdbcb01f..06259f06 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -7,7 +7,6 @@ import { v4 as uuidv4 } from './universal/uuid.js' import { HNSWIndex } from './hnsw/hnswIndex.js' -import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js' import { TypeAwareHNSWIndex } from './hnsw/typeAwareHNSWIndex.js' import { createStorage } from './storage/storageFactory.js' import { BaseStorage } from './storage/baseStorage.js' @@ -90,7 +89,7 @@ export class Brainy implements BrainyInterface { private static instances: Brainy[] = [] // Core components - private index!: HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex + private index!: HNSWIndex | TypeAwareHNSWIndex private storage!: BaseStorage private metadataIndex!: MetadataIndexManager private graphIndex!: GraphAdjacencyIndex @@ -1010,7 +1009,7 @@ export class Brainy implements BrainyInterface { metadata.noun as any ) ) - } else if (this.index instanceof HNSWIndex || this.index instanceof HNSWIndexOptimized) { + } else if (this.index instanceof HNSWIndex) { tx.addOperation( new RemoveFromHNSWOperation(this.index, id, noun.vector) ) @@ -2253,7 +2252,7 @@ export class Brainy implements BrainyInterface { metadata.noun as any ) ) - } else if (this.index instanceof HNSWIndex || this.index instanceof HNSWIndexOptimized) { + } else if (this.index instanceof HNSWIndex) { tx.addOperation( new RemoveFromHNSWOperation(this.index, id, noun.vector) ) @@ -4770,7 +4769,7 @@ export class Brainy implements BrainyInterface { * - 10x faster type-specific queries * - Automatic type routing */ - private setupIndex(): HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex { + private setupIndex(): HNSWIndex | TypeAwareHNSWIndex { const indexConfig = { ...this.config.index, distanceFunction: this.distance diff --git a/src/hnsw/distributedSearch.ts b/src/hnsw/distributedSearch.ts deleted file mode 100644 index e9550e2f..00000000 --- a/src/hnsw/distributedSearch.ts +++ /dev/null @@ -1,636 +0,0 @@ -/** - * Distributed Search System for Large-Scale HNSW Indices - * Implements parallel search across multiple partitions and instances - */ - -import { Vector, HNSWNoun } from '../coreTypes.js' -import { PartitionedHNSWIndex } from './partitionedHNSWIndex.js' -import { executeInThread } from '../utils/workerUtils.js' - -// Search task for parallel execution -interface SearchTask { - partitionId: string - queryVector: Vector - k: number - searchId: string - priority: number -} - -// Search result from a partition -interface PartitionSearchResult { - partitionId: string - results: Array<[string, number]> - searchTime: number - nodesVisited: number - error?: Error -} - -// Distributed search configuration -interface DistributedSearchConfig { - maxConcurrentSearches?: number - searchTimeout?: number - resultMergeStrategy?: 'distance' | 'score' | 'hybrid' - adaptivePartitionSelection?: boolean - redundantSearches?: number - loadBalancing?: boolean -} - -// Search coordination strategies -export enum SearchStrategy { - BROADCAST = 'broadcast', // Search all partitions - SELECTIVE = 'selective', // Search subset of partitions - ADAPTIVE = 'adaptive', // Dynamically adjust based on results - HIERARCHICAL = 'hierarchical' // Multi-level search -} - -// Worker thread pool for parallel search -interface SearchWorker { - id: string - busy: boolean - tasksCompleted: number - averageTaskTime: number - lastTaskTime: number -} - -/** - * Distributed search coordinator for large-scale vector search - */ -export class DistributedSearchSystem { - private config: Required - private searchWorkers: Map = new Map() - private searchQueue: SearchTask[] = [] - private activeSearches: Map> = new Map() - private partitionStats: Map = new Map() - - // Performance monitoring - private searchStats = { - totalSearches: 0, - averageLatency: 0, - parallelEfficiency: 0, - cacheHitRate: 0, - partitionUtilization: new Map() - } - - constructor(config: Partial = {}) { - this.config = { - maxConcurrentSearches: 10, - searchTimeout: 30000, // 30 seconds - resultMergeStrategy: 'hybrid', - adaptivePartitionSelection: true, - redundantSearches: 0, - loadBalancing: true, - ...config - } - - this.initializeWorkerPool() - } - - /** - * Execute distributed search across multiple partitions - */ - public async distributedSearch( - partitionedIndex: PartitionedHNSWIndex, - queryVector: Vector, - k: number, - strategy: SearchStrategy = SearchStrategy.ADAPTIVE - ): Promise> { - const searchId = this.generateSearchId() - const startTime = Date.now() - - try { - // Select partitions to search based on strategy - const partitionsToSearch = await this.selectPartitions( - partitionedIndex, - queryVector, - strategy - ) - - // Create search tasks - const searchTasks = this.createSearchTasks( - partitionsToSearch, - queryVector, - k, - searchId - ) - - // Execute searches in parallel - const searchResults = await this.executeParallelSearches( - partitionedIndex, - searchTasks - ) - - // Merge results from all partitions - const mergedResults = this.mergeSearchResults(searchResults, k) - - // Update statistics - this.updateSearchStats(searchId, startTime, searchResults) - - return mergedResults - - } catch (error) { - console.error(`Distributed search ${searchId} failed:`, error) - throw error - } - } - - /** - * Select partitions to search based on strategy - */ - private async selectPartitions( - partitionedIndex: PartitionedHNSWIndex, - queryVector: Vector, - strategy: SearchStrategy - ): Promise { - const stats = partitionedIndex.getPartitionStats() - const allPartitionIds = stats.partitionDetails.map(p => p.id) - - switch (strategy) { - case SearchStrategy.BROADCAST: - return allPartitionIds - - case SearchStrategy.SELECTIVE: - return this.selectTopPartitions(allPartitionIds, 3) - - case SearchStrategy.ADAPTIVE: - return await this.adaptivePartitionSelection(allPartitionIds, queryVector) - - case SearchStrategy.HIERARCHICAL: - return this.hierarchicalPartitionSelection(allPartitionIds) - - default: - return allPartitionIds - } - } - - /** - * Adaptive partition selection based on historical performance - */ - private async adaptivePartitionSelection( - partitionIds: string[], - queryVector: Vector - ): Promise { - const candidates: Array<{ id: string; score: number }> = [] - - for (const partitionId of partitionIds) { - const stats = this.partitionStats.get(partitionId) - let score = 1.0 - - if (stats) { - // Score based on performance metrics - const speedScore = 1000 / Math.max(stats.averageSearchTime, 1) - const loadScore = Math.max(0, 1 - stats.load) - const qualityScore = stats.quality - const recencyScore = Math.max(0, 1 - (Date.now() - stats.lastUsed) / 3600000) - - score = speedScore * 0.3 + loadScore * 0.25 + qualityScore * 0.3 + recencyScore * 0.15 - } - - candidates.push({ id: partitionId, score }) - } - - // Sort by score and select top partitions - candidates.sort((a, b) => b.score - a.score) - const selectedCount = Math.min(Math.ceil(partitionIds.length * 0.6), 8) - - return candidates.slice(0, selectedCount).map(c => c.id) - } - - /** - * Select top-performing partitions - */ - private selectTopPartitions(partitionIds: string[], count: number): string[] { - const withStats = partitionIds.map(id => ({ - id, - stats: this.partitionStats.get(id) - })) - - // Sort by average search time (faster is better) - withStats.sort((a, b) => { - const timeA = a.stats?.averageSearchTime || 1000 - const timeB = b.stats?.averageSearchTime || 1000 - return timeA - timeB - }) - - return withStats.slice(0, count).map(p => p.id) - } - - /** - * Hierarchical partition selection for very large datasets - */ - private hierarchicalPartitionSelection(partitionIds: string[]): string[] { - // First level: select representative partitions - const firstLevel = partitionIds.filter((_, index) => index % 3 === 0) - - // Could implement a two-phase search here: - // 1. Quick search on representative partitions - // 2. Detailed search on promising partitions - - return firstLevel - } - - /** - * Create search tasks for parallel execution - */ - private createSearchTasks( - partitionIds: string[], - queryVector: Vector, - k: number, - searchId: string - ): SearchTask[] { - const tasks: SearchTask[] = [] - - for (let i = 0; i < partitionIds.length; i++) { - const partitionId = partitionIds[i] - const stats = this.partitionStats.get(partitionId) - - // Calculate priority based on partition performance - const priority = stats ? (1000 - stats.averageSearchTime) : 500 - - tasks.push({ - partitionId, - queryVector: [...queryVector], // Clone vector - k: Math.max(k * 2, 20), // Search for more results per partition - searchId, - priority - }) - - // Add redundant searches if configured - if (this.config.redundantSearches > 0 && i < this.config.redundantSearches) { - tasks.push({ - partitionId, - queryVector: [...queryVector], - k: Math.max(k * 2, 20), - searchId: `${searchId}_redundant_${i}`, - priority: priority - 100 // Lower priority for redundant searches - }) - } - } - - // Sort tasks by priority - tasks.sort((a, b) => b.priority - a.priority) - return tasks - } - - /** - * Execute searches in parallel across selected partitions - */ - private async executeParallelSearches( - partitionedIndex: PartitionedHNSWIndex, - searchTasks: SearchTask[] - ): Promise { - const results: PartitionSearchResult[] = [] - const semaphore = new Semaphore(this.config.maxConcurrentSearches) - - // Execute tasks with controlled concurrency - const taskPromises = searchTasks.map(async (task) => { - await semaphore.acquire() - - try { - const startTime = Date.now() - - // Execute search with timeout - const searchPromise = this.executePartitionSearch(partitionedIndex, task) - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => reject(new Error('Search timeout')), this.config.searchTimeout) - }) - - const result = await Promise.race([searchPromise, timeoutPromise]) - result.searchTime = Date.now() - startTime - - return result - - } catch (error) { - return { - partitionId: task.partitionId, - results: [], - searchTime: this.config.searchTimeout, - nodesVisited: 0, - error: error as Error - } - } finally { - semaphore.release() - } - }) - - // Wait for all searches to complete - const taskResults = await Promise.allSettled(taskPromises) - - for (const result of taskResults) { - if (result.status === 'fulfilled') { - results.push(result.value) - } - } - - return results - } - - /** - * Execute search on a single partition - */ - private async executePartitionSearch( - partitionedIndex: PartitionedHNSWIndex, - task: SearchTask - ): Promise { - try { - // Use thread pool for compute-intensive operations - if (this.shouldUseWorkerThread(task)) { - return await this.executeInWorkerThread(partitionedIndex, task) - } - - // Execute search directly - const results = await partitionedIndex.search( - task.queryVector, - task.k, - { partitionIds: [task.partitionId] } - ) - - return { - partitionId: task.partitionId, - results, - searchTime: 0, // Will be set by caller - nodesVisited: results.length // Approximation - } - - } catch (error) { - throw new Error(`Partition search failed: ${error}`) - } - } - - /** - * Determine if search should use worker thread - */ - private shouldUseWorkerThread(task: SearchTask): boolean { - // Use worker threads for high-dimensional vectors or large k - return task.queryVector.length > 512 || task.k > 100 - } - - /** - * Execute search in worker thread - */ - private async executeInWorkerThread( - partitionedIndex: PartitionedHNSWIndex, - task: SearchTask - ): Promise { - const worker = this.getAvailableWorker() - - if (!worker) { - // No available workers, execute synchronously - return this.executePartitionSearch(partitionedIndex, task) - } - - try { - worker.busy = true - const startTime = Date.now() - - // Execute in thread (simplified - would need proper worker setup) - const searchFunction = ` - return partitionedIndex.search( - task.queryVector, - task.k, - { partitionIds: [task.partitionId] } - ) - ` - const results = await executeInThread>(searchFunction, { - queryVector: task.queryVector, - k: task.k, - partitionId: task.partitionId - }) - - const searchTime = Date.now() - startTime - worker.averageTaskTime = (worker.averageTaskTime + searchTime) / 2 - worker.tasksCompleted++ - - return { - partitionId: task.partitionId, - results: results || [] as Array<[string, number]>, - searchTime, - nodesVisited: results ? results.length : 0 - } - - } finally { - worker.busy = false - worker.lastTaskTime = Date.now() - } - } - - /** - * Get available worker from pool - */ - private getAvailableWorker(): SearchWorker | null { - for (const worker of this.searchWorkers.values()) { - if (!worker.busy) { - return worker - } - } - return null - } - - /** - * Merge search results from multiple partitions - */ - private mergeSearchResults( - partitionResults: PartitionSearchResult[], - k: number - ): Array<[string, number]> { - const allResults: Array<[string, number]> = [] - const seenIds = new Set() - - // Collect all unique results - for (const partitionResult of partitionResults) { - if (partitionResult.error) { - console.warn(`Partition ${partitionResult.partitionId} failed:`, partitionResult.error) - continue - } - - for (const [id, distance] of partitionResult.results) { - if (!seenIds.has(id)) { - allResults.push([id, distance]) - seenIds.add(id) - } - } - } - - // Sort and return top k results - switch (this.config.resultMergeStrategy) { - case 'distance': - allResults.sort((a, b) => a[1] - b[1]) - break - - case 'score': - // Convert distance to score (1 / (1 + distance)) - allResults.sort((a, b) => { - const scoreA = 1 / (1 + a[1]) - const scoreB = 1 / (1 + b[1]) - return scoreB - scoreA - }) - break - - case 'hybrid': - // Weighted combination of distance and partition quality - allResults.sort((a, b) => { - const qualityWeightA = this.getPartitionQuality(a[0]) - const qualityWeightB = this.getPartitionQuality(b[0]) - - const adjustedDistanceA = a[1] / (qualityWeightA + 0.1) - const adjustedDistanceB = b[1] / (qualityWeightB + 0.1) - - return adjustedDistanceA - adjustedDistanceB - }) - break - } - - return allResults.slice(0, k) - } - - /** - * Get partition quality score - */ - private getPartitionQuality(nodeId: string): number { - // This would require knowing which partition a node came from - // For now, return a default quality score - return 1.0 - } - - /** - * Update search statistics - */ - private updateSearchStats( - searchId: string, - startTime: number, - results: PartitionSearchResult[] - ): void { - const totalTime = Date.now() - startTime - const successfulSearches = results.filter(r => !r.error) - - // Update global stats - this.searchStats.totalSearches++ - this.searchStats.averageLatency = - (this.searchStats.averageLatency + totalTime) / 2 - - // Calculate parallel efficiency - const totalPartitionTime = results.reduce((sum, r) => sum + r.searchTime, 0) - this.searchStats.parallelEfficiency = - totalPartitionTime > 0 ? totalTime / totalPartitionTime : 0 - - // Update partition statistics - for (const result of successfulSearches) { - let stats = this.partitionStats.get(result.partitionId) - - if (!stats) { - stats = { - averageSearchTime: result.searchTime, - load: 0, - quality: 1.0, - lastUsed: Date.now() - } - } else { - stats.averageSearchTime = (stats.averageSearchTime + result.searchTime) / 2 - stats.lastUsed = Date.now() - } - - this.partitionStats.set(result.partitionId, stats) - this.searchStats.partitionUtilization.set( - result.partitionId, - (this.searchStats.partitionUtilization.get(result.partitionId) || 0) + 1 - ) - } - } - - /** - * Initialize worker thread pool - */ - private initializeWorkerPool(): void { - const workerCount = Math.min(navigator.hardwareConcurrency || 4, 8) - - for (let i = 0; i < workerCount; i++) { - const worker: SearchWorker = { - id: `worker_${i}`, - busy: false, - tasksCompleted: 0, - averageTaskTime: 0, - lastTaskTime: 0 - } - - this.searchWorkers.set(worker.id, worker) - } - - console.log(`Initialized worker pool with ${workerCount} workers`) - } - - /** - * Generate unique search ID - */ - private generateSearchId(): string { - return `search_${Date.now()}_${Math.random().toString(36).substr(2, 9)}` - } - - /** - * Get search performance statistics - */ - public getSearchStats(): typeof this.searchStats & { - workerStats: SearchWorker[] - partitionStats: Array<{ id: string; stats: any }> - } { - return { - ...this.searchStats, - workerStats: Array.from(this.searchWorkers.values()), - partitionStats: Array.from(this.partitionStats.entries()).map(([id, stats]) => ({ - id, - stats - })) - } - } - - /** - * Cleanup resources - */ - public cleanup(): void { - // Clear active searches - this.activeSearches.clear() - - // Reset worker states - for (const worker of this.searchWorkers.values()) { - worker.busy = false - } - - // Clear statistics - this.partitionStats.clear() - } -} - -/** - * Simple semaphore for concurrency control - */ -class Semaphore { - private permits: number - private waiting: Array<() => void> = [] - - constructor(permits: number) { - this.permits = permits - } - - async acquire(): Promise { - if (this.permits > 0) { - this.permits-- - return Promise.resolve() - } - - return new Promise((resolve) => { - this.waiting.push(resolve) - }) - } - - release(): void { - if (this.waiting.length > 0) { - const resolve = this.waiting.shift()! - resolve() - } else { - this.permits++ - } - } -} \ No newline at end of file diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 251c7c13..1486cd5e 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -490,6 +490,29 @@ export class HNSWIndex { return id } + /** + * O(1) entry point recovery using highLevelNodes index (v6.2.3). + * At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist. + * For tiny indexes with only level 0-1 nodes, any node works as entry point. + */ + private recoverEntryPointO1(): { id: string | null; level: number } { + // O(1) recovery: check highLevelNodes from highest to lowest level + for (let level = this.MAX_TRACKED_LEVELS; level >= 2; level--) { + const nodesAtLevel = this.highLevelNodes.get(level) + if (nodesAtLevel && nodesAtLevel.size > 0) { + for (const nodeId of nodesAtLevel) { + if (this.nouns.has(nodeId)) { + return { id: nodeId, level } + } + } + } + } + + // No high-level nodes - use any available node (works fine for HNSW) + const firstNode = this.nouns.keys().next().value + return { id: firstNode ?? null, level: 0 } + } + /** * Search for nearest neighbors */ @@ -514,20 +537,12 @@ export class HNSWIndex { } // Start from the entry point - // If entry point is null but nouns exist, attempt recovery (v6.2.2) + // If entry point is null but nouns exist, attempt O(1) recovery (v6.2.3) if (!this.entryPointId && this.nouns.size > 0) { - // Corrupted state: nouns exist but entry point is null - recover - let maxLevel = 0 - let recoveredId: string | null = null - for (const [id, noun] of this.nouns.entries()) { - if (noun.level >= maxLevel) { - maxLevel = noun.level - recoveredId = id - } - } + const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1() if (recoveredId) { this.entryPointId = recoveredId - this.maxLevel = maxLevel + this.maxLevel = recoveredLevel } } @@ -538,19 +553,12 @@ export class HNSWIndex { let entryPoint = this.nouns.get(this.entryPointId) if (!entryPoint) { - // Entry point ID exists but noun was deleted - attempt recovery + // Entry point ID exists but noun was deleted - O(1) recovery (v6.2.3) if (this.nouns.size > 0) { - let maxLevel = 0 - let recoveredId: string | null = null - for (const [id, noun] of this.nouns.entries()) { - if (noun.level >= maxLevel) { - maxLevel = noun.level - recoveredId = id - } - } + const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1() if (recoveredId) { this.entryPointId = recoveredId - this.maxLevel = maxLevel + this.maxLevel = recoveredLevel entryPoint = this.nouns.get(recoveredId) } } @@ -1200,28 +1208,20 @@ export class HNSWIndex { } } - // Step 5: CRITICAL - Recover entry point if missing (v6.2.2) + // Step 5: CRITICAL - Recover entry point if missing (v6.2.3 - O(1)) // This ensures consistency even if getHNSWSystem() returned null if (this.nouns.size > 0 && this.entryPointId === null) { - prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering from loaded data') + prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup') - let maxLevel = 0 - let recoveredEntryPointId: string | null = null + const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1() - for (const [id, noun] of this.nouns.entries()) { - if (noun.level >= maxLevel) { - maxLevel = noun.level - recoveredEntryPointId = id - } - } + this.entryPointId = recoveredId + this.maxLevel = recoveredLevel - this.entryPointId = recoveredEntryPointId - this.maxLevel = maxLevel - - prodLog.info(`HNSW entry point recovered: ${recoveredEntryPointId} at level ${maxLevel}`) + prodLog.info(`HNSW entry point recovered: ${recoveredId} at level ${recoveredLevel}`) // Persist recovered state to prevent future recovery - if (this.storage && recoveredEntryPointId) { + if (this.storage && recoveredId) { await this.storage.saveHNSWSystem({ entryPointId: this.entryPointId, maxLevel: this.maxLevel @@ -1233,18 +1233,11 @@ export class HNSWIndex { // Step 6: Validate entry point exists if set (handles stale/deleted entry point) if (this.entryPointId && !this.nouns.has(this.entryPointId)) { - prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering`) + prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering with O(1) lookup`) - let maxLevel = 0 - let recoveredId: string | null = null - for (const [id, noun] of this.nouns.entries()) { - if (noun.level >= maxLevel) { - maxLevel = noun.level - recoveredId = id - } - } + const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1() this.entryPointId = recoveredId - this.maxLevel = maxLevel + this.maxLevel = recoveredLevel // Persist corrected state if (this.storage && recoveredId) { diff --git a/src/hnsw/hnswIndexOptimized.ts b/src/hnsw/hnswIndexOptimized.ts deleted file mode 100644 index 6a3fca8e..00000000 --- a/src/hnsw/hnswIndexOptimized.ts +++ /dev/null @@ -1,585 +0,0 @@ -/** - * Optimized HNSW (Hierarchical Navigable Small World) Index implementation - * Extends the base HNSW implementation with support for large datasets - * Uses product quantization for dimensionality reduction and disk-based storage when needed - */ - -import { - DistanceFunction, - HNSWConfig, - HNSWNoun, - Vector, - VectorDocument -} from '../coreTypes.js' -import { HNSWIndex } from './hnswIndex.js' -import type { BaseStorage } from '../storage/baseStorage.js' - -// Configuration for the optimized HNSW index -export interface HNSWOptimizedConfig extends HNSWConfig { - // Memory threshold in bytes - when exceeded, will use disk-based approach - memoryThreshold?: number - - // Product quantization settings - productQuantization?: { - // Whether to use product quantization - enabled: boolean - // Number of subvectors to split the vector into - numSubvectors?: number - // Number of centroids per subvector - numCentroids?: number - } - - // Whether to use disk-based storage for the index - useDiskBasedIndex?: boolean -} - -// Default configuration for the optimized HNSW index -const DEFAULT_OPTIMIZED_CONFIG: HNSWOptimizedConfig = { - M: 16, - efConstruction: 200, - efSearch: 50, - ml: 16, - memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold - productQuantization: { - enabled: false, - numSubvectors: 16, - numCentroids: 256 - }, - useDiskBasedIndex: false -} - -/** - * Product Quantization implementation - * Reduces vector dimensionality by splitting vectors into subvectors - * and quantizing each subvector to the nearest centroid - */ -class ProductQuantizer { - private numSubvectors: number - private numCentroids: number - private centroids: Vector[][] = [] - private subvectorSize: number = 0 - private initialized: boolean = false - private dimension: number = 0 - - constructor(numSubvectors: number = 16, numCentroids: number = 256) { - this.numSubvectors = numSubvectors - this.numCentroids = numCentroids - } - - /** - * Initialize the product quantizer with training data - * @param vectors Training vectors to use for learning centroids - */ - public train(vectors: Vector[]): void { - if (vectors.length === 0) { - throw new Error('Cannot train product quantizer with empty vector set') - } - - this.dimension = vectors[0].length - this.subvectorSize = Math.ceil(this.dimension / this.numSubvectors) - - // Initialize centroids for each subvector - for (let i = 0; i < this.numSubvectors; i++) { - // Extract subvectors from training data - const subvectors: Vector[] = vectors.map((vector) => { - const start = i * this.subvectorSize - const end = Math.min(start + this.subvectorSize, this.dimension) - return vector.slice(start, end) - }) - - // Initialize centroids for this subvector using k-means++ - this.centroids[i] = this.kMeansPlusPlus(subvectors, this.numCentroids) - } - - this.initialized = true - } - - /** - * Quantize a vector using product quantization - * @param vector Vector to quantize - * @returns Array of centroid indices, one for each subvector - */ - public quantize(vector: Vector): number[] { - if (!this.initialized) { - throw new Error('Product quantizer not initialized. Call train() first.') - } - - if (vector.length !== this.dimension) { - throw new Error( - `Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}` - ) - } - - const codes: number[] = [] - - // Quantize each subvector - for (let i = 0; i < this.numSubvectors; i++) { - const start = i * this.subvectorSize - const end = Math.min(start + this.subvectorSize, this.dimension) - const subvector = vector.slice(start, end) - - // Find nearest centroid - let minDist = Number.MAX_VALUE - let nearestCentroidIndex = 0 - - for (let j = 0; j < this.centroids[i].length; j++) { - const centroid = this.centroids[i][j] - const dist = this.euclideanDistanceSquared(subvector, centroid) - - if (dist < minDist) { - minDist = dist - nearestCentroidIndex = j - } - } - - codes.push(nearestCentroidIndex) - } - - return codes - } - - /** - * Reconstruct a vector from its quantized representation - * @param codes Array of centroid indices - * @returns Reconstructed vector - */ - public reconstruct(codes: number[]): Vector { - if (!this.initialized) { - throw new Error('Product quantizer not initialized. Call train() first.') - } - - if (codes.length !== this.numSubvectors) { - throw new Error( - `Code length mismatch: expected ${this.numSubvectors}, got ${codes.length}` - ) - } - - const reconstructed: Vector = [] - - // Reconstruct each subvector - for (let i = 0; i < this.numSubvectors; i++) { - const centroidIndex = codes[i] - const centroid = this.centroids[i][centroidIndex] - - // Add centroid components to reconstructed vector - for (const component of centroid) { - reconstructed.push(component) - } - } - - // Trim to original dimension if needed - return reconstructed.slice(0, this.dimension) - } - - /** - * Compute squared Euclidean distance between two vectors - * @param a First vector - * @param b Second vector - * @returns Squared Euclidean distance - */ - private euclideanDistanceSquared(a: Vector, b: Vector): number { - let sum = 0 - const length = Math.min(a.length, b.length) - - for (let i = 0; i < length; i++) { - const diff = a[i] - b[i] - sum += diff * diff - } - - return sum - } - - /** - * Implement k-means++ algorithm to initialize centroids - * @param vectors Vectors to cluster - * @param k Number of clusters - * @returns Array of centroids - */ - private kMeansPlusPlus(vectors: Vector[], k: number): Vector[] { - if (vectors.length < k) { - // If we have fewer vectors than centroids, use the vectors as centroids - return [...vectors] - } - - const centroids: Vector[] = [] - - // Choose first centroid randomly - const firstIndex = Math.floor(Math.random() * vectors.length) - centroids.push([...vectors[firstIndex]]) - - // Choose remaining centroids - for (let i = 1; i < k; i++) { - // Compute distances to nearest centroid for each vector - const distances: number[] = vectors.map((vector) => { - let minDist = Number.MAX_VALUE - - for (const centroid of centroids) { - const dist = this.euclideanDistanceSquared(vector, centroid) - minDist = Math.min(minDist, dist) - } - - return minDist - }) - - // Compute sum of distances - const distSum = distances.reduce((sum, dist) => sum + dist, 0) - - // Choose next centroid with probability proportional to distance - let r = Math.random() * distSum - let nextIndex = 0 - - for (let j = 0; j < distances.length; j++) { - r -= distances[j] - if (r <= 0) { - nextIndex = j - break - } - } - - centroids.push([...vectors[nextIndex]]) - } - - return centroids - } - - /** - * Get the centroids for each subvector - * @returns Array of centroid arrays - */ - public getCentroids(): Vector[][] { - return this.centroids - } - - /** - * Set the centroids for each subvector - * @param centroids Array of centroid arrays - */ - public setCentroids(centroids: Vector[][]): void { - this.centroids = centroids - this.numSubvectors = centroids.length - this.numCentroids = centroids[0].length - this.initialized = true - } - - /** - * Get the dimension of the vectors - * @returns Dimension - */ - public getDimension(): number { - return this.dimension - } - - /** - * Set the dimension of the vectors - * @param dimension Dimension - */ - public setDimension(dimension: number): void { - this.dimension = dimension - this.subvectorSize = Math.ceil(dimension / this.numSubvectors) - } -} - -/** - * Optimized HNSW Index implementation - * Extends the base HNSW implementation with support for large datasets - * Uses product quantization for dimensionality reduction and disk-based storage when needed - */ -export class HNSWIndexOptimized extends HNSWIndex { - private optimizedConfig: HNSWOptimizedConfig - private productQuantizer: ProductQuantizer | null = null - private useDiskBasedIndex: boolean = false - private useProductQuantization: boolean = false - private quantizedVectors: Map = new Map() - private memoryUsage: number = 0 - private vectorCount: number = 0 - - // Thread safety for memory usage tracking - private memoryUpdateLock: Promise = Promise.resolve() - - constructor( - config: Partial = {}, - distanceFunction: DistanceFunction, - storage: BaseStorage | null = null - ) { - // Initialize base HNSW index with standard config and storage - super(config, distanceFunction, { storage: storage || undefined }) - - // Set optimized config - this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config } - - // Initialize product quantizer if enabled - if (this.optimizedConfig.productQuantization?.enabled) { - this.useProductQuantization = true - this.productQuantizer = new ProductQuantizer( - this.optimizedConfig.productQuantization.numSubvectors, - this.optimizedConfig.productQuantization.numCentroids - ) - } - - // Set disk-based index flag - this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false - // Note: UnifiedCache is inherited from base HNSWIndex class - } - - /** - * Thread-safe method to update memory usage - * @param memoryDelta Change in memory usage (can be negative) - * @param vectorCountDelta Change in vector count (can be negative) - */ - private async updateMemoryUsage(memoryDelta: number, vectorCountDelta: number): Promise { - this.memoryUpdateLock = this.memoryUpdateLock.then(async () => { - this.memoryUsage = Math.max(0, this.memoryUsage + memoryDelta) - this.vectorCount = Math.max(0, this.vectorCount + vectorCountDelta) - }) - await this.memoryUpdateLock - } - - /** - * Thread-safe method to get current memory usage - * @returns Current memory usage and vector count - */ - private async getMemoryUsageAsync(): Promise<{ memoryUsage: number; vectorCount: number }> { - await this.memoryUpdateLock - return { - memoryUsage: this.memoryUsage, - vectorCount: this.vectorCount - } - } - - /** - * Add a vector to the index - * Uses product quantization if enabled and memory threshold is exceeded - */ - public override async addItem(item: VectorDocument): Promise { - // Check if item is defined - if (!item) { - throw new Error('Item is undefined or null') - } - - const { id, vector } = item - - // Check if vector is defined - if (!vector) { - throw new Error('Vector is undefined or null') - } - - // Estimate memory usage for this vector - const vectorMemory = vector.length * 8 // 8 bytes per number (Float64) - const connectionsMemory = this.optimizedConfig.M * this.optimizedConfig.ml * 16 // Estimate for connections - const totalMemory = vectorMemory + connectionsMemory - - // Update memory usage estimate (thread-safe) - await this.updateMemoryUsage(totalMemory, 1) - - // Check if we should switch to product quantization - const currentMemoryUsage = await this.getMemoryUsageAsync() - if ( - this.useProductQuantization && - currentMemoryUsage.memoryUsage > this.optimizedConfig.memoryThreshold! && - this.productQuantizer && - !this.productQuantizer.getDimension() - ) { - // Initialize product quantizer with existing vectors - this.initializeProductQuantizer() - } - - // If product quantization is active, quantize the vector - if ( - this.useProductQuantization && - this.productQuantizer && - this.productQuantizer.getDimension() > 0 - ) { - // Quantize the vector - const codes = this.productQuantizer.quantize(vector) - - // Store the quantized vector - this.quantizedVectors.set(id, codes) - - // Reconstruct the vector for indexing - const reconstructedVector = this.productQuantizer.reconstruct(codes) - - // Add the reconstructed vector to the index - return await super.addItem({ id, vector: reconstructedVector }) - } - - // If disk-based index is active and storage is available, store the vector - if (this.useDiskBasedIndex) { - // Storage is handled by the base class now via HNSW persistence - // No additional storage needed here - } - - // Add the vector to the in-memory index - return await super.addItem(item) - } - - /** - * Search for nearest neighbors - * Uses product quantization if enabled - */ - public override async search( - queryVector: Vector, - k: number = 10 - ): Promise> { - // Check if query vector is defined - if (!queryVector) { - throw new Error('Query vector is undefined or null') - } - - // If product quantization is active, quantize the query vector - if ( - this.useProductQuantization && - this.productQuantizer && - this.productQuantizer.getDimension() > 0 - ) { - // Quantize the query vector - const codes = this.productQuantizer.quantize(queryVector) - - // Reconstruct the query vector - const reconstructedVector = this.productQuantizer.reconstruct(codes) - - // Search with the reconstructed vector - return await super.search(reconstructedVector, k) - } - - // Otherwise, use the standard search - return await super.search(queryVector, k) - } - - /** - * Remove an item from the index - */ - public override async removeItem(id: string): Promise { - // If product quantization is active, remove the quantized vector - if (this.useProductQuantization) { - this.quantizedVectors.delete(id) - } - - // If disk-based index is active, removal is handled by base class - // No additional removal needed here - - // Update memory usage estimate (async operation, but don't block removal) - this.getMemoryUsageAsync().then((currentMemoryUsage) => { - if (currentMemoryUsage.vectorCount > 0) { - const memoryPerVector = currentMemoryUsage.memoryUsage / currentMemoryUsage.vectorCount - this.updateMemoryUsage(-memoryPerVector, -1) - } - }).catch((error) => { - console.error('Failed to update memory usage after removal:', error) - }) - - // Remove the item from the in-memory index - return await super.removeItem(id) - } - - /** - * Clear the index - */ - public override async clear(): Promise { - // Clear product quantization data - if (this.useProductQuantization) { - this.quantizedVectors.clear() - this.productQuantizer = new ProductQuantizer( - this.optimizedConfig.productQuantization!.numSubvectors, - this.optimizedConfig.productQuantization!.numCentroids - ) - } - - // Reset memory usage (thread-safe) - const currentMemoryUsage = await this.getMemoryUsageAsync() - await this.updateMemoryUsage(-currentMemoryUsage.memoryUsage, -currentMemoryUsage.vectorCount) - - // Clear the in-memory index - super.clear() - } - - /** - * Initialize product quantizer with existing vectors - */ - private initializeProductQuantizer(): void { - if (!this.productQuantizer) { - return - } - - // Get all vectors from the index - const nouns = super.getNouns() - const vectors: Vector[] = [] - - // Extract vectors - for (const [_, noun] of nouns) { - vectors.push(noun.vector) - } - - // Train the product quantizer - if (vectors.length > 0) { - this.productQuantizer.train(vectors) - - // Quantize all existing vectors - for (const [id, noun] of nouns) { - const codes = this.productQuantizer.quantize(noun.vector) - this.quantizedVectors.set(id, codes) - } - - console.log( - `Initialized product quantizer with ${vectors.length} vectors` - ) - } - } - - /** - * Get the product quantizer - * @returns Product quantizer or null if not enabled - */ - public getProductQuantizer(): ProductQuantizer | null { - return this.productQuantizer - } - - /** - * Get the optimized configuration - * @returns Optimized configuration - */ - public getOptimizedConfig(): HNSWOptimizedConfig { - return { ...this.optimizedConfig } - } - - /** - * Get the estimated memory usage - * @returns Estimated memory usage in bytes - */ - public getMemoryUsage(): number { - return this.memoryUsage - } - - // Storage methods removed - now handled by base class - - /** - * Set whether to use disk-based index - * @param useDiskBasedIndex Whether to use disk-based index - */ - public setUseDiskBasedIndex(useDiskBasedIndex: boolean): void { - this.useDiskBasedIndex = useDiskBasedIndex - } - - /** - * Get whether disk-based index is used - * @returns Whether disk-based index is used - */ - public getUseDiskBasedIndex(): boolean { - return this.useDiskBasedIndex - } - - /** - * Set whether to use product quantization - * @param useProductQuantization Whether to use product quantization - */ - public setUseProductQuantization(useProductQuantization: boolean): void { - this.useProductQuantization = useProductQuantization - } - - /** - * Get whether product quantization is used - * @returns Whether product quantization is used - */ - public getUseProductQuantization(): boolean { - return this.useProductQuantization - } -} diff --git a/src/hnsw/optimizedHNSWIndex.ts b/src/hnsw/optimizedHNSWIndex.ts deleted file mode 100644 index f5744483..00000000 --- a/src/hnsw/optimizedHNSWIndex.ts +++ /dev/null @@ -1,431 +0,0 @@ -/** - * Optimized HNSW Index for Large-Scale Vector Search - * Implements dynamic parameter tuning and performance optimizations - */ - -import { - DistanceFunction, - HNSWConfig, - HNSWNoun, - Vector, - VectorDocument -} from '../coreTypes.js' -import { HNSWIndex } from './hnswIndex.js' -import { euclideanDistance } from '../utils/index.js' - -export interface OptimizedHNSWConfig extends HNSWConfig { - // Dynamic tuning parameters - dynamicParameterTuning?: boolean - targetSearchLatency?: number // ms - targetRecall?: number // 0.0 to 1.0 - - // Large-scale optimizations - maxNodes?: number - memoryBudget?: number // bytes - diskCacheEnabled?: boolean - compressionEnabled?: boolean - - // Performance monitoring - performanceTracking?: boolean - adaptiveEfSearch?: boolean - - // Advanced optimizations - levelMultiplier?: number - seedConnections?: number - pruningStrategy?: 'simple' | 'diverse' | 'hybrid' -} - -interface PerformanceMetrics { - averageSearchTime: number - averageRecall: number - memoryUsage: number - indexSize: number - apiCalls: number - cacheHitRate: number -} - -interface DynamicParameters { - efSearch: number - efConstruction: number - M: number - ml: number -} - -/** - * Optimized HNSW Index with dynamic parameter tuning for large datasets - */ -export class OptimizedHNSWIndex extends HNSWIndex { - private optimizedConfig: Required> & { maxConcurrentNeighborWrites?: number } - private performanceMetrics: PerformanceMetrics - private dynamicParams: DynamicParameters - private searchHistory: Array<{ latency: number; k: number; timestamp: number }> = [] - private parameterTuningInterval?: NodeJS.Timeout - - constructor( - config: Partial = {}, - distanceFunction: DistanceFunction = euclideanDistance - ) { - // Set optimized defaults for large scale - const defaultConfig: Required> = { - M: 32, // Higher connectivity for better recall - efConstruction: 400, // Better build quality - efSearch: 100, // Dynamic - will be tuned - ml: 24, // Deeper hierarchy - useDiskBasedIndex: false, // Added missing property - dynamicParameterTuning: true, - targetSearchLatency: 100, // 100ms target - targetRecall: 0.95, // 95% recall target - maxNodes: 1000000, // 1M node limit - memoryBudget: 8 * 1024 * 1024 * 1024, // 8GB - diskCacheEnabled: true, - compressionEnabled: false, // Disabled by default for compatibility - performanceTracking: true, - adaptiveEfSearch: true, - levelMultiplier: 16, - seedConnections: 8, - pruningStrategy: 'hybrid' - // maxConcurrentNeighborWrites intentionally omitted - optional property from parent HNSWConfig (v4.10.0+) - } - - const mergedConfig = { ...defaultConfig, ...config } - - // Initialize parent with base config - super( - { - M: mergedConfig.M, - efConstruction: mergedConfig.efConstruction, - efSearch: mergedConfig.efSearch, - ml: mergedConfig.ml - }, - distanceFunction, - { useParallelization: true } - ) - - this.optimizedConfig = mergedConfig - - // Initialize dynamic parameters - this.dynamicParams = { - efSearch: mergedConfig.efSearch, - efConstruction: mergedConfig.efConstruction, - M: mergedConfig.M, - ml: mergedConfig.ml - } - - // Initialize performance metrics - this.performanceMetrics = { - averageSearchTime: 0, - averageRecall: 0, - memoryUsage: 0, - indexSize: 0, - apiCalls: 0, - cacheHitRate: 0 - } - - // Start parameter tuning if enabled - if (this.optimizedConfig.dynamicParameterTuning) { - this.startParameterTuning() - } - } - - /** - * Optimized search with dynamic parameter adjustment - */ - public async search( - queryVector: Vector, - k: number = 10, - filter?: (id: string) => Promise - ): Promise> { - const startTime = Date.now() - - // Adjust efSearch dynamically based on k and performance history - if (this.optimizedConfig.adaptiveEfSearch) { - this.adjustEfSearch(k) - } - - // Check memory usage and trigger optimizations if needed - if (this.optimizedConfig.performanceTracking) { - this.checkMemoryUsage() - } - - // Perform the search with current parameters - const originalConfig = this.getConfig() - - // Temporarily update search parameters - const tempConfig = { - ...originalConfig, - efSearch: this.dynamicParams.efSearch - } - - // Use the parent's search method with optimized parameters - let results: Array<[string, number]> - - try { - // This is a simplified approach - in practice, we'd need to modify - // the parent class to accept runtime parameter changes - results = await super.search(queryVector, k, filter) - } catch (error) { - console.error('Optimized search failed, falling back to default:', error) - results = await super.search(queryVector, k, filter) - } - - // Record performance metrics - const searchTime = Date.now() - startTime - this.recordSearchMetrics(searchTime, k, results.length) - - return results - } - - /** - * Dynamically adjust efSearch based on performance requirements - */ - private adjustEfSearch(k: number): void { - const recentSearches = this.searchHistory.slice(-10) - - if (recentSearches.length < 3) { - // Not enough data, use heuristic - this.dynamicParams.efSearch = Math.max(k * 2, 50) - return - } - - const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length - const targetLatency = this.optimizedConfig.targetSearchLatency - - // Adjust efSearch based on latency performance - if (averageLatency > targetLatency * 1.2) { - // Too slow, reduce efSearch - this.dynamicParams.efSearch = Math.max( - Math.floor(this.dynamicParams.efSearch * 0.9), - k - ) - } else if (averageLatency < targetLatency * 0.8) { - // Fast enough, can increase efSearch for better recall - this.dynamicParams.efSearch = Math.min( - Math.floor(this.dynamicParams.efSearch * 1.1), - 500 // Maximum efSearch - ) - } - - // Ensure efSearch is at least k - this.dynamicParams.efSearch = Math.max(this.dynamicParams.efSearch, k) - } - - /** - * Record search performance metrics - */ - private recordSearchMetrics(latency: number, k: number, resultCount: number): void { - if (!this.optimizedConfig.performanceTracking) { - return - } - - // Add to search history - this.searchHistory.push({ - latency, - k, - timestamp: Date.now() - }) - - // Keep only recent history (last 100 searches) - if (this.searchHistory.length > 100) { - this.searchHistory.shift() - } - - // Update performance metrics - const recentSearches = this.searchHistory.slice(-20) - this.performanceMetrics.averageSearchTime = - recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length - - // Estimate recall (simplified - would need ground truth for accurate measurement) - this.performanceMetrics.averageRecall = Math.min(resultCount / k, 1.0) - } - - /** - * Check memory usage and trigger optimizations - */ - private checkMemoryUsage(): void { - // Estimate memory usage (simplified) - const estimatedMemory = this.size() * 1000 // Rough estimate per node - this.performanceMetrics.memoryUsage = estimatedMemory - - if (estimatedMemory > this.optimizedConfig.memoryBudget * 0.9) { - console.warn('Memory usage approaching limit, consider index partitioning') - - // Could trigger automatic partitioning or compression here - if (this.optimizedConfig.compressionEnabled) { - this.compressIndex() - } - } - } - - /** - * Compress index to reduce memory usage (placeholder) - */ - private compressIndex(): void { - console.log('Index compression not implemented yet') - // This would implement vector quantization or other compression techniques - } - - /** - * Start automatic parameter tuning - */ - private startParameterTuning(): void { - this.parameterTuningInterval = setInterval(() => { - this.tuneParameters() - }, 30000) // Tune every 30 seconds - } - - /** - * Automatic parameter tuning based on performance metrics - */ - private tuneParameters(): void { - if (this.searchHistory.length < 10) { - return // Not enough data - } - - const recentSearches = this.searchHistory.slice(-20) - const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length - - // Tune based on performance vs targets - const latencyRatio = averageLatency / this.optimizedConfig.targetSearchLatency - const recallRatio = this.performanceMetrics.averageRecall / this.optimizedConfig.targetRecall - - // Adjust M (connectivity) for long-term performance - if (this.size() > 10000) { // Only tune for larger indices - if (recallRatio < 0.95 && latencyRatio < 1.5) { - // Recall is low but we have latency budget, increase M - this.dynamicParams.M = Math.min(this.dynamicParams.M + 2, 64) - } else if (latencyRatio > 1.2 && recallRatio > 1.0) { - // Latency is high but recall is good, can reduce M - this.dynamicParams.M = Math.max(this.dynamicParams.M - 2, 16) - } - } - - console.log(`Parameter tuning: efSearch=${this.dynamicParams.efSearch}, M=${this.dynamicParams.M}, latency=${averageLatency.toFixed(1)}ms`) - } - - /** - * Get optimized configuration recommendations for current dataset size - */ - public getOptimizedConfig(): OptimizedHNSWConfig { - const currentSize = this.size() - - let recommendedConfig: Partial = {} - - if (currentSize < 10000) { - // Small dataset - optimize for speed - recommendedConfig = { - M: 16, - efConstruction: 200, - efSearch: 50, - ml: 16 - } - } else if (currentSize < 100000) { - // Medium dataset - balance speed and recall - recommendedConfig = { - M: 24, - efConstruction: 300, - efSearch: 75, - ml: 20 - } - } else if (currentSize < 1000000) { - // Large dataset - optimize for recall - recommendedConfig = { - M: 32, - efConstruction: 400, - efSearch: 100, - ml: 24 - } - } else { - // Very large dataset - maximum quality - recommendedConfig = { - M: 48, - efConstruction: 500, - efSearch: 150, - ml: 28 - } - } - - return { - ...this.optimizedConfig, - ...recommendedConfig - } - } - - /** - * Get current performance metrics - */ - public getPerformanceMetrics(): PerformanceMetrics & { - currentParams: DynamicParameters - searchHistorySize: number - } { - return { - ...this.performanceMetrics, - currentParams: { ...this.dynamicParams }, - searchHistorySize: this.searchHistory.length - } - } - - /** - * Apply optimized bulk insertion strategy - */ - public async bulkInsert(items: VectorDocument[]): Promise { - console.log(`Starting optimized bulk insert of ${items.length} items`) - - // Sort items to optimize insertion order (by vector similarity) - const sortedItems = this.optimizeInsertionOrder(items) - - // Temporarily adjust construction parameters for bulk operations - const originalEfConstruction = this.dynamicParams.efConstruction - this.dynamicParams.efConstruction = Math.min( - this.dynamicParams.efConstruction * 1.5, - 800 - ) - - const results: string[] = [] - const batchSize = 100 - - try { - // Process in batches to manage memory - for (let i = 0; i < sortedItems.length; i += batchSize) { - const batch = sortedItems.slice(i, i + batchSize) - - for (const item of batch) { - const id = await this.addItem(item) - results.push(id) - } - - // Periodic memory check - if (i % (batchSize * 10) === 0) { - this.checkMemoryUsage() - } - } - } finally { - // Restore original construction parameters - this.dynamicParams.efConstruction = originalEfConstruction - } - - console.log(`Completed bulk insert of ${results.length} items`) - return results - } - - /** - * Optimize insertion order to improve index quality - */ - private optimizeInsertionOrder(items: VectorDocument[]): VectorDocument[] { - if (items.length < 100) { - return items // Not worth optimizing small batches - } - - // Simple clustering-based ordering - // In practice, you might use more sophisticated methods - return items.sort(() => Math.random() - 0.5) // Shuffle for now - } - - /** - * Cleanup resources - */ - public destroy(): void { - if (this.parameterTuningInterval) { - clearInterval(this.parameterTuningInterval) - } - } -} \ No newline at end of file diff --git a/src/hnsw/partitionedHNSWIndex.ts b/src/hnsw/partitionedHNSWIndex.ts deleted file mode 100644 index ba823389..00000000 --- a/src/hnsw/partitionedHNSWIndex.ts +++ /dev/null @@ -1,413 +0,0 @@ -/** - * Partitioned HNSW Index for Large-Scale Vector Search - * Implements sharding strategies to handle millions of vectors efficiently - */ - -import { - DistanceFunction, - HNSWConfig, - HNSWNoun, - Vector, - VectorDocument -} from '../coreTypes.js' -import { HNSWIndex } from './hnswIndex.js' -import { euclideanDistance } from '../utils/index.js' - -export interface PartitionConfig { - maxNodesPerPartition: number - partitionStrategy: 'semantic' | 'hash' // Simplified to focus on useful strategies - semanticClusters?: number // Auto-configured based on dataset size - autoTuneSemanticClusters?: boolean // Automatically adjust cluster count -} - -export interface PartitionMetadata { - id: string - nodeCount: number - bounds?: { - centroid: Vector - radius: number - } - strategy: string - created: Date -} - -/** - * Partitioned HNSW Index that splits large datasets across multiple smaller indices - * This enables efficient search across millions of vectors by reducing memory usage - * and parallelizing search operations - */ -export class PartitionedHNSWIndex { - private partitions: Map = new Map() - private partitionMetadata: Map = new Map() - private config: PartitionConfig - private hnswConfig: HNSWConfig - private distanceFunction: DistanceFunction - private dimension: number | null = null - private nextPartitionId = 0 - - constructor( - partitionConfig: Partial = {}, - hnswConfig: Partial = {}, - distanceFunction: DistanceFunction = euclideanDistance - ) { - this.config = { - maxNodesPerPartition: 50000, // Optimal size for memory efficiency - partitionStrategy: 'semantic', // Default to semantic for better performance - semanticClusters: 8, // Auto-tuned based on dataset - autoTuneSemanticClusters: true, - ...partitionConfig - } - - // Optimized HNSW parameters for large scale - this.hnswConfig = { - M: 32, // Higher connectivity for better recall - efConstruction: 400, // Better build quality - efSearch: 100, // Balance speed vs accuracy - ml: 24, // Deeper hierarchy - ...hnswConfig - } - - this.distanceFunction = distanceFunction - } - - /** - * Add a vector to the partitioned index - */ - public async addItem(item: VectorDocument): Promise { - if (this.dimension === null) { - this.dimension = item.vector.length - } - - // Determine which partition this item belongs to - const partitionId = await this.selectPartition(item) - - // Get or create the partition - let partition = this.partitions.get(partitionId) - if (!partition) { - partition = new HNSWIndex( - this.hnswConfig, - this.distanceFunction, - { useParallelization: true } - ) - this.partitions.set(partitionId, partition) - - // Initialize partition metadata - this.partitionMetadata.set(partitionId, { - id: partitionId, - nodeCount: 0, - strategy: this.config.partitionStrategy, - created: new Date() - }) - } - - // Add item to the selected partition - await partition.addItem(item) - - // Update partition metadata - const metadata = this.partitionMetadata.get(partitionId)! - metadata.nodeCount = partition.size() - - // Update bounds for semantic strategy - if (this.config.partitionStrategy === 'semantic') { - this.updatePartitionBounds(partitionId, item.vector) - } - - // Check if partition is getting too large and needs splitting - if (metadata.nodeCount > this.config.maxNodesPerPartition * 1.2) { - await this.splitPartition(partitionId) - } - - return item.id - } - - /** - * Search across all partitions for nearest neighbors - */ - public async search( - queryVector: Vector, - k: number = 10, - searchScope?: { - partitionIds?: string[] - maxPartitions?: number - } - ): Promise> { - if (this.partitions.size === 0) { - return [] - } - - // Determine which partitions to search - const partitionsToSearch = await this.selectSearchPartitions(queryVector, searchScope) - - // Search partitions in parallel - const searchPromises = partitionsToSearch.map(async (partitionId) => { - const partition = this.partitions.get(partitionId) - if (!partition) return [] - - // Search with higher k to get better global results - const partitionK = Math.min(k * 2, partition.size()) - return partition.search(queryVector, partitionK) - }) - - const partitionResults = await Promise.all(searchPromises) - - // Merge and sort results from all partitions - const allResults: Array<[string, number]> = [] - for (const results of partitionResults) { - allResults.push(...results) - } - - // Sort by distance and return top k - allResults.sort((a, b) => a[1] - b[1]) - return allResults.slice(0, k) - } - - /** - * Select the appropriate partition for a new item - * Automatically chooses semantic partitioning when beneficial, falls back to hash - */ - private async selectPartition(item: VectorDocument): Promise { - // Auto-tune semantic clusters based on current dataset size - if (this.config.autoTuneSemanticClusters && this.config.partitionStrategy === 'semantic') { - this.autoTuneSemanticClusters() - } - - switch (this.config.partitionStrategy) { - case 'semantic': - return await this.semanticPartition(item.vector) - - case 'hash': - default: - return this.hashPartition(item.id) - } - } - - /** - * Hash-based partitioning for even distribution - */ - private hashPartition(id: string): string { - const hash = this.simpleHash(id) - const existingPartitions = Array.from(this.partitions.keys()) - - // Find partition with space, or create new one - for (const partitionId of existingPartitions) { - const metadata = this.partitionMetadata.get(partitionId) - if (metadata && metadata.nodeCount < this.config.maxNodesPerPartition) { - return partitionId - } - } - - // Create new partition - return `partition_${this.nextPartitionId++}` - } - - /** - * Semantic clustering partitioning - */ - private async semanticPartition(vector: Vector): Promise { - // Find closest partition centroid - let closestPartition = '' - let minDistance = Infinity - - for (const [partitionId, metadata] of this.partitionMetadata.entries()) { - if (metadata.bounds?.centroid) { - const distance = this.distanceFunction(vector, metadata.bounds.centroid) - if (distance < minDistance) { - minDistance = distance - closestPartition = partitionId - } - } - } - - // If no suitable partition found or it's full, create new one - if (!closestPartition || - this.partitionMetadata.get(closestPartition)!.nodeCount >= this.config.maxNodesPerPartition) { - closestPartition = `semantic_${this.nextPartitionId++}` - } - - return closestPartition - } - - /** - * Auto-tune semantic clusters based on dataset size and performance - */ - private autoTuneSemanticClusters(): void { - const totalNodes = this.size() - const currentPartitions = this.partitions.size - - // Optimal clusters based on dataset size - let optimalClusters = Math.max(4, Math.min(32, Math.floor(totalNodes / 10000))) - - // Adjust based on current partition performance - if (currentPartitions > 0) { - const avgNodesPerPartition = totalNodes / currentPartitions - - if (avgNodesPerPartition > this.config.maxNodesPerPartition * 0.8) { - // Partitions are getting full, increase clusters - optimalClusters = Math.min(32, this.config.semanticClusters! + 2) - } else if (avgNodesPerPartition < this.config.maxNodesPerPartition * 0.3 && currentPartitions > 4) { - // Partitions are underutilized, decrease clusters - optimalClusters = Math.max(4, this.config.semanticClusters! - 1) - } - } - - if (optimalClusters !== this.config.semanticClusters) { - console.log(`Auto-tuning semantic clusters: ${this.config.semanticClusters} → ${optimalClusters}`) - this.config.semanticClusters = optimalClusters - } - } - - /** - * Select which partitions to search based on query - */ - private async selectSearchPartitions( - queryVector: Vector, - searchScope?: { - partitionIds?: string[] - maxPartitions?: number - } - ): Promise { - if (searchScope?.partitionIds) { - return searchScope.partitionIds.filter(id => this.partitions.has(id)) - } - - const maxPartitions = searchScope?.maxPartitions || Math.min(5, this.partitions.size) - - if (this.config.partitionStrategy === 'semantic') { - // Search partitions with closest centroids - const distances: Array<[string, number]> = [] - - for (const [partitionId, metadata] of this.partitionMetadata.entries()) { - if (metadata.bounds?.centroid) { - const distance = this.distanceFunction(queryVector, metadata.bounds.centroid) - distances.push([partitionId, distance]) - } - } - - distances.sort((a, b) => a[1] - b[1]) - return distances.slice(0, maxPartitions).map(([id]) => id) - } - - // For other strategies, search all partitions or random subset - const allPartitionIds = Array.from(this.partitions.keys()) - - if (allPartitionIds.length <= maxPartitions) { - return allPartitionIds - } - - // Return random subset - const shuffled = [...allPartitionIds].sort(() => Math.random() - 0.5) - return shuffled.slice(0, maxPartitions) - } - - /** - * Update partition bounds for semantic clustering - */ - private updatePartitionBounds(partitionId: string, vector: Vector): void { - const metadata = this.partitionMetadata.get(partitionId)! - - if (!metadata.bounds) { - metadata.bounds = { - centroid: [...vector], - radius: 0 - } - return - } - - // Update centroid using incremental mean - const { centroid } = metadata.bounds - const nodeCount = metadata.nodeCount - - for (let i = 0; i < centroid.length; i++) { - centroid[i] = (centroid[i] * (nodeCount - 1) + vector[i]) / nodeCount - } - - // Update radius - const distance = this.distanceFunction(vector, centroid) - metadata.bounds.radius = Math.max(metadata.bounds.radius, distance) - } - - /** - * Split an overgrown partition into smaller partitions - */ - private async splitPartition(partitionId: string): Promise { - const partition = this.partitions.get(partitionId) - if (!partition) return - - console.log(`Splitting partition ${partitionId} with ${partition.size()} nodes`) - - // For now, we'll implement a simple strategy - // In a full implementation, you'd want to analyze the data distribution - // and create more intelligent splits - - // This is a placeholder - actual implementation would require - // accessing the internal nodes of the HNSW index - } - - /** - * Simple hash function for consistent partitioning - */ - private simpleHash(str: string): number { - let hash = 0 - for (let i = 0; i < str.length; i++) { - const char = str.charCodeAt(i) - hash = ((hash << 5) - hash) + char - hash = hash & hash // Convert to 32-bit integer - } - return Math.abs(hash) - } - - /** - * Get partition statistics - */ - public getPartitionStats(): { - totalPartitions: number - totalNodes: number - averageNodesPerPartition: number - partitionDetails: PartitionMetadata[] - } { - const partitionDetails = Array.from(this.partitionMetadata.values()) - const totalNodes = partitionDetails.reduce((sum, p) => sum + p.nodeCount, 0) - - return { - totalPartitions: partitionDetails.length, - totalNodes, - averageNodesPerPartition: totalNodes / partitionDetails.length || 0, - partitionDetails - } - } - - /** - * Remove an item from the index - */ - public async removeItem(id: string): Promise { - // Find which partition contains this item - for (const [partitionId, partition] of this.partitions.entries()) { - if (await partition.removeItem(id)) { - // Update metadata - const metadata = this.partitionMetadata.get(partitionId)! - metadata.nodeCount = partition.size() - return true - } - } - return false - } - - /** - * Clear all partitions - */ - public clear(): void { - for (const partition of this.partitions.values()) { - partition.clear() - } - this.partitions.clear() - this.partitionMetadata.clear() - this.nextPartitionId = 0 - } - - /** - * Get total size across all partitions - */ - public size(): number { - return Array.from(this.partitions.values()).reduce((sum, partition) => sum + partition.size(), 0) - } -} \ No newline at end of file diff --git a/src/hnsw/scaledHNSWSystem.ts b/src/hnsw/scaledHNSWSystem.ts deleted file mode 100644 index 874d850c..00000000 --- a/src/hnsw/scaledHNSWSystem.ts +++ /dev/null @@ -1,745 +0,0 @@ -/** - * Scaled HNSW System - Integration of All Optimization Strategies - * Production-ready system for handling millions of vectors with sub-second search - */ - -import { Vector, VectorDocument, HNSWConfig } from '../coreTypes.js' -import { PartitionedHNSWIndex, PartitionConfig } from './partitionedHNSWIndex.js' -import { OptimizedHNSWIndex, OptimizedHNSWConfig } from './optimizedHNSWIndex.js' -import { DistributedSearchSystem, SearchStrategy } from './distributedSearch.js' -import { EnhancedCacheManager } from '../storage/enhancedCacheManager.js' -import { BatchS3Operations } from '../storage/adapters/batchS3Operations.js' -import { ReadOnlyOptimizations } from '../storage/readOnlyOptimizations.js' -import { euclideanDistance } from '../utils/index.js' -import { autoConfigureBrainy, AutoConfiguration } from '../utils/autoConfiguration.js' - -export interface ScaledHNSWConfig { - // Required: Basic dataset expectations (can be auto-detected if not provided) - expectedDatasetSize?: number // Auto-detected if not provided - maxMemoryUsage?: number // Auto-detected based on environment - targetSearchLatency?: number // Auto-configured based on environment - - // Storage configuration (optional - auto-detects S3 availability) - s3Config?: { - bucketName: string - region: string - endpoint?: string - accessKeyId?: string // Falls back to env vars - secretAccessKey?: string // Falls back to env vars - } - - // Auto-configuration options - autoConfigureEnvironment?: boolean // Default: true - learningEnabled?: boolean // Default: true - adapts to performance - - // Manual overrides (optional - auto-configured if not provided) - enablePartitioning?: boolean - enableCompression?: boolean - enableDistributedSearch?: boolean - enablePredictiveCaching?: boolean - - // Advanced manual tuning (optional) - partitionConfig?: Partial - hnswConfig?: Partial - readOnlyMode?: boolean -} - -/** - * High-performance HNSW system with all optimizations integrated - * Handles datasets from thousands to millions of vectors - */ -export class ScaledHNSWSystem { - private config: ScaledHNSWConfig & { - expectedDatasetSize: number - maxMemoryUsage: number - targetSearchLatency: number - autoConfigureEnvironment: boolean - learningEnabled: boolean - enablePartitioning: boolean - enableCompression: boolean - enableDistributedSearch: boolean - enablePredictiveCaching: boolean - readOnlyMode: boolean - } - private autoConfig: AutoConfiguration - private partitionedIndex?: PartitionedHNSWIndex - private distributedSearch?: DistributedSearchSystem - private cacheManager?: EnhancedCacheManager - private batchOperations?: BatchS3Operations - private readOnlyOptimizations?: ReadOnlyOptimizations - - // Performance monitoring and learning - private performanceMetrics = { - totalSearches: 0, - averageSearchTime: 0, - cacheHitRate: 0, - compressionRatio: 0, - memoryUsage: 0, - indexSize: 0, - lastLearningUpdate: Date.now() - } - - constructor(config: ScaledHNSWConfig = {}) { - this.autoConfig = AutoConfiguration.getInstance() - - // Set basic defaults - these will be overridden by auto-configuration - this.config = { - expectedDatasetSize: 100000, - maxMemoryUsage: 4 * 1024 * 1024 * 1024, - targetSearchLatency: 150, - autoConfigureEnvironment: true, - learningEnabled: true, - enablePartitioning: true, - enableCompression: true, - enableDistributedSearch: true, - enablePredictiveCaching: true, - readOnlyMode: false, - ...config - } - - this.initializeOptimizedSystem() - } - - /** - * Initialize the optimized system based on configuration - */ - private async initializeOptimizedSystem(): Promise { - console.log('Initializing Scaled HNSW System with auto-configuration...') - - // Auto-configure if enabled - if (this.config.autoConfigureEnvironment) { - const autoConfigResult = await this.autoConfig.detectAndConfigure({ - expectedDataSize: this.config.expectedDatasetSize, - s3Available: !!this.config.s3Config, - memoryBudget: this.config.maxMemoryUsage - }) - - console.log(`Detected environment: ${autoConfigResult.environment}`) - console.log(`Available memory: ${(autoConfigResult.availableMemory / 1024 / 1024 / 1024).toFixed(1)}GB`) - console.log(`CPU cores: ${autoConfigResult.cpuCores}`) - - // Override config with auto-detected values - this.config = { - ...this.config, - expectedDatasetSize: autoConfigResult.recommendedConfig.expectedDatasetSize, - maxMemoryUsage: autoConfigResult.recommendedConfig.maxMemoryUsage, - targetSearchLatency: autoConfigResult.recommendedConfig.targetSearchLatency, - enablePartitioning: autoConfigResult.recommendedConfig.enablePartitioning, - enableCompression: autoConfigResult.recommendedConfig.enableCompression, - enableDistributedSearch: autoConfigResult.recommendedConfig.enableDistributedSearch, - enablePredictiveCaching: autoConfigResult.recommendedConfig.enablePredictiveCaching - } - } - - // Determine optimal configuration - const optimizedConfig = this.calculateOptimalConfiguration() - - // Initialize partitioned index with semantic partitioning as default - if (this.config.enablePartitioning) { - this.partitionedIndex = new PartitionedHNSWIndex( - { - ...optimizedConfig.partitionConfig, - partitionStrategy: 'semantic', // Always use semantic for better performance - autoTuneSemanticClusters: true // Enable auto-tuning - }, - optimizedConfig.hnswConfig, - euclideanDistance - ) - console.log('✓ Partitioned index initialized with semantic clustering') - } - - // Initialize distributed search system - if (this.config.enableDistributedSearch && this.partitionedIndex) { - this.distributedSearch = new DistributedSearchSystem({ - maxConcurrentSearches: optimizedConfig.maxConcurrentSearches, - searchTimeout: this.config.targetSearchLatency * 5, - adaptivePartitionSelection: true, - loadBalancing: true - }) - console.log('✓ Distributed search system initialized') - } - - // Initialize batch S3 operations - if (this.config.s3Config) { - // Create S3 client from config - const { S3Client } = await import('@aws-sdk/client-s3') - const s3Client = new S3Client({ - region: this.config.s3Config.region || 'us-east-1', - endpoint: this.config.s3Config.endpoint, - credentials: this.config.s3Config.accessKeyId && this.config.s3Config.secretAccessKey ? { - accessKeyId: this.config.s3Config.accessKeyId, - secretAccessKey: this.config.s3Config.secretAccessKey - } : undefined // Will use default credentials from environment - }) - - this.batchOperations = new BatchS3Operations( - s3Client, - this.config.s3Config.bucketName, - { - maxConcurrency: 50, - useS3Select: this.config.expectedDatasetSize > 100000 - } - ) - console.log('✓ Batch S3 operations initialized') - } - - // Initialize enhanced caching - if (this.config.enablePredictiveCaching) { - this.cacheManager = new EnhancedCacheManager({ - hotCacheMaxSize: optimizedConfig.hotCacheSize, - warmCacheMaxSize: optimizedConfig.warmCacheSize, - prefetchEnabled: true, - prefetchStrategy: 'hybrid' as any, // Type casting for enum compatibility - prefetchBatchSize: 50 - }) - - if (this.batchOperations) { - this.cacheManager.setStorageAdapters(null as any, this.batchOperations) - } - console.log('✓ Enhanced cache manager initialized') - } - - // Initialize read-only optimizations - if (this.config.readOnlyMode && this.config.enableCompression) { - this.readOnlyOptimizations = new ReadOnlyOptimizations({ - compression: { - vectorCompression: 'quantization' as any, - metadataCompression: 'gzip' as any, - quantizationType: 'scalar' as any, - quantizationBits: 8 - }, - segmentSize: optimizedConfig.segmentSize, - memoryMapped: true, - cacheIndexInMemory: optimizedConfig.cacheIndexInMemory - }) - console.log('✓ Read-only optimizations initialized') - } - - console.log('Scaled HNSW System ready for', this.config.expectedDatasetSize, 'vectors') - } - - /** - * Calculate optimal configuration based on dataset size and constraints - */ - private calculateOptimalConfiguration(): { - partitionConfig: PartitionConfig - hnswConfig: OptimizedHNSWConfig - hotCacheSize: number - warmCacheSize: number - maxConcurrentSearches: number - segmentSize: number - cacheIndexInMemory: boolean - } { - const size = this.config.expectedDatasetSize - const memoryBudget = this.config.maxMemoryUsage - - let config: any = {} - - if (size <= 10000) { - // Small dataset - optimize for speed - config = { - partitionConfig: { - maxNodesPerPartition: 10000, - partitionStrategy: 'hash' as const - }, - hnswConfig: { - M: 16, - efConstruction: 200, - efSearch: 50, - targetSearchLatency: this.config.targetSearchLatency - }, - hotCacheSize: 1000, - warmCacheSize: 5000, - maxConcurrentSearches: 4, - segmentSize: 5000, - cacheIndexInMemory: true - } - } else if (size <= 100000) { - // Medium dataset - balance performance and memory - config = { - partitionConfig: { - maxNodesPerPartition: 25000, - partitionStrategy: 'semantic' as const, - semanticClusters: 8 - }, - hnswConfig: { - M: 24, - efConstruction: 300, - efSearch: 75, - targetSearchLatency: this.config.targetSearchLatency, - dynamicParameterTuning: true - }, - hotCacheSize: 2000, - warmCacheSize: 15000, - maxConcurrentSearches: 8, - segmentSize: 10000, - cacheIndexInMemory: memoryBudget > 2 * 1024 * 1024 * 1024 // 2GB - } - } else if (size <= 1000000) { - // Large dataset - optimize for scale - config = { - partitionConfig: { - maxNodesPerPartition: 50000, - partitionStrategy: 'semantic' as const, - semanticClusters: 16 - }, - hnswConfig: { - M: 32, - efConstruction: 400, - efSearch: 100, - targetSearchLatency: this.config.targetSearchLatency, - dynamicParameterTuning: true, - memoryBudget: memoryBudget - }, - hotCacheSize: 5000, - warmCacheSize: 25000, - maxConcurrentSearches: 12, - segmentSize: 20000, - cacheIndexInMemory: memoryBudget > 8 * 1024 * 1024 * 1024 // 8GB - } - } else { - // Very large dataset - maximum optimization - config = { - partitionConfig: { - maxNodesPerPartition: 100000, - partitionStrategy: 'hybrid' as const, - semanticClusters: 32 - }, - hnswConfig: { - M: 48, - efConstruction: 500, - efSearch: 150, - targetSearchLatency: this.config.targetSearchLatency, - dynamicParameterTuning: true, - memoryBudget: memoryBudget, - diskCacheEnabled: true - }, - hotCacheSize: 10000, - warmCacheSize: 50000, - maxConcurrentSearches: 20, - segmentSize: 50000, - cacheIndexInMemory: false // Too large for memory - } - } - - return config - } - - /** - * Add vector to the scaled system - */ - public async addVector(item: VectorDocument): Promise { - if (!this.partitionedIndex) { - throw new Error('System not properly initialized') - } - - const startTime = Date.now() - const result = await this.partitionedIndex.addItem(item) - - // Update performance metrics - this.performanceMetrics.indexSize = this.partitionedIndex.size() - - return result - } - - /** - * Bulk insert vectors with optimizations - */ - public async bulkInsert(items: VectorDocument[]): Promise { - if (!this.partitionedIndex) { - throw new Error('System not properly initialized') - } - - console.log(`Starting optimized bulk insert of ${items.length} vectors`) - const startTime = Date.now() - - // Sort items for optimal insertion order - const sortedItems = this.optimizeInsertionOrder(items) - - const results: string[] = [] - const batchSize = this.calculateOptimalBatchSize(items.length) - - // Process in batches - for (let i = 0; i < sortedItems.length; i += batchSize) { - const batch = sortedItems.slice(i, i + batchSize) - - for (const item of batch) { - const id = await this.partitionedIndex.addItem(item) - results.push(id) - } - - // Progress logging - if (i % (batchSize * 10) === 0) { - const progress = ((i / sortedItems.length) * 100).toFixed(1) - console.log(`Bulk insert progress: ${progress}%`) - } - } - - const totalTime = Date.now() - startTime - console.log(`Bulk insert completed: ${results.length} vectors in ${totalTime}ms`) - - return results - } - - /** - * High-performance vector search with all optimizations - */ - public async search( - queryVector: Vector, - k: number = 10, - options: { - strategy?: SearchStrategy - useCache?: boolean - maxPartitions?: number - } = {} - ): Promise> { - const startTime = Date.now() - - try { - let results: Array<[string, number]> - - if (this.distributedSearch && this.partitionedIndex) { - // Use distributed search for optimal performance - results = await this.distributedSearch.distributedSearch( - this.partitionedIndex, - queryVector, - k, - options.strategy || SearchStrategy.ADAPTIVE - ) - } else if (this.partitionedIndex) { - // Fall back to partitioned search - results = await this.partitionedIndex.search( - queryVector, - k, - { maxPartitions: options.maxPartitions } - ) - } else { - throw new Error('No search system available') - } - - // Update performance metrics and learn from performance - const searchTime = Date.now() - startTime - this.updateSearchMetrics(searchTime, results.length) - - // Adaptive learning - adjust configuration based on performance - if (this.config.learningEnabled && this.shouldTriggerLearning()) { - await this.adaptivelyLearnFromPerformance() - } - - return results - - } catch (error) { - console.error('Search failed:', error) - throw error - } - } - - /** - * Get system performance metrics - */ - public getPerformanceMetrics(): typeof this.performanceMetrics & { - partitionStats?: any - cacheStats?: any - compressionStats?: any - distributedSearchStats?: any - } { - const metrics = { ...this.performanceMetrics } - - // Add subsystem metrics - if (this.partitionedIndex) { - (metrics as any).partitionStats = this.partitionedIndex.getPartitionStats() - } - - if (this.cacheManager) { - (metrics as any).cacheStats = this.cacheManager.getStats() - } - - if (this.readOnlyOptimizations) { - (metrics as any).compressionStats = this.readOnlyOptimizations.getCompressionStats() - } - - if (this.distributedSearch) { - (metrics as any).distributedSearchStats = this.distributedSearch.getSearchStats() - } - - return metrics - } - - /** - * Optimize insertion order for better index quality - */ - private optimizeInsertionOrder(items: VectorDocument[]): VectorDocument[] { - if (items.length < 1000) { - return items // Not worth optimizing small batches - } - - // Simple clustering-based approach for better HNSW construction - // In production, you might use more sophisticated clustering - return items.sort(() => Math.random() - 0.5) - } - - /** - * Calculate optimal batch size based on system resources - */ - private calculateOptimalBatchSize(totalItems: number): number { - const memoryBudget = this.config.maxMemoryUsage - const estimatedItemSize = 1000 // Rough estimate per item in bytes - - const maxBatch = Math.floor(memoryBudget * 0.1 / estimatedItemSize) - const targetBatch = Math.min(1000, Math.max(100, maxBatch)) - - return Math.min(targetBatch, totalItems) - } - - /** - * Update search performance metrics - */ - private updateSearchMetrics(searchTime: number, resultCount: number): void { - this.performanceMetrics.totalSearches++ - this.performanceMetrics.averageSearchTime = - (this.performanceMetrics.averageSearchTime + searchTime) / 2 - - // Update other metrics - if (this.cacheManager) { - const cacheStats = this.cacheManager.getStats() - const totalOps = cacheStats.hotCacheHits + cacheStats.hotCacheMisses + - cacheStats.warmCacheHits + cacheStats.warmCacheMisses - - this.performanceMetrics.cacheHitRate = totalOps > 0 ? - (cacheStats.hotCacheHits + cacheStats.warmCacheHits) / totalOps : 0 - } - - if (this.readOnlyOptimizations) { - const compressionStats = this.readOnlyOptimizations.getCompressionStats() - this.performanceMetrics.compressionRatio = compressionStats.compressionRatio - } - - // Estimate memory usage - this.performanceMetrics.memoryUsage = this.estimateMemoryUsage() - } - - /** - * Estimate current memory usage - */ - private estimateMemoryUsage(): number { - let totalMemory = 0 - - if (this.partitionedIndex) { - // Rough estimate: 1KB per vector - totalMemory += this.partitionedIndex.size() * 1024 - } - - if (this.cacheManager) { - const cacheStats = this.cacheManager.getStats() - totalMemory += (cacheStats.hotCacheSize + cacheStats.warmCacheSize) * 1024 - } - - return totalMemory - } - - /** - * Generate performance report - */ - public generatePerformanceReport(): string { - const metrics = this.getPerformanceMetrics() - - return ` -=== Scaled HNSW System Performance Report === - -Dataset Configuration: -- Expected Size: ${this.config.expectedDatasetSize.toLocaleString()} vectors -- Current Size: ${metrics.indexSize.toLocaleString()} vectors -- Memory Budget: ${(this.config.maxMemoryUsage / 1024 / 1024 / 1024).toFixed(1)}GB -- Target Latency: ${this.config.targetSearchLatency}ms - -Performance Metrics: -- Total Searches: ${metrics.totalSearches.toLocaleString()} -- Average Search Time: ${metrics.averageSearchTime.toFixed(1)}ms -- Cache Hit Rate: ${(metrics.cacheHitRate * 100).toFixed(1)}% -- Memory Usage: ${(metrics.memoryUsage / 1024 / 1024).toFixed(1)}MB -- Compression Ratio: ${metrics.compressionRatio ? (metrics.compressionRatio * 100).toFixed(1) + '%' : 'N/A'} - -System Status: ${this.getSystemStatus()} - `.trim() - } - - /** - * Get overall system status - */ - private getSystemStatus(): string { - const metrics = this.getPerformanceMetrics() - - if (metrics.averageSearchTime <= this.config.targetSearchLatency) { - return '✅ OPTIMAL' - } else if (metrics.averageSearchTime <= this.config.targetSearchLatency * 2) { - return '⚠️ ACCEPTABLE' - } else { - return '❌ NEEDS OPTIMIZATION' - } - } - - /** - * Check if adaptive learning should be triggered - */ - private shouldTriggerLearning(): boolean { - const timeSinceLastLearning = Date.now() - this.performanceMetrics.lastLearningUpdate - const minLearningInterval = 30000 // 30 seconds - const minSearches = 20 // Minimum searches before learning - - return timeSinceLastLearning > minLearningInterval && - this.performanceMetrics.totalSearches > minSearches && - this.performanceMetrics.totalSearches % 50 === 0 // Learn every 50 searches - } - - /** - * Adaptively learn from performance and adjust configuration - */ - private async adaptivelyLearnFromPerformance(): Promise { - try { - const currentMetrics = { - averageSearchTime: this.performanceMetrics.averageSearchTime, - memoryUsage: this.performanceMetrics.memoryUsage, - cacheHitRate: this.performanceMetrics.cacheHitRate, - errorRate: 0 // Could be tracked separately - } - - const adjustments = await this.autoConfig.learnFromPerformance(currentMetrics) - - if (Object.keys(adjustments).length > 0) { - console.log('🧠 Adaptive learning: Adjusting configuration based on performance') - - // Apply learned adjustments - let configChanged = false - - if (adjustments.enableDistributedSearch !== undefined && - adjustments.enableDistributedSearch !== this.config.enableDistributedSearch) { - this.config.enableDistributedSearch = adjustments.enableDistributedSearch - configChanged = true - } - - if (adjustments.enableCompression !== undefined && - adjustments.enableCompression !== this.config.enableCompression) { - this.config.enableCompression = adjustments.enableCompression - configChanged = true - } - - if (adjustments.enablePredictiveCaching !== undefined && - adjustments.enablePredictiveCaching !== this.config.enablePredictiveCaching) { - this.config.enablePredictiveCaching = adjustments.enablePredictiveCaching - configChanged = true - } - - // Apply partition adjustments - if (adjustments.maxNodesPerPartition && - this.partitionedIndex && - adjustments.maxNodesPerPartition !== this.partitionedIndex.getPartitionStats().averageNodesPerPartition) { - // This would require rebuilding the index in a real implementation - console.log(`Learning suggests partition size: ${adjustments.maxNodesPerPartition}`) - } - - if (configChanged) { - console.log('✅ Configuration updated based on performance learning') - } - } - - this.performanceMetrics.lastLearningUpdate = Date.now() - - } catch (error) { - console.warn('Adaptive learning failed:', error) - } - } - - /** - * Update dataset analysis for better auto-configuration - */ - public async updateDatasetAnalysis(vectorCount: number, vectorDimension?: number): Promise { - if (this.config.autoConfigureEnvironment) { - const analysis = { - estimatedSize: vectorCount, - vectorDimension, - accessPatterns: this.inferAccessPatterns() - } - - await this.autoConfig.adaptToDataset(analysis) - console.log(`📊 Dataset analysis updated: ${vectorCount} vectors${vectorDimension ? `, ${vectorDimension}D` : ''}`) - } - } - - /** - * Infer access patterns from current metrics - */ - private inferAccessPatterns(): 'read-heavy' | 'write-heavy' | 'balanced' { - // Simple heuristic - in practice, this would track read/write ratios - if (this.performanceMetrics.totalSearches > 100) { - return 'read-heavy' - } - return 'balanced' - } - - /** - * Cleanup system resources - */ - public cleanup(): void { - this.distributedSearch?.cleanup() - this.cacheManager?.clear() - this.readOnlyOptimizations?.cleanup() - this.partitionedIndex?.clear() - this.autoConfig.resetCache() - - console.log('Scaled HNSW System cleaned up') - } -} - -// Export convenience factory functions - -/** - * Create a fully auto-configured Brainy system - minimal setup required! - * Just provide S3 config if you want persistence beyond the current session - */ -export function createAutoBrainy(s3Config?: { - bucketName: string - region?: string - accessKeyId?: string - secretAccessKey?: string -}): ScaledHNSWSystem { - return new ScaledHNSWSystem({ - s3Config: s3Config ? { - bucketName: s3Config.bucketName, - region: s3Config.region || 'us-east-1', - accessKeyId: s3Config.accessKeyId, - secretAccessKey: s3Config.secretAccessKey - } : undefined, - autoConfigureEnvironment: true, - learningEnabled: true - }) -} - -/** - * Create a Brainy system optimized for specific scenarios - */ -export async function createQuickBrainy( - scenario: 'small' | 'medium' | 'large' | 'enterprise', - s3Config?: { bucketName: string; region?: string } -): Promise { - const { getQuickSetup } = await import('../utils/autoConfiguration.js') - const quickConfig = await getQuickSetup(scenario) - - return new ScaledHNSWSystem({ - ...quickConfig, - s3Config: s3Config && quickConfig.s3Required ? { - bucketName: s3Config.bucketName, - region: s3Config.region || 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY - } : undefined, - autoConfigureEnvironment: true, - learningEnabled: true - }) -} - -/** - * Legacy factory function - still works but consider using createAutoBrainy() instead - */ -export function createScaledHNSWSystem(config: ScaledHNSWConfig = {}): ScaledHNSWSystem { - return new ScaledHNSWSystem(config) -} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 7a046bbb..7117fc48 100644 --- a/src/index.ts +++ b/src/index.ts @@ -346,14 +346,10 @@ import type { StorageAdapter } from './coreTypes.js' -// Export HNSW index and optimized version +// Export HNSW index import { HNSWIndex } from './hnsw/hnswIndex.js' -import { - HNSWIndexOptimized, - HNSWOptimizedConfig -} from './hnsw/hnswIndexOptimized.js' -export { HNSWIndex, HNSWIndexOptimized } +export { HNSWIndex } export type { Vector, @@ -365,7 +361,6 @@ export type { HNSWNoun, HNSWVerb, HNSWConfig, - HNSWOptimizedConfig, StorageAdapter } diff --git a/src/triple/TripleIntelligenceSystem.ts b/src/triple/TripleIntelligenceSystem.ts index fe6db8e6..3bb0d4f4 100644 --- a/src/triple/TripleIntelligenceSystem.ts +++ b/src/triple/TripleIntelligenceSystem.ts @@ -14,7 +14,6 @@ */ import { HNSWIndex } from '../hnsw/hnswIndex.js' -import { HNSWIndexOptimized } from '../hnsw/hnswIndexOptimized.js' import { TypeAwareHNSWIndex } from '../hnsw/typeAwareHNSWIndex.js' import { MetadataIndexManager } from '../utils/metadataIndex.js' import { Vector } from '../coreTypes.js' @@ -233,7 +232,7 @@ class QueryPlanner { */ export class TripleIntelligenceSystem { private metadataIndex: MetadataIndexManager - private hnswIndex: HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex + private hnswIndex: HNSWIndex | TypeAwareHNSWIndex private graphIndex: GraphAdjacencyIndex private metrics: PerformanceMetrics private planner: QueryPlanner @@ -242,7 +241,7 @@ export class TripleIntelligenceSystem { constructor( metadataIndex: MetadataIndexManager, - hnswIndex: HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex, + hnswIndex: HNSWIndex | TypeAwareHNSWIndex, graphIndex: GraphAdjacencyIndex, embedder: (text: string) => Promise, storage: any