refactor: remove 3,700+ LOC of unused HNSW implementations

Delete dead code island that was never instantiated in production: - OptimizedHNSWIndex (430 LOC) - PartitionedHNSWIndex (412 LOC) - DistributedSearchSystem (635 LOC) - ScaledHNSWSystem (744 LOC) - HNSWIndexOptimized (585 LOC) - brainy-backup.ts stale example (903 LOC) Also upgrades entry point recovery from O(n) to O(1) using existing highLevelNodes index structure. Production uses only: HNSWIndex (memory) and TypeAwareHNSWIndex (persistent)
2025-11-25 15:36:49 -08:00 · 2025-11-25 15:36:49 -08:00 · e3146ce11d
commit e3146ce11d
parent 52eae67cb8
10 changed files with 47 additions and 3774 deletions
--- a/examples/brainy-backup.ts
+++ b/examples/brainy-backup.ts
@ -1,903 +0,0 @@
 /**
 * 🧠 Brainy 3.0 - The Future of Neural Databases
 *
 * Beautiful, Professional, Planet-Scale, Fun to Use
 * NO STUBS, NO MOCKS, REAL IMPLEMENTATION
 */
 import { v4 as uuidv4 } from './universal/uuid.js'
 import { HNSWIndex } from './hnsw/hnswIndex.js'
 import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js'
 import { createStorage } from './storage/storageFactory.js'
 import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb } from './coreTypes.js'
 import {
  defaultEmbeddingFunction,
  cosineDistance
 } from './utils/index.js'
 import { validateNounType, validateVerbType } from './utils/typeValidation.js'
 import { matchesMetadataFilter } from './utils/metadataFilter.js'
 import { AugmentationRegistry, AugmentationContext } from './augmentations/brainyAugmentation.js'
 import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js'
 import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js'
 import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
 import {
  Entity,
  Relation,
  Result,
  AddParams,
  UpdateParams,
  RelateParams,
  FindParams,
  SimilarParams,
  GetRelationsParams,
  AddManyParams,
  DeleteManyParams,
  BatchResult,
  BrainyConfig
 } from './types/brainy.types.js'
 import { NounType, VerbType } from './types/graphTypes.js'
 /**
 * The main Brainy class - Clean, Beautiful, Powerful
 * REAL IMPLEMENTATION - No stubs, no mocks
 */
 export class Brainy<T = any> {
  // Core components
  private index!: HNSWIndex | HNSWIndexOptimized
  private storage!: StorageAdapter
  private embedder: EmbeddingFunction
  private distance: DistanceFunction
  private augmentationRegistry: AugmentationRegistry
  private config: Required<BrainyConfig>
  // Sub-APIs (lazy-loaded)
  private _neural?: ImprovedNeuralAPI
  private _nlp?: NaturalLanguageProcessor
  // State
  private initialized = false
  private dimensions?: number
  constructor(config?: BrainyConfig) {
    // Normalize configuration with defaults
    this.config = this.normalizeConfig(config)
    // Setup core components
    this.distance = cosineDistance
    this.embedder = this.setupEmbedder()
    this.augmentationRegistry = this.setupAugmentations()
    // Index and storage are initialized in init() because they may need each other
  }
  /**
   * Initialize Brainy - MUST be called before use
   */
  async init(): Promise<void> {
    if (this.initialized) {
      return
    }
    try {
      // Setup and initialize storage
      this.storage = await this.setupStorage()
      await this.storage.init()
      // Setup index now that we have storage
      this.index = this.setupIndex()
      // Initialize augmentations
      await this.augmentationRegistry.initializeAll({
        brain: this,
        storage: this.storage,
        config: this.config,
        log: (message: string, level = 'info') => {
          // Simple logging for now
          if (level === 'error') {
            console.error(message)
          } else if (level === 'warn') {
            console.warn(message)
          } else {
            console.log(message)
          }
        }
      })
      // Warm up if configured
      if (this.config.warmup) {
        await this.warmup()
      }
      this.initialized = true
    } catch (error) {
      throw new Error(`Failed to initialize Brainy: ${error}`)
    }
  }
  /**
   * Ensure Brainy is initialized
   */
  private async ensureInitialized(): Promise<void> {
    if (!this.initialized) {
      throw new Error('Brainy not initialized. Call init() first.')
    }
  }
  // ============= CORE CRUD OPERATIONS =============
  /**
   * Add an entity to the database
   */
  async add(params: AddParams<T>): Promise<string> {
    await this.ensureInitialized()
    // Validate parameters
    if (!params.data && !params.vector) {
      throw new Error('Either data or vector is required')
    }
    validateNounType(params.type)
    // Generate ID if not provided
    const id = params.id || uuidv4()
    // Get or compute vector
    const vector = params.vector || (await this.embed(params.data))
    // Ensure dimensions are set
    if (!this.dimensions) {
      this.dimensions = vector.length
    } else if (vector.length !== this.dimensions) {
      throw new Error(
        `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}`
      )
    }
    // Execute through augmentation pipeline
    return this.augmentationRegistry.execute('add', params, async () => {
      // Add to index
      await this.index.addItem({ id, vector })
      // Save to storage
      await this.storage.saveNoun({
        id,
        vector,
        connections: new Map(),
        level: 0,
        metadata: {
          ...params.metadata,
          noun: params.type,
          service: params.service,
          createdAt: Date.now()
        }
      })
      return id
    })
  }
  /**
   * Get an entity by ID
   */
  async get(id: string): Promise<Entity<T> | null> {
    await this.ensureInitialized()
    return this.augmentationRegistry.execute('get', { id }, async () => {
      // Get from storage
      const noun = await this.storage.getNoun(id)
      if (!noun) {
        return null
      }
      // Extract metadata - separate user metadata from system metadata
      const { noun: nounType, service, createdAt, updatedAt, ...userMetadata } = noun.metadata || {}
      // Construct entity from noun
      const entity: Entity<T> = {
        id: noun.id,
        vector: noun.vector,
        type: (nounType as NounType) || NounType.Thing,
        metadata: userMetadata as T,
        service: service as string,
        createdAt: (createdAt as number) || Date.now(),
        updatedAt: updatedAt as number
      }
      return entity
    })
  }
  /**
   * Update an entity
   */
  async update(params: UpdateParams<T>): Promise<void> {
    await this.ensureInitialized()
    if (!params.id) {
      throw new Error('ID is required for update')
    }
    return this.augmentationRegistry.execute('update', params, async () => {
      // Get existing entity
      const existing = await this.get(params.id)
      if (!existing) {
        throw new Error(`Entity ${params.id} not found`)
      }
      // Update vector if data changed
      let vector = existing.vector
      if (params.data) {
        vector = params.vector || (await this.embed(params.data))
        // Update in index (remove and re-add since no update method)
        await this.index.removeItem(params.id)
        await this.index.addItem({ id: params.id, vector })
      }
      // Always update the noun with new metadata
      const newMetadata = params.merge !== false
        ? { ...existing.metadata, ...params.metadata }
        : params.metadata || existing.metadata
      await this.storage.saveNoun({
        id: params.id,
        vector,
        connections: new Map(),
        level: 0,
        metadata: {
          ...newMetadata,
          noun: params.type || existing.type,
          service: existing.service,
          createdAt: existing.createdAt,
          updatedAt: Date.now()
        }
      })
    })
  }
  /**
   * Delete an entity
   */
  async delete(id: string): Promise<void> {
    await this.ensureInitialized()
    return this.augmentationRegistry.execute('delete', { id }, async () => {
      // Remove from index
      await this.index.removeItem(id)
      // Delete from storage
      await this.storage.deleteNoun(id)
      // Delete metadata (if it exists as separate)
      try {
        await this.storage.saveMetadata(id, null as any) // Clear metadata
      } catch {
        // Ignore if not supported
      }
      // Delete related verbs
      const verbs = await this.storage.getVerbsBySource(id)
      const targetVerbs = await this.storage.getVerbsByTarget(id)
      const allVerbs = [...verbs, ...targetVerbs]
      for (const verb of allVerbs) {
        await this.storage.deleteVerb(verb.id)
      }
    })
  }
  // ============= RELATIONSHIP OPERATIONS =============
  /**
   * Create a relationship between entities
   */
  async relate(params: RelateParams<T>): Promise<string> {
    await this.ensureInitialized()
    // Validate parameters
    if (!params.from || !params.to) {
      throw new Error('Both from and to are required')
    }
    validateVerbType(params.type)
    // Verify entities exist
    const fromEntity = await this.get(params.from)
    const toEntity = await this.get(params.to)
    if (!fromEntity) {
      throw new Error(`Source entity ${params.from} not found`)
    }
    if (!toEntity) {
      throw new Error(`Target entity ${params.to} not found`)
    }
    // Generate ID
    const id = uuidv4()
    // Compute relationship vector (average of entities)
    const relationVector = fromEntity.vector.map(
      (v, i) => (v + toEntity.vector[i]) / 2
    )
    return this.augmentationRegistry.execute('relate', params, async () => {
      // Save to storage
      const verb: GraphVerb = {
        id,
        vector: relationVector,
        sourceId: params.from,
        targetId: params.to,
        source: fromEntity.type,
        target: toEntity.type,
        verb: params.type,
        type: params.type,
        weight: params.weight ?? 1.0,
        metadata: params.metadata as any,
        createdAt: Date.now()
      } as any
      await this.storage.saveVerb(verb)
      // Create bidirectional if requested
      if (params.bidirectional) {
        const reverseId = uuidv4()
        const reverseVerb: GraphVerb = {
          ...verb,
          id: reverseId,
          sourceId: params.to,
          targetId: params.from,
          source: toEntity.type,
          target: fromEntity.type
        } as any
        await this.storage.saveVerb(reverseVerb)
      }
      return id
    })
  }
  /**
   * Delete a relationship
   */
  async unrelate(id: string): Promise<void> {
    await this.ensureInitialized()
    return this.augmentationRegistry.execute('unrelate', { id }, async () => {
      await this.storage.deleteVerb(id)
    })
  }
  /**
   * Get relationships
   */
  async getRelations(
    params: GetRelationsParams = {}
  ): Promise<Relation<T>[]> {
    await this.ensureInitialized()
    const relations: Relation<T>[] = []
    if (params.from) {
      const verbs = await this.storage.getVerbsBySource(params.from)
      relations.push(...this.verbsToRelations(verbs))
    }
    if (params.to) {
      const verbs = await this.storage.getVerbsByTarget(params.to)
      relations.push(...this.verbsToRelations(verbs))
    }
    // Filter by type
    let filtered = relations
    if (params.type) {
      const types = Array.isArray(params.type) ? params.type : [params.type]
      filtered = relations.filter((r) => types.includes(r.type))
    }
    // Filter by service
    if (params.service) {
      filtered = filtered.filter((r) => r.service === params.service)
    }
    // Apply pagination
    const limit = params.limit || 100
    const offset = params.offset || 0
    return filtered.slice(offset, offset + limit)
  }
  // ============= SEARCH & DISCOVERY =============
  /**
   * Unified find method - supports natural language and structured queries
   */
  async find(query: string | FindParams<T>): Promise<Result<T>[]> {
    await this.ensureInitialized()
    // Parse natural language queries
    const params: FindParams<T> =
      typeof query === 'string' ? await this.parseNaturalQuery(query) : query
    return this.augmentationRegistry.execute('find', params, async () => {
      let results: Result<T>[] = []
      // Vector search
      if (params.query || params.vector) {
        const vector = params.vector || (await this.embed(params.query!))
        const limit = params.limit || 10
        // Search index - returns array of [id, score] tuples
        const searchResults = await this.index.search(vector, limit * 2) // Get extra for filtering
        // Hydrate results
        for (const [id, score] of searchResults) {
          const entity = await this.get(id)
          if (entity) {
            results.push({
              id,
              score,
              entity
            })
          }
        }
      }
      // Proximity search
      if (params.near) {
        const nearEntity = await this.get(params.near.id)
        if (nearEntity) {
          const nearResults = await this.index.search(
            nearEntity.vector,
            params.limit || 10
          )
          for (const [id, score] of nearResults) {
            if (score >= (params.near.threshold || 0.7)) {
              const entity = await this.get(id)
              if (entity) {
                results.push({
                  id,
                  score,
                  entity
                })
              }
            }
          }
        }
      }
      // Apply filters
      if (params.where) {
        results = results.filter((r) =>
          matchesMetadataFilter(r.entity.metadata, params.where!)
        )
      }
      if (params.type) {
        const types = Array.isArray(params.type) ? params.type : [params.type]
        results = results.filter((r) => types.includes(r.entity.type))
      }
      if (params.service) {
        results = results.filter((r) => r.entity.service === params.service)
      }
      // Graph constraints
      if (params.connected) {
        results = await this.applyGraphConstraints(results, params.connected)
      }
      // Sort by score and limit
      results.sort((a, b) => b.score - a.score)
      const limit = params.limit || 10
      const offset = params.offset || 0
      return results.slice(offset, offset + limit)
    })
  }
  /**
   * Find similar entities
   */
  async similar(params: SimilarParams<T>): Promise<Result<T>[]> {
    await this.ensureInitialized()
    // Get target vector
    let targetVector: Vector
    if (typeof params.to === 'string') {
      const entity = await this.get(params.to)
      if (!entity) {
        throw new Error(`Entity ${params.to} not found`)
      }
      targetVector = entity.vector
    } else if (Array.isArray(params.to)) {
      targetVector = params.to as Vector
    } else {
      targetVector = (params.to as Entity<T>).vector
    }
    // Use find with vector
    return this.find({
      vector: targetVector,
      limit: params.limit,
      type: params.type,
      where: params.where,
      service: params.service
    })
  }
  // ============= BATCH OPERATIONS =============
  /**
   * Add multiple entities
   */
  async addMany(params: AddManyParams<T>): Promise<BatchResult<string>> {
    await this.ensureInitialized()
    const result: BatchResult<string> = {
      successful: [],
      failed: [],
      total: params.items.length,
      duration: 0
    }
    const startTime = Date.now()
    const chunkSize = params.chunkSize || 100
    // Process in chunks
    for (let i = 0; i < params.items.length; i += chunkSize) {
      const chunk = params.items.slice(i, i + chunkSize)
      const promises = chunk.map(async (item) => {
        try {
          const id = await this.add(item)
          result.successful.push(id)
        } catch (error) {
          result.failed.push({
            item,
            error: (error as Error).message
          })
          if (!params.continueOnError) {
            throw error
          }
        }
      })
      if (params.parallel !== false) {
        await Promise.allSettled(promises)
      } else {
        for (const promise of promises) {
          await promise
        }
      }
      // Report progress
      if (params.onProgress) {
        params.onProgress(
          result.successful.length + result.failed.length,
          result.total
        )
      }
    }
    result.duration = Date.now() - startTime
    return result
  }
  /**
   * Delete multiple entities
   */
  async deleteMany(params: DeleteManyParams): Promise<BatchResult<string>> {
    await this.ensureInitialized()
    // Determine what to delete
    let idsToDelete: string[] = []
    if (params.ids) {
      idsToDelete = params.ids
    } else if (params.type || params.where) {
      // Find entities to delete
      const entities = await this.find({
        type: params.type,
        where: params.where,
        limit: params.limit || 1000
      })
      idsToDelete = entities.map((e) => e.id)
    }
    const result: BatchResult<string> = {
      successful: [],
      failed: [],
      total: idsToDelete.length,
      duration: 0
    }
    const startTime = Date.now()
    for (const id of idsToDelete) {
      try {
        await this.delete(id)
        result.successful.push(id)
      } catch (error) {
        result.failed.push({
          item: id,
          error: (error as Error).message
        })
      }
      if (params.onProgress) {
        params.onProgress(
          result.successful.length + result.failed.length,
          result.total
        )
      }
    }
    result.duration = Date.now() - startTime
    return result
  }
  // ============= SUB-APIS =============
  /**
   * Neural API - Advanced AI operations
   */
  neural() {
    if (!this._neural) {
      this._neural = new ImprovedNeuralAPI(this as any)
    }
    return this._neural
  }
    return this._neural
  }
  /**
   * Neural API - Advanced AI operations
   */
  neural() {
    if (!this._neural) {
      this._neural = new ImprovedNeuralAPI(this as any)
    }
    return this._neural
  }
  /**
   * Natural Language Processing API
   */
  nlp() {
    if (!this._nlp) {
      this._nlp = new NaturalLanguageProcessor(this as any)
    }
    return this._nlp
  }
  /**
   * Create a streaming pipeline
   */
  stream() {
    const { Pipeline } = require('./streaming/pipeline.js')
    return new Pipeline(this)
  }
  /**
   * Get insights about the data
   */
  async insights(): Promise<{
    entities: number
    relationships: number
    types: Record<string, number>
    services: string[]
    density: number
  }> {
    await this.ensureInitialized()
    // Get all entities count
    const allEntities = await this.storage.getAllNouns()
    const entities = allEntities.length
    // Get relationships count
    const allVerbs = await this.storage.getAllVerbs()
    const relationships = allVerbs.length
    // Count by type
    const types: Record<string, number> = {}
    for (const entity of allEntities) {
      const type = entity.metadata?.noun || 'unknown'
      types[type] = (types[type] || 0) + 1
    }
    // Get unique services
    const services = [...new Set(allEntities.map(e => e.metadata?.service).filter(Boolean))]
    // Calculate density (relationships per entity)
    const density = entities > 0 ? relationships / entities : 0
    return {
      entities,
      relationships,
      types,
      services,
      density
    }
  }
  /**
   * Augmentations API - Clean and simple
   */
  get augmentations() {
    return {
      list: () => this.augmentationRegistry.getAll().map(a => a.name),
      get: (name: string) => this.augmentationRegistry.getAll().find(a => a.name === name),
      has: (name: string) => this.augmentationRegistry.getAll().some(a => a.name === name)
    }
  }
  // ============= HELPER METHODS =============
  /**
   * Parse natural language query
   */
  private async parseNaturalQuery(query: string): Promise<FindParams<T>> {
    if (!this._nlp) {
      this._nlp = new NaturalLanguageProcessor(this as any)
    }
    const parsed = await this._nlp.processNaturalQuery(query)
    return parsed as FindParams<T>
  }
  /**
   * Apply graph constraints to results
   */
  private async applyGraphConstraints(
    results: Result<T>[],
    constraints: any
  ): Promise<Result<T>[]> {
    // Filter by graph connections
    if (constraints.to || constraints.from) {
      const filtered: Result<T>[] = []
      for (const result of results) {
        if (constraints.to) {
          const verbs = await this.storage.getVerbsBySource(result.id)
          const hasConnection = verbs.some(v => v.targetId === constraints.to)
          if (hasConnection) {
            filtered.push(result)
          }
        }
        if (constraints.from) {
          const verbs = await this.storage.getVerbsByTarget(result.id)
          const hasConnection = verbs.some(v => v.sourceId === constraints.from)
          if (hasConnection) {
            filtered.push(result)
          }
        }
      }
      return filtered
    }
    return results
  }
  /**
   * Convert verbs to relations
   */
  private verbsToRelations(verbs: GraphVerb[]): Relation<T>[] {
    return verbs.map((v) => ({
      id: v.id,
      from: v.sourceId,
      to: v.targetId,
      type: (v.verb || v.type) as VerbType,
      weight: v.weight,
      metadata: v.metadata,
      service: v.metadata?.service as string,
      createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now()
    }))
  }
  /**
   * Embed data into vector
   */
  private async embed(data: any): Promise<Vector> {
    return this.embedder(data)
  }
  /**
   * Warm up the system
   */
  private async warmup(): Promise<void> {
    // Warm up embedder
    await this.embed('warmup')
  }
  /**
   * Setup embedder
   */
  private setupEmbedder(): EmbeddingFunction {
    if (this.config.model?.type === 'custom' && this.config.model.name) {
      // TODO: Load custom model
      return defaultEmbeddingFunction
    }
    return defaultEmbeddingFunction
  }
  /**
   * Setup storage
   */
  private async setupStorage(): Promise<StorageAdapter> {
    const storage = await createStorage({
      type: this.config.storage?.type || 'memory',
      ...this.config.storage?.options
    })
    return storage
  }
  /**
   * Setup index
   */
  private setupIndex(): HNSWIndex | HNSWIndexOptimized {
    const indexConfig = {
      ...this.config.index,
      distanceFunction: this.distance
    }
    // Use optimized index for larger datasets
    if (this.config.storage?.type !== 'memory') {
      return new HNSWIndexOptimized(indexConfig, this.distance, this.storage)
    }
    return new HNSWIndex(indexConfig as any)
  }
  /**
   * Setup augmentations
   */
  private setupAugmentations(): AugmentationRegistry {
    const registry = new AugmentationRegistry()
    // Register default augmentations
    const defaults = createDefaultAugmentations(this.config.augmentations)
    for (const aug of defaults) {
      registry.register(aug)
    }
    return registry
  }
  /**
   * Normalize configuration
   */
  private normalizeConfig(config?: BrainyConfig): Required<BrainyConfig> {
    return {
      storage: config?.storage || { type: 'memory' },
      model: config?.model || { type: 'fast' },
      index: config?.index || {},
      cache: config?.cache ?? true,
      augmentations: config?.augmentations || {},
      warmup: config?.warmup ?? false,
      realtime: config?.realtime ?? false,
      multiTenancy: config?.multiTenancy ?? false,
      telemetry: config?.telemetry ?? false
    }
  }
  /**
   * Close and cleanup
   */
  async close(): Promise<void> {
    // Shutdown augmentations
    const augs = this.augmentationRegistry.getAll()
    for (const aug of augs) {
      if ('shutdown' in aug && typeof aug.shutdown === 'function') {
        await aug.shutdown()
      }
    }
    // Storage doesn't have close in current interface
    // We'll just mark as not initialized
    this.initialized = false
  }
 }
 // Re-export types for convenience
 export * from './types/brainy.types.js'
 export { NounType, VerbType } from './types/graphTypes.js'
--- a/src/brainy.ts
+++ b/src/brainy.ts
@ -7,7 +7,6 @@
 import { v4 as uuidv4 } from './universal/uuid.js'
 import { HNSWIndex } from './hnsw/hnswIndex.js'
 import { HNSWIndexOptimized } from './hnsw/hnswIndexOptimized.js'
 import { TypeAwareHNSWIndex } from './hnsw/typeAwareHNSWIndex.js'
 import { createStorage } from './storage/storageFactory.js'
 import { BaseStorage } from './storage/baseStorage.js'
@ -90,7 +89,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
  private static instances: Brainy[] = []
  // Core components
-  private index!: HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex
+  private index!: HNSWIndex | TypeAwareHNSWIndex
  private storage!: BaseStorage
  private metadataIndex!: MetadataIndexManager
  private graphIndex!: GraphAdjacencyIndex
@ -1010,7 +1009,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
                metadata.noun as any
              )
            )
-          } else if (this.index instanceof HNSWIndex || this.index instanceof HNSWIndexOptimized) {
+          } else if (this.index instanceof HNSWIndex) {
            tx.addOperation(
              new RemoveFromHNSWOperation(this.index, id, noun.vector)
            )
@ -2253,7 +2252,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
                      metadata.noun as any
                    )
                  )
-                } else if (this.index instanceof HNSWIndex || this.index instanceof HNSWIndexOptimized) {
+                } else if (this.index instanceof HNSWIndex) {
                  tx.addOperation(
                    new RemoveFromHNSWOperation(this.index, id, noun.vector)
                  )
@ -4770,7 +4769,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   * - 10x faster type-specific queries
   * - Automatic type routing
   */
-  private setupIndex(): HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex {
+  private setupIndex(): HNSWIndex | TypeAwareHNSWIndex {
    const indexConfig = {
      ...this.config.index,
      distanceFunction: this.distance
--- a/src/hnsw/distributedSearch.ts
+++ b/src/hnsw/distributedSearch.ts
@ -1,636 +0,0 @@
 /**
 * Distributed Search System for Large-Scale HNSW Indices
 * Implements parallel search across multiple partitions and instances
 */
 import { Vector, HNSWNoun } from '../coreTypes.js'
 import { PartitionedHNSWIndex } from './partitionedHNSWIndex.js'
 import { executeInThread } from '../utils/workerUtils.js'
 // Search task for parallel execution
 interface SearchTask {
  partitionId: string
  queryVector: Vector
  k: number
  searchId: string
  priority: number
 }
 // Search result from a partition
 interface PartitionSearchResult {
  partitionId: string
  results: Array<[string, number]>
  searchTime: number
  nodesVisited: number
  error?: Error
 }
 // Distributed search configuration
 interface DistributedSearchConfig {
  maxConcurrentSearches?: number
  searchTimeout?: number
  resultMergeStrategy?: 'distance' | 'score' | 'hybrid'
  adaptivePartitionSelection?: boolean
  redundantSearches?: number
  loadBalancing?: boolean
 }
 // Search coordination strategies
 export enum SearchStrategy {
  BROADCAST = 'broadcast', // Search all partitions
  SELECTIVE = 'selective', // Search subset of partitions
  ADAPTIVE = 'adaptive',   // Dynamically adjust based on results
  HIERARCHICAL = 'hierarchical' // Multi-level search
 }
 // Worker thread pool for parallel search
 interface SearchWorker {
  id: string
  busy: boolean
  tasksCompleted: number
  averageTaskTime: number
  lastTaskTime: number
 }
 /**
 * Distributed search coordinator for large-scale vector search
 */
 export class DistributedSearchSystem {
  private config: Required<DistributedSearchConfig>
  private searchWorkers: Map<string, SearchWorker> = new Map()
  private searchQueue: SearchTask[] = []
  private activeSearches: Map<string, Promise<PartitionSearchResult[]>> = new Map()
  private partitionStats: Map<string, {
    averageSearchTime: number
    load: number
    quality: number
    lastUsed: number
  }> = new Map()
  // Performance monitoring
  private searchStats = {
    totalSearches: 0,
    averageLatency: 0,
    parallelEfficiency: 0,
    cacheHitRate: 0,
    partitionUtilization: new Map<string, number>()
  }
  constructor(config: Partial<DistributedSearchConfig> = {}) {
    this.config = {
      maxConcurrentSearches: 10,
      searchTimeout: 30000, // 30 seconds
      resultMergeStrategy: 'hybrid',
      adaptivePartitionSelection: true,
      redundantSearches: 0,
      loadBalancing: true,
      ...config
    }
    this.initializeWorkerPool()
  }
  /**
   * Execute distributed search across multiple partitions
   */
  public async distributedSearch(
    partitionedIndex: PartitionedHNSWIndex,
    queryVector: Vector,
    k: number,
    strategy: SearchStrategy = SearchStrategy.ADAPTIVE
  ): Promise<Array<[string, number]>> {
    const searchId = this.generateSearchId()
    const startTime = Date.now()
    try {
      // Select partitions to search based on strategy
      const partitionsToSearch = await this.selectPartitions(
        partitionedIndex,
        queryVector,
        strategy
      )
      // Create search tasks
      const searchTasks = this.createSearchTasks(
        partitionsToSearch,
        queryVector,
        k,
        searchId
      )
      // Execute searches in parallel
      const searchResults = await this.executeParallelSearches(
        partitionedIndex,
        searchTasks
      )
      // Merge results from all partitions
      const mergedResults = this.mergeSearchResults(searchResults, k)
      // Update statistics
      this.updateSearchStats(searchId, startTime, searchResults)
      return mergedResults
    } catch (error) {
      console.error(`Distributed search ${searchId} failed:`, error)
      throw error
    }
  }
  /**
   * Select partitions to search based on strategy
   */
  private async selectPartitions(
    partitionedIndex: PartitionedHNSWIndex,
    queryVector: Vector,
    strategy: SearchStrategy
  ): Promise<string[]> {
    const stats = partitionedIndex.getPartitionStats()
    const allPartitionIds = stats.partitionDetails.map(p => p.id)
    switch (strategy) {
      case SearchStrategy.BROADCAST:
        return allPartitionIds
      case SearchStrategy.SELECTIVE:
        return this.selectTopPartitions(allPartitionIds, 3)
      case SearchStrategy.ADAPTIVE:
        return await this.adaptivePartitionSelection(allPartitionIds, queryVector)
      case SearchStrategy.HIERARCHICAL:
        return this.hierarchicalPartitionSelection(allPartitionIds)
      default:
        return allPartitionIds
    }
  }
  /**
   * Adaptive partition selection based on historical performance
   */
  private async adaptivePartitionSelection(
    partitionIds: string[],
    queryVector: Vector
  ): Promise<string[]> {
    const candidates: Array<{ id: string; score: number }> = []
    for (const partitionId of partitionIds) {
      const stats = this.partitionStats.get(partitionId)
      let score = 1.0
      if (stats) {
        // Score based on performance metrics
        const speedScore = 1000 / Math.max(stats.averageSearchTime, 1)
        const loadScore = Math.max(0, 1 - stats.load)
        const qualityScore = stats.quality
        const recencyScore = Math.max(0, 1 - (Date.now() - stats.lastUsed) / 3600000)
        score = speedScore * 0.3 + loadScore * 0.25 + qualityScore * 0.3 + recencyScore * 0.15
      }
      candidates.push({ id: partitionId, score })
    }
    // Sort by score and select top partitions
    candidates.sort((a, b) => b.score - a.score)
    const selectedCount = Math.min(Math.ceil(partitionIds.length * 0.6), 8)
    return candidates.slice(0, selectedCount).map(c => c.id)
  }
  /**
   * Select top-performing partitions
   */
  private selectTopPartitions(partitionIds: string[], count: number): string[] {
    const withStats = partitionIds.map(id => ({
      id,
      stats: this.partitionStats.get(id)
    }))
    // Sort by average search time (faster is better)
    withStats.sort((a, b) => {
      const timeA = a.stats?.averageSearchTime || 1000
      const timeB = b.stats?.averageSearchTime || 1000
      return timeA - timeB
    })
    return withStats.slice(0, count).map(p => p.id)
  }
  /**
   * Hierarchical partition selection for very large datasets
   */
  private hierarchicalPartitionSelection(partitionIds: string[]): string[] {
    // First level: select representative partitions
    const firstLevel = partitionIds.filter((_, index) => index % 3 === 0)
    // Could implement a two-phase search here:
    // 1. Quick search on representative partitions
    // 2. Detailed search on promising partitions
    return firstLevel
  }
  /**
   * Create search tasks for parallel execution
   */
  private createSearchTasks(
    partitionIds: string[],
    queryVector: Vector,
    k: number,
    searchId: string
  ): SearchTask[] {
    const tasks: SearchTask[] = []
    for (let i = 0; i < partitionIds.length; i++) {
      const partitionId = partitionIds[i]
      const stats = this.partitionStats.get(partitionId)
      // Calculate priority based on partition performance
      const priority = stats ? (1000 - stats.averageSearchTime) : 500
      tasks.push({
        partitionId,
        queryVector: [...queryVector], // Clone vector
        k: Math.max(k * 2, 20), // Search for more results per partition
        searchId,
        priority
      })
      // Add redundant searches if configured
      if (this.config.redundantSearches > 0 && i < this.config.redundantSearches) {
        tasks.push({
          partitionId,
          queryVector: [...queryVector],
          k: Math.max(k * 2, 20),
          searchId: `${searchId}_redundant_${i}`,
          priority: priority - 100 // Lower priority for redundant searches
        })
      }
    }
    // Sort tasks by priority
    tasks.sort((a, b) => b.priority - a.priority)
    return tasks
  }
  /**
   * Execute searches in parallel across selected partitions
   */
  private async executeParallelSearches(
    partitionedIndex: PartitionedHNSWIndex,
    searchTasks: SearchTask[]
  ): Promise<PartitionSearchResult[]> {
    const results: PartitionSearchResult[] = []
    const semaphore = new Semaphore(this.config.maxConcurrentSearches)
    // Execute tasks with controlled concurrency
    const taskPromises = searchTasks.map(async (task) => {
      await semaphore.acquire()
      try {
        const startTime = Date.now()
        // Execute search with timeout
        const searchPromise = this.executePartitionSearch(partitionedIndex, task)
        const timeoutPromise = new Promise<PartitionSearchResult>((_, reject) => {
          setTimeout(() => reject(new Error('Search timeout')), this.config.searchTimeout)
        })
        const result = await Promise.race([searchPromise, timeoutPromise])
        result.searchTime = Date.now() - startTime
        return result
      } catch (error) {
        return {
          partitionId: task.partitionId,
          results: [],
          searchTime: this.config.searchTimeout,
          nodesVisited: 0,
          error: error as Error
        }
      } finally {
        semaphore.release()
      }
    })
    // Wait for all searches to complete
    const taskResults = await Promise.allSettled(taskPromises)
    for (const result of taskResults) {
      if (result.status === 'fulfilled') {
        results.push(result.value)
      }
    }
    return results
  }
  /**
   * Execute search on a single partition
   */
  private async executePartitionSearch(
    partitionedIndex: PartitionedHNSWIndex,
    task: SearchTask
  ): Promise<PartitionSearchResult> {
    try {
      // Use thread pool for compute-intensive operations
      if (this.shouldUseWorkerThread(task)) {
        return await this.executeInWorkerThread(partitionedIndex, task)
      }
      // Execute search directly
      const results = await partitionedIndex.search(
        task.queryVector,
        task.k,
        { partitionIds: [task.partitionId] }
      )
      return {
        partitionId: task.partitionId,
        results,
        searchTime: 0, // Will be set by caller
        nodesVisited: results.length // Approximation
      }
    } catch (error) {
      throw new Error(`Partition search failed: ${error}`)
    }
  }
  /**
   * Determine if search should use worker thread
   */
  private shouldUseWorkerThread(task: SearchTask): boolean {
    // Use worker threads for high-dimensional vectors or large k
    return task.queryVector.length > 512 || task.k > 100
  }
  /**
   * Execute search in worker thread
   */
  private async executeInWorkerThread(
    partitionedIndex: PartitionedHNSWIndex,
    task: SearchTask
  ): Promise<PartitionSearchResult> {
    const worker = this.getAvailableWorker()
    if (!worker) {
      // No available workers, execute synchronously
      return this.executePartitionSearch(partitionedIndex, task)
    }
    try {
      worker.busy = true
      const startTime = Date.now()
      // Execute in thread (simplified - would need proper worker setup)
      const searchFunction = `
        return partitionedIndex.search(
          task.queryVector,
          task.k,
          { partitionIds: [task.partitionId] }
        )
      `
      const results = await executeInThread<Array<[string, number]>>(searchFunction, {
        queryVector: task.queryVector,
        k: task.k,
        partitionId: task.partitionId
      })
      const searchTime = Date.now() - startTime
      worker.averageTaskTime = (worker.averageTaskTime + searchTime) / 2
      worker.tasksCompleted++
      return {
        partitionId: task.partitionId,
        results: results || [] as Array<[string, number]>,
        searchTime,
        nodesVisited: results ? results.length : 0
      }
    } finally {
      worker.busy = false
      worker.lastTaskTime = Date.now()
    }
  }
  /**
   * Get available worker from pool
   */
  private getAvailableWorker(): SearchWorker | null {
    for (const worker of this.searchWorkers.values()) {
      if (!worker.busy) {
        return worker
      }
    }
    return null
  }
  /**
   * Merge search results from multiple partitions
   */
  private mergeSearchResults(
    partitionResults: PartitionSearchResult[],
    k: number
  ): Array<[string, number]> {
    const allResults: Array<[string, number]> = []
    const seenIds = new Set<string>()
    // Collect all unique results
    for (const partitionResult of partitionResults) {
      if (partitionResult.error) {
        console.warn(`Partition ${partitionResult.partitionId} failed:`, partitionResult.error)
        continue
      }
      for (const [id, distance] of partitionResult.results) {
        if (!seenIds.has(id)) {
          allResults.push([id, distance])
          seenIds.add(id)
        }
      }
    }
    // Sort and return top k results
    switch (this.config.resultMergeStrategy) {
      case 'distance':
        allResults.sort((a, b) => a[1] - b[1])
        break
      case 'score':
        // Convert distance to score (1 / (1 + distance))
        allResults.sort((a, b) => {
          const scoreA = 1 / (1 + a[1])
          const scoreB = 1 / (1 + b[1])
          return scoreB - scoreA
        })
        break
      case 'hybrid':
        // Weighted combination of distance and partition quality
        allResults.sort((a, b) => {
          const qualityWeightA = this.getPartitionQuality(a[0])
          const qualityWeightB = this.getPartitionQuality(b[0])
          const adjustedDistanceA = a[1] / (qualityWeightA + 0.1)
          const adjustedDistanceB = b[1] / (qualityWeightB + 0.1)
          return adjustedDistanceA - adjustedDistanceB
        })
        break
    }
    return allResults.slice(0, k)
  }
  /**
   * Get partition quality score
   */
  private getPartitionQuality(nodeId: string): number {
    // This would require knowing which partition a node came from
    // For now, return a default quality score
    return 1.0
  }
  /**
   * Update search statistics
   */
  private updateSearchStats(
    searchId: string,
    startTime: number,
    results: PartitionSearchResult[]
  ): void {
    const totalTime = Date.now() - startTime
    const successfulSearches = results.filter(r => !r.error)
    // Update global stats
    this.searchStats.totalSearches++
    this.searchStats.averageLatency = 
      (this.searchStats.averageLatency + totalTime) / 2
    // Calculate parallel efficiency
    const totalPartitionTime = results.reduce((sum, r) => sum + r.searchTime, 0)
    this.searchStats.parallelEfficiency = 
      totalPartitionTime > 0 ? totalTime / totalPartitionTime : 0
    // Update partition statistics
    for (const result of successfulSearches) {
      let stats = this.partitionStats.get(result.partitionId)
      if (!stats) {
        stats = {
          averageSearchTime: result.searchTime,
          load: 0,
          quality: 1.0,
          lastUsed: Date.now()
        }
      } else {
        stats.averageSearchTime = (stats.averageSearchTime + result.searchTime) / 2
        stats.lastUsed = Date.now()
      }
      this.partitionStats.set(result.partitionId, stats)
      this.searchStats.partitionUtilization.set(
        result.partitionId,
        (this.searchStats.partitionUtilization.get(result.partitionId) || 0) + 1
      )
    }
  }
  /**
   * Initialize worker thread pool
   */
  private initializeWorkerPool(): void {
    const workerCount = Math.min(navigator.hardwareConcurrency || 4, 8)
    for (let i = 0; i < workerCount; i++) {
      const worker: SearchWorker = {
        id: `worker_${i}`,
        busy: false,
        tasksCompleted: 0,
        averageTaskTime: 0,
        lastTaskTime: 0
      }
      this.searchWorkers.set(worker.id, worker)
    }
    console.log(`Initialized worker pool with ${workerCount} workers`)
  }
  /**
   * Generate unique search ID
   */
  private generateSearchId(): string {
    return `search_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`
  }
  /**
   * Get search performance statistics
   */
  public getSearchStats(): typeof this.searchStats & {
    workerStats: SearchWorker[]
    partitionStats: Array<{ id: string; stats: any }>
  } {
    return {
      ...this.searchStats,
      workerStats: Array.from(this.searchWorkers.values()),
      partitionStats: Array.from(this.partitionStats.entries()).map(([id, stats]) => ({
        id,
        stats
      }))
    }
  }
  /**
   * Cleanup resources
   */
  public cleanup(): void {
    // Clear active searches
    this.activeSearches.clear()
    // Reset worker states
    for (const worker of this.searchWorkers.values()) {
      worker.busy = false
    }
    // Clear statistics
    this.partitionStats.clear()
  }
 }
 /**
 * Simple semaphore for concurrency control
 */
 class Semaphore {
  private permits: number
  private waiting: Array<() => void> = []
  constructor(permits: number) {
    this.permits = permits
  }
  async acquire(): Promise<void> {
    if (this.permits > 0) {
      this.permits--
      return Promise.resolve()
    }
    return new Promise<void>((resolve) => {
      this.waiting.push(resolve)
    })
  }
  release(): void {
    if (this.waiting.length > 0) {
      const resolve = this.waiting.shift()!
      resolve()
    } else {
      this.permits++
    }
  }
 }
--- a/src/hnsw/hnswIndex.ts
+++ b/src/hnsw/hnswIndex.ts
@ -490,6 +490,29 @@ export class HNSWIndex {
    return id
  }
  /**
   * O(1) entry point recovery using highLevelNodes index (v6.2.3).
   * At any reasonable scale (1000+ nodes), level 2+ nodes are guaranteed to exist.
   * For tiny indexes with only level 0-1 nodes, any node works as entry point.
   */
  private recoverEntryPointO1(): { id: string | null; level: number } {
    // O(1) recovery: check highLevelNodes from highest to lowest level
    for (let level = this.MAX_TRACKED_LEVELS; level >= 2; level--) {
      const nodesAtLevel = this.highLevelNodes.get(level)
      if (nodesAtLevel && nodesAtLevel.size > 0) {
        for (const nodeId of nodesAtLevel) {
          if (this.nouns.has(nodeId)) {
            return { id: nodeId, level }
          }
        }
      }
    }
    // No high-level nodes - use any available node (works fine for HNSW)
    const firstNode = this.nouns.keys().next().value
    return { id: firstNode ?? null, level: 0 }
  }
  /**
   * Search for nearest neighbors
   */
@ -514,20 +537,12 @@ export class HNSWIndex {
    }
    // Start from the entry point
-    // If entry point is null but nouns exist, attempt recovery (v6.2.2)
+    // If entry point is null but nouns exist, attempt O(1) recovery (v6.2.3)
    if (!this.entryPointId && this.nouns.size > 0) {
-      // Corrupted state: nouns exist but entry point is null - recover
+      const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
      let maxLevel = 0
      let recoveredId: string | null = null
      for (const [id, noun] of this.nouns.entries()) {
        if (noun.level >= maxLevel) {
          maxLevel = noun.level
          recoveredId = id
        }
      }
      if (recoveredId) {
        this.entryPointId = recoveredId
-        this.maxLevel = maxLevel
+        this.maxLevel = recoveredLevel
      }
    }
@ -538,19 +553,12 @@ export class HNSWIndex {
    let entryPoint = this.nouns.get(this.entryPointId)
    if (!entryPoint) {
-      // Entry point ID exists but noun was deleted - attempt recovery
+      // Entry point ID exists but noun was deleted - O(1) recovery (v6.2.3)
      if (this.nouns.size > 0) {
-        let maxLevel = 0
+        const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
        let recoveredId: string | null = null
        for (const [id, noun] of this.nouns.entries()) {
          if (noun.level >= maxLevel) {
            maxLevel = noun.level
            recoveredId = id
          }
        }
        if (recoveredId) {
          this.entryPointId = recoveredId
-          this.maxLevel = maxLevel
+          this.maxLevel = recoveredLevel
          entryPoint = this.nouns.get(recoveredId)
        }
      }
@ -1200,28 +1208,20 @@ export class HNSWIndex {
        }
      }
-      // Step 5: CRITICAL - Recover entry point if missing (v6.2.2)
+      // Step 5: CRITICAL - Recover entry point if missing (v6.2.3 - O(1))
      // This ensures consistency even if getHNSWSystem() returned null
      if (this.nouns.size > 0 && this.entryPointId === null) {
-        prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering from loaded data')
+        prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering with O(1) lookup')
-        let maxLevel = 0
+        const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
        let recoveredEntryPointId: string | null = null
-        for (const [id, noun] of this.nouns.entries()) {
+        this.entryPointId = recoveredId
-          if (noun.level >= maxLevel) {
+        this.maxLevel = recoveredLevel
            maxLevel = noun.level
            recoveredEntryPointId = id
          }
        }
-        this.entryPointId = recoveredEntryPointId
+        prodLog.info(`HNSW entry point recovered: ${recoveredId} at level ${recoveredLevel}`)
        this.maxLevel = maxLevel
        prodLog.info(`HNSW entry point recovered: ${recoveredEntryPointId} at level ${maxLevel}`)
        // Persist recovered state to prevent future recovery
-        if (this.storage && recoveredEntryPointId) {
+        if (this.storage && recoveredId) {
          await this.storage.saveHNSWSystem({
            entryPointId: this.entryPointId,
            maxLevel: this.maxLevel
@ -1233,18 +1233,11 @@ export class HNSWIndex {
      // Step 6: Validate entry point exists if set (handles stale/deleted entry point)
      if (this.entryPointId && !this.nouns.has(this.entryPointId)) {
-        prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering`)
+        prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering with O(1) lookup`)
-        let maxLevel = 0
+        const { id: recoveredId, level: recoveredLevel } = this.recoverEntryPointO1()
        let recoveredId: string | null = null
        for (const [id, noun] of this.nouns.entries()) {
          if (noun.level >= maxLevel) {
            maxLevel = noun.level
            recoveredId = id
          }
        }
        this.entryPointId = recoveredId
-        this.maxLevel = maxLevel
+        this.maxLevel = recoveredLevel
        // Persist corrected state
        if (this.storage && recoveredId) {
--- a/src/hnsw/hnswIndexOptimized.ts
+++ b/src/hnsw/hnswIndexOptimized.ts
@ -1,585 +0,0 @@
 /**
 * Optimized HNSW (Hierarchical Navigable Small World) Index implementation
 * Extends the base HNSW implementation with support for large datasets
 * Uses product quantization for dimensionality reduction and disk-based storage when needed
 */
 import {
  DistanceFunction,
  HNSWConfig,
  HNSWNoun,
  Vector,
  VectorDocument
 } from '../coreTypes.js'
 import { HNSWIndex } from './hnswIndex.js'
 import type { BaseStorage } from '../storage/baseStorage.js'
 // Configuration for the optimized HNSW index
 export interface HNSWOptimizedConfig extends HNSWConfig {
  // Memory threshold in bytes - when exceeded, will use disk-based approach
  memoryThreshold?: number
  // Product quantization settings
  productQuantization?: {
    // Whether to use product quantization
    enabled: boolean
    // Number of subvectors to split the vector into
    numSubvectors?: number
    // Number of centroids per subvector
    numCentroids?: number
  }
  // Whether to use disk-based storage for the index
  useDiskBasedIndex?: boolean
 }
 // Default configuration for the optimized HNSW index
 const DEFAULT_OPTIMIZED_CONFIG: HNSWOptimizedConfig = {
  M: 16,
  efConstruction: 200,
  efSearch: 50,
  ml: 16,
  memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
  productQuantization: {
    enabled: false,
    numSubvectors: 16,
    numCentroids: 256
  },
  useDiskBasedIndex: false
 }
 /**
 * Product Quantization implementation
 * Reduces vector dimensionality by splitting vectors into subvectors
 * and quantizing each subvector to the nearest centroid
 */
 class ProductQuantizer {
  private numSubvectors: number
  private numCentroids: number
  private centroids: Vector[][] = []
  private subvectorSize: number = 0
  private initialized: boolean = false
  private dimension: number = 0
  constructor(numSubvectors: number = 16, numCentroids: number = 256) {
    this.numSubvectors = numSubvectors
    this.numCentroids = numCentroids
  }
  /**
   * Initialize the product quantizer with training data
   * @param vectors Training vectors to use for learning centroids
   */
  public train(vectors: Vector[]): void {
    if (vectors.length === 0) {
      throw new Error('Cannot train product quantizer with empty vector set')
    }
    this.dimension = vectors[0].length
    this.subvectorSize = Math.ceil(this.dimension / this.numSubvectors)
    // Initialize centroids for each subvector
    for (let i = 0; i < this.numSubvectors; i++) {
      // Extract subvectors from training data
      const subvectors: Vector[] = vectors.map((vector) => {
        const start = i * this.subvectorSize
        const end = Math.min(start + this.subvectorSize, this.dimension)
        return vector.slice(start, end)
      })
      // Initialize centroids for this subvector using k-means++
      this.centroids[i] = this.kMeansPlusPlus(subvectors, this.numCentroids)
    }
    this.initialized = true
  }
  /**
   * Quantize a vector using product quantization
   * @param vector Vector to quantize
   * @returns Array of centroid indices, one for each subvector
   */
  public quantize(vector: Vector): number[] {
    if (!this.initialized) {
      throw new Error('Product quantizer not initialized. Call train() first.')
    }
    if (vector.length !== this.dimension) {
      throw new Error(
        `Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`
      )
    }
    const codes: number[] = []
    // Quantize each subvector
    for (let i = 0; i < this.numSubvectors; i++) {
      const start = i * this.subvectorSize
      const end = Math.min(start + this.subvectorSize, this.dimension)
      const subvector = vector.slice(start, end)
      // Find nearest centroid
      let minDist = Number.MAX_VALUE
      let nearestCentroidIndex = 0
      for (let j = 0; j < this.centroids[i].length; j++) {
        const centroid = this.centroids[i][j]
        const dist = this.euclideanDistanceSquared(subvector, centroid)
        if (dist < minDist) {
          minDist = dist
          nearestCentroidIndex = j
        }
      }
      codes.push(nearestCentroidIndex)
    }
    return codes
  }
  /**
   * Reconstruct a vector from its quantized representation
   * @param codes Array of centroid indices
   * @returns Reconstructed vector
   */
  public reconstruct(codes: number[]): Vector {
    if (!this.initialized) {
      throw new Error('Product quantizer not initialized. Call train() first.')
    }
    if (codes.length !== this.numSubvectors) {
      throw new Error(
        `Code length mismatch: expected ${this.numSubvectors}, got ${codes.length}`
      )
    }
    const reconstructed: Vector = []
    // Reconstruct each subvector
    for (let i = 0; i < this.numSubvectors; i++) {
      const centroidIndex = codes[i]
      const centroid = this.centroids[i][centroidIndex]
      // Add centroid components to reconstructed vector
      for (const component of centroid) {
        reconstructed.push(component)
      }
    }
    // Trim to original dimension if needed
    return reconstructed.slice(0, this.dimension)
  }
  /**
   * Compute squared Euclidean distance between two vectors
   * @param a First vector
   * @param b Second vector
   * @returns Squared Euclidean distance
   */
  private euclideanDistanceSquared(a: Vector, b: Vector): number {
    let sum = 0
    const length = Math.min(a.length, b.length)
    for (let i = 0; i < length; i++) {
      const diff = a[i] - b[i]
      sum += diff * diff
    }
    return sum
  }
  /**
   * Implement k-means++ algorithm to initialize centroids
   * @param vectors Vectors to cluster
   * @param k Number of clusters
   * @returns Array of centroids
   */
  private kMeansPlusPlus(vectors: Vector[], k: number): Vector[] {
    if (vectors.length < k) {
      // If we have fewer vectors than centroids, use the vectors as centroids
      return [...vectors]
    }
    const centroids: Vector[] = []
    // Choose first centroid randomly
    const firstIndex = Math.floor(Math.random() * vectors.length)
    centroids.push([...vectors[firstIndex]])
    // Choose remaining centroids
    for (let i = 1; i < k; i++) {
      // Compute distances to nearest centroid for each vector
      const distances: number[] = vectors.map((vector) => {
        let minDist = Number.MAX_VALUE
        for (const centroid of centroids) {
          const dist = this.euclideanDistanceSquared(vector, centroid)
          minDist = Math.min(minDist, dist)
        }
        return minDist
      })
      // Compute sum of distances
      const distSum = distances.reduce((sum, dist) => sum + dist, 0)
      // Choose next centroid with probability proportional to distance
      let r = Math.random() * distSum
      let nextIndex = 0
      for (let j = 0; j < distances.length; j++) {
        r -= distances[j]
        if (r <= 0) {
          nextIndex = j
          break
        }
      }
      centroids.push([...vectors[nextIndex]])
    }
    return centroids
  }
  /**
   * Get the centroids for each subvector
   * @returns Array of centroid arrays
   */
  public getCentroids(): Vector[][] {
    return this.centroids
  }
  /**
   * Set the centroids for each subvector
   * @param centroids Array of centroid arrays
   */
  public setCentroids(centroids: Vector[][]): void {
    this.centroids = centroids
    this.numSubvectors = centroids.length
    this.numCentroids = centroids[0].length
    this.initialized = true
  }
  /**
   * Get the dimension of the vectors
   * @returns Dimension
   */
  public getDimension(): number {
    return this.dimension
  }
  /**
   * Set the dimension of the vectors
   * @param dimension Dimension
   */
  public setDimension(dimension: number): void {
    this.dimension = dimension
    this.subvectorSize = Math.ceil(dimension / this.numSubvectors)
  }
 }
 /**
 * Optimized HNSW Index implementation
 * Extends the base HNSW implementation with support for large datasets
 * Uses product quantization for dimensionality reduction and disk-based storage when needed
 */
 export class HNSWIndexOptimized extends HNSWIndex {
  private optimizedConfig: HNSWOptimizedConfig
  private productQuantizer: ProductQuantizer | null = null
  private useDiskBasedIndex: boolean = false
  private useProductQuantization: boolean = false
  private quantizedVectors: Map<string, number[]> = new Map()
  private memoryUsage: number = 0
  private vectorCount: number = 0
  // Thread safety for memory usage tracking
  private memoryUpdateLock: Promise<void> = Promise.resolve()
  constructor(
    config: Partial<HNSWOptimizedConfig> = {},
    distanceFunction: DistanceFunction,
    storage: BaseStorage | null = null
  ) {
    // Initialize base HNSW index with standard config and storage
    super(config, distanceFunction, { storage: storage || undefined })
    // Set optimized config
    this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config }
    // Initialize product quantizer if enabled
    if (this.optimizedConfig.productQuantization?.enabled) {
      this.useProductQuantization = true
      this.productQuantizer = new ProductQuantizer(
        this.optimizedConfig.productQuantization.numSubvectors,
        this.optimizedConfig.productQuantization.numCentroids
      )
    }
    // Set disk-based index flag
    this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false
    // Note: UnifiedCache is inherited from base HNSWIndex class
  }
  /**
   * Thread-safe method to update memory usage
   * @param memoryDelta Change in memory usage (can be negative)
   * @param vectorCountDelta Change in vector count (can be negative)
   */
  private async updateMemoryUsage(memoryDelta: number, vectorCountDelta: number): Promise<void> {
    this.memoryUpdateLock = this.memoryUpdateLock.then(async () => {
      this.memoryUsage = Math.max(0, this.memoryUsage + memoryDelta)
      this.vectorCount = Math.max(0, this.vectorCount + vectorCountDelta)
    })
    await this.memoryUpdateLock
  }
  /**
   * Thread-safe method to get current memory usage
   * @returns Current memory usage and vector count
   */
  private async getMemoryUsageAsync(): Promise<{ memoryUsage: number; vectorCount: number }> {
    await this.memoryUpdateLock
    return {
      memoryUsage: this.memoryUsage,
      vectorCount: this.vectorCount
    }
  }
  /**
   * Add a vector to the index
   * Uses product quantization if enabled and memory threshold is exceeded
   */
  public override async addItem(item: VectorDocument): Promise<string> {
    // Check if item is defined
    if (!item) {
      throw new Error('Item is undefined or null')
    }
    const { id, vector } = item
    // Check if vector is defined
    if (!vector) {
      throw new Error('Vector is undefined or null')
    }
    // Estimate memory usage for this vector
    const vectorMemory = vector.length * 8 // 8 bytes per number (Float64)
    const connectionsMemory = this.optimizedConfig.M * this.optimizedConfig.ml * 16 // Estimate for connections
    const totalMemory = vectorMemory + connectionsMemory
    // Update memory usage estimate (thread-safe)
    await this.updateMemoryUsage(totalMemory, 1)
    // Check if we should switch to product quantization
    const currentMemoryUsage = await this.getMemoryUsageAsync()
    if (
      this.useProductQuantization &&
      currentMemoryUsage.memoryUsage > this.optimizedConfig.memoryThreshold! &&
      this.productQuantizer &&
      !this.productQuantizer.getDimension()
    ) {
      // Initialize product quantizer with existing vectors
      this.initializeProductQuantizer()
    }
    // If product quantization is active, quantize the vector
    if (
      this.useProductQuantization &&
      this.productQuantizer &&
      this.productQuantizer.getDimension() > 0
    ) {
      // Quantize the vector
      const codes = this.productQuantizer.quantize(vector)
      // Store the quantized vector
      this.quantizedVectors.set(id, codes)
      // Reconstruct the vector for indexing
      const reconstructedVector = this.productQuantizer.reconstruct(codes)
      // Add the reconstructed vector to the index
      return await super.addItem({ id, vector: reconstructedVector })
    }
    // If disk-based index is active and storage is available, store the vector
    if (this.useDiskBasedIndex) {
      // Storage is handled by the base class now via HNSW persistence
      // No additional storage needed here
    }
    // Add the vector to the in-memory index
    return await super.addItem(item)
  }
  /**
   * Search for nearest neighbors
   * Uses product quantization if enabled
   */
  public override async search(
    queryVector: Vector,
    k: number = 10
  ): Promise<Array<[string, number]>> {
    // Check if query vector is defined
    if (!queryVector) {
      throw new Error('Query vector is undefined or null')
    }
    // If product quantization is active, quantize the query vector
    if (
      this.useProductQuantization &&
      this.productQuantizer &&
      this.productQuantizer.getDimension() > 0
    ) {
      // Quantize the query vector
      const codes = this.productQuantizer.quantize(queryVector)
      // Reconstruct the query vector
      const reconstructedVector = this.productQuantizer.reconstruct(codes)
      // Search with the reconstructed vector
      return await super.search(reconstructedVector, k)
    }
    // Otherwise, use the standard search
    return await super.search(queryVector, k)
  }
  /**
   * Remove an item from the index
   */
  public override async removeItem(id: string): Promise<boolean> {
    // If product quantization is active, remove the quantized vector
    if (this.useProductQuantization) {
      this.quantizedVectors.delete(id)
    }
    // If disk-based index is active, removal is handled by base class
    // No additional removal needed here
    // Update memory usage estimate (async operation, but don't block removal)
    this.getMemoryUsageAsync().then((currentMemoryUsage) => {
      if (currentMemoryUsage.vectorCount > 0) {
        const memoryPerVector = currentMemoryUsage.memoryUsage / currentMemoryUsage.vectorCount
        this.updateMemoryUsage(-memoryPerVector, -1)
      }
    }).catch((error) => {
      console.error('Failed to update memory usage after removal:', error)
    })
    // Remove the item from the in-memory index
    return await super.removeItem(id)
  }
  /**
   * Clear the index
   */
  public override async clear(): Promise<void> {
    // Clear product quantization data
    if (this.useProductQuantization) {
      this.quantizedVectors.clear()
      this.productQuantizer = new ProductQuantizer(
        this.optimizedConfig.productQuantization!.numSubvectors,
        this.optimizedConfig.productQuantization!.numCentroids
      )
    }
    // Reset memory usage (thread-safe)
    const currentMemoryUsage = await this.getMemoryUsageAsync()
    await this.updateMemoryUsage(-currentMemoryUsage.memoryUsage, -currentMemoryUsage.vectorCount)
    // Clear the in-memory index
    super.clear()
  }
  /**
   * Initialize product quantizer with existing vectors
   */
  private initializeProductQuantizer(): void {
    if (!this.productQuantizer) {
      return
    }
    // Get all vectors from the index
    const nouns = super.getNouns()
    const vectors: Vector[] = []
    // Extract vectors
    for (const [_, noun] of nouns) {
      vectors.push(noun.vector)
    }
    // Train the product quantizer
    if (vectors.length > 0) {
      this.productQuantizer.train(vectors)
      // Quantize all existing vectors
      for (const [id, noun] of nouns) {
        const codes = this.productQuantizer.quantize(noun.vector)
        this.quantizedVectors.set(id, codes)
      }
      console.log(
        `Initialized product quantizer with ${vectors.length} vectors`
      )
    }
  }
  /**
   * Get the product quantizer
   * @returns Product quantizer or null if not enabled
   */
  public getProductQuantizer(): ProductQuantizer | null {
    return this.productQuantizer
  }
  /**
   * Get the optimized configuration
   * @returns Optimized configuration
   */
  public getOptimizedConfig(): HNSWOptimizedConfig {
    return { ...this.optimizedConfig }
  }
  /**
   * Get the estimated memory usage
   * @returns Estimated memory usage in bytes
   */
  public getMemoryUsage(): number {
    return this.memoryUsage
  }
  // Storage methods removed - now handled by base class
  /**
   * Set whether to use disk-based index
   * @param useDiskBasedIndex Whether to use disk-based index
   */
  public setUseDiskBasedIndex(useDiskBasedIndex: boolean): void {
    this.useDiskBasedIndex = useDiskBasedIndex
  }
  /**
   * Get whether disk-based index is used
   * @returns Whether disk-based index is used
   */
  public getUseDiskBasedIndex(): boolean {
    return this.useDiskBasedIndex
  }
  /**
   * Set whether to use product quantization
   * @param useProductQuantization Whether to use product quantization
   */
  public setUseProductQuantization(useProductQuantization: boolean): void {
    this.useProductQuantization = useProductQuantization
  }
  /**
   * Get whether product quantization is used
   * @returns Whether product quantization is used
   */
  public getUseProductQuantization(): boolean {
    return this.useProductQuantization
  }
 }
--- a/src/hnsw/optimizedHNSWIndex.ts
+++ b/src/hnsw/optimizedHNSWIndex.ts
@ -1,431 +0,0 @@
 /**
 * Optimized HNSW Index for Large-Scale Vector Search
 * Implements dynamic parameter tuning and performance optimizations
 */
 import {
  DistanceFunction,
  HNSWConfig,
  HNSWNoun,
  Vector,
  VectorDocument
 } from '../coreTypes.js'
 import { HNSWIndex } from './hnswIndex.js'
 import { euclideanDistance } from '../utils/index.js'
 export interface OptimizedHNSWConfig extends HNSWConfig {
  // Dynamic tuning parameters
  dynamicParameterTuning?: boolean
  targetSearchLatency?: number // ms
  targetRecall?: number // 0.0 to 1.0
  // Large-scale optimizations
  maxNodes?: number
  memoryBudget?: number // bytes
  diskCacheEnabled?: boolean
  compressionEnabled?: boolean
  // Performance monitoring
  performanceTracking?: boolean
  adaptiveEfSearch?: boolean
  // Advanced optimizations
  levelMultiplier?: number
  seedConnections?: number
  pruningStrategy?: 'simple' | 'diverse' | 'hybrid'
 }
 interface PerformanceMetrics {
  averageSearchTime: number
  averageRecall: number
  memoryUsage: number
  indexSize: number
  apiCalls: number
  cacheHitRate: number
 }
 interface DynamicParameters {
  efSearch: number
  efConstruction: number
  M: number
  ml: number
 }
 /**
 * Optimized HNSW Index with dynamic parameter tuning for large datasets
 */
 export class OptimizedHNSWIndex extends HNSWIndex {
  private optimizedConfig: Required<Omit<OptimizedHNSWConfig, 'maxConcurrentNeighborWrites'>> & { maxConcurrentNeighborWrites?: number }
  private performanceMetrics: PerformanceMetrics
  private dynamicParams: DynamicParameters
  private searchHistory: Array<{ latency: number; k: number; timestamp: number }> = []
  private parameterTuningInterval?: NodeJS.Timeout
  constructor(
    config: Partial<OptimizedHNSWConfig> = {},
    distanceFunction: DistanceFunction = euclideanDistance
  ) {
    // Set optimized defaults for large scale
    const defaultConfig: Required<Omit<OptimizedHNSWConfig, 'maxConcurrentNeighborWrites'>> = {
      M: 32, // Higher connectivity for better recall
      efConstruction: 400, // Better build quality
      efSearch: 100, // Dynamic - will be tuned
      ml: 24, // Deeper hierarchy
      useDiskBasedIndex: false, // Added missing property
      dynamicParameterTuning: true,
      targetSearchLatency: 100, // 100ms target
      targetRecall: 0.95, // 95% recall target
      maxNodes: 1000000, // 1M node limit
      memoryBudget: 8 * 1024 * 1024 * 1024, // 8GB
      diskCacheEnabled: true,
      compressionEnabled: false, // Disabled by default for compatibility
      performanceTracking: true,
      adaptiveEfSearch: true,
      levelMultiplier: 16,
      seedConnections: 8,
      pruningStrategy: 'hybrid'
      // maxConcurrentNeighborWrites intentionally omitted - optional property from parent HNSWConfig (v4.10.0+)
    }
    const mergedConfig = { ...defaultConfig, ...config }
    // Initialize parent with base config
    super(
      {
        M: mergedConfig.M,
        efConstruction: mergedConfig.efConstruction,
        efSearch: mergedConfig.efSearch,
        ml: mergedConfig.ml
      },
      distanceFunction,
      { useParallelization: true }
    )
    this.optimizedConfig = mergedConfig
    // Initialize dynamic parameters
    this.dynamicParams = {
      efSearch: mergedConfig.efSearch,
      efConstruction: mergedConfig.efConstruction,
      M: mergedConfig.M,
      ml: mergedConfig.ml
    }
    // Initialize performance metrics
    this.performanceMetrics = {
      averageSearchTime: 0,
      averageRecall: 0,
      memoryUsage: 0,
      indexSize: 0,
      apiCalls: 0,
      cacheHitRate: 0
    }
    // Start parameter tuning if enabled
    if (this.optimizedConfig.dynamicParameterTuning) {
      this.startParameterTuning()
    }
  }
  /**
   * Optimized search with dynamic parameter adjustment
   */
  public async search(
    queryVector: Vector,
    k: number = 10,
    filter?: (id: string) => Promise<boolean>
  ): Promise<Array<[string, number]>> {
    const startTime = Date.now()
    // Adjust efSearch dynamically based on k and performance history
    if (this.optimizedConfig.adaptiveEfSearch) {
      this.adjustEfSearch(k)
    }
    // Check memory usage and trigger optimizations if needed
    if (this.optimizedConfig.performanceTracking) {
      this.checkMemoryUsage()
    }
    // Perform the search with current parameters
    const originalConfig = this.getConfig()
    // Temporarily update search parameters
    const tempConfig = {
      ...originalConfig,
      efSearch: this.dynamicParams.efSearch
    }
    // Use the parent's search method with optimized parameters
    let results: Array<[string, number]>
    try {
      // This is a simplified approach - in practice, we'd need to modify
      // the parent class to accept runtime parameter changes
      results = await super.search(queryVector, k, filter)
    } catch (error) {
      console.error('Optimized search failed, falling back to default:', error)
      results = await super.search(queryVector, k, filter)
    }
    // Record performance metrics
    const searchTime = Date.now() - startTime
    this.recordSearchMetrics(searchTime, k, results.length)
    return results
  }
  /**
   * Dynamically adjust efSearch based on performance requirements
   */
  private adjustEfSearch(k: number): void {
    const recentSearches = this.searchHistory.slice(-10)
    if (recentSearches.length < 3) {
      // Not enough data, use heuristic
      this.dynamicParams.efSearch = Math.max(k * 2, 50)
      return
    }
    const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length
    const targetLatency = this.optimizedConfig.targetSearchLatency
    // Adjust efSearch based on latency performance
    if (averageLatency > targetLatency * 1.2) {
      // Too slow, reduce efSearch
      this.dynamicParams.efSearch = Math.max(
        Math.floor(this.dynamicParams.efSearch * 0.9),
        k
      )
    } else if (averageLatency < targetLatency * 0.8) {
      // Fast enough, can increase efSearch for better recall
      this.dynamicParams.efSearch = Math.min(
        Math.floor(this.dynamicParams.efSearch * 1.1),
        500 // Maximum efSearch
      )
    }
    // Ensure efSearch is at least k
    this.dynamicParams.efSearch = Math.max(this.dynamicParams.efSearch, k)
  }
  /**
   * Record search performance metrics
   */
  private recordSearchMetrics(latency: number, k: number, resultCount: number): void {
    if (!this.optimizedConfig.performanceTracking) {
      return
    }
    // Add to search history
    this.searchHistory.push({
      latency,
      k,
      timestamp: Date.now()
    })
    // Keep only recent history (last 100 searches)
    if (this.searchHistory.length > 100) {
      this.searchHistory.shift()
    }
    // Update performance metrics
    const recentSearches = this.searchHistory.slice(-20)
    this.performanceMetrics.averageSearchTime = 
      recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length
    // Estimate recall (simplified - would need ground truth for accurate measurement)
    this.performanceMetrics.averageRecall = Math.min(resultCount / k, 1.0)
  }
  /**
   * Check memory usage and trigger optimizations
   */
  private checkMemoryUsage(): void {
    // Estimate memory usage (simplified)
    const estimatedMemory = this.size() * 1000 // Rough estimate per node
    this.performanceMetrics.memoryUsage = estimatedMemory
    if (estimatedMemory > this.optimizedConfig.memoryBudget * 0.9) {
      console.warn('Memory usage approaching limit, consider index partitioning')
      // Could trigger automatic partitioning or compression here
      if (this.optimizedConfig.compressionEnabled) {
        this.compressIndex()
      }
    }
  }
  /**
   * Compress index to reduce memory usage (placeholder)
   */
  private compressIndex(): void {
    console.log('Index compression not implemented yet')
    // This would implement vector quantization or other compression techniques
  }
  /**
   * Start automatic parameter tuning
   */
  private startParameterTuning(): void {
    this.parameterTuningInterval = setInterval(() => {
      this.tuneParameters()
    }, 30000) // Tune every 30 seconds
  }
  /**
   * Automatic parameter tuning based on performance metrics
   */
  private tuneParameters(): void {
    if (this.searchHistory.length < 10) {
      return // Not enough data
    }
    const recentSearches = this.searchHistory.slice(-20)
    const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length
    // Tune based on performance vs targets
    const latencyRatio = averageLatency / this.optimizedConfig.targetSearchLatency
    const recallRatio = this.performanceMetrics.averageRecall / this.optimizedConfig.targetRecall
    // Adjust M (connectivity) for long-term performance
    if (this.size() > 10000) { // Only tune for larger indices
      if (recallRatio < 0.95 && latencyRatio < 1.5) {
        // Recall is low but we have latency budget, increase M
        this.dynamicParams.M = Math.min(this.dynamicParams.M + 2, 64)
      } else if (latencyRatio > 1.2 && recallRatio > 1.0) {
        // Latency is high but recall is good, can reduce M
        this.dynamicParams.M = Math.max(this.dynamicParams.M - 2, 16)
      }
    }
    console.log(`Parameter tuning: efSearch=${this.dynamicParams.efSearch}, M=${this.dynamicParams.M}, latency=${averageLatency.toFixed(1)}ms`)
  }
  /**
   * Get optimized configuration recommendations for current dataset size
   */
  public getOptimizedConfig(): OptimizedHNSWConfig {
    const currentSize = this.size()
    let recommendedConfig: Partial<OptimizedHNSWConfig> = {}
    if (currentSize < 10000) {
      // Small dataset - optimize for speed
      recommendedConfig = {
        M: 16,
        efConstruction: 200,
        efSearch: 50,
        ml: 16
      }
    } else if (currentSize < 100000) {
      // Medium dataset - balance speed and recall
      recommendedConfig = {
        M: 24,
        efConstruction: 300,
        efSearch: 75,
        ml: 20
      }
    } else if (currentSize < 1000000) {
      // Large dataset - optimize for recall
      recommendedConfig = {
        M: 32,
        efConstruction: 400,
        efSearch: 100,
        ml: 24
      }
    } else {
      // Very large dataset - maximum quality
      recommendedConfig = {
        M: 48,
        efConstruction: 500,
        efSearch: 150,
        ml: 28
      }
    }
    return {
      ...this.optimizedConfig,
      ...recommendedConfig
    }
  }
  /**
   * Get current performance metrics
   */
  public getPerformanceMetrics(): PerformanceMetrics & {
    currentParams: DynamicParameters
    searchHistorySize: number
  } {
    return {
      ...this.performanceMetrics,
      currentParams: { ...this.dynamicParams },
      searchHistorySize: this.searchHistory.length
    }
  }
  /**
   * Apply optimized bulk insertion strategy
   */
  public async bulkInsert(items: VectorDocument[]): Promise<string[]> {
    console.log(`Starting optimized bulk insert of ${items.length} items`)
    // Sort items to optimize insertion order (by vector similarity)
    const sortedItems = this.optimizeInsertionOrder(items)
    // Temporarily adjust construction parameters for bulk operations
    const originalEfConstruction = this.dynamicParams.efConstruction
    this.dynamicParams.efConstruction = Math.min(
      this.dynamicParams.efConstruction * 1.5,
      800
    )
    const results: string[] = []
    const batchSize = 100
    try {
      // Process in batches to manage memory
      for (let i = 0; i < sortedItems.length; i += batchSize) {
        const batch = sortedItems.slice(i, i + batchSize)
        for (const item of batch) {
          const id = await this.addItem(item)
          results.push(id)
        }
        // Periodic memory check
        if (i % (batchSize * 10) === 0) {
          this.checkMemoryUsage()
        }
      }
    } finally {
      // Restore original construction parameters
      this.dynamicParams.efConstruction = originalEfConstruction
    }
    console.log(`Completed bulk insert of ${results.length} items`)
    return results
  }
  /**
   * Optimize insertion order to improve index quality
   */
  private optimizeInsertionOrder(items: VectorDocument[]): VectorDocument[] {
    if (items.length < 100) {
      return items // Not worth optimizing small batches
    }
    // Simple clustering-based ordering
    // In practice, you might use more sophisticated methods
    return items.sort(() => Math.random() - 0.5) // Shuffle for now
  }
  /**
   * Cleanup resources
   */
  public destroy(): void {
    if (this.parameterTuningInterval) {
      clearInterval(this.parameterTuningInterval)
    }
  }
 }
--- a/src/hnsw/partitionedHNSWIndex.ts
+++ b/src/hnsw/partitionedHNSWIndex.ts
@ -1,413 +0,0 @@
 /**
 * Partitioned HNSW Index for Large-Scale Vector Search
 * Implements sharding strategies to handle millions of vectors efficiently
 */
 import {
  DistanceFunction,
  HNSWConfig,
  HNSWNoun,
  Vector,
  VectorDocument
 } from '../coreTypes.js'
 import { HNSWIndex } from './hnswIndex.js'
 import { euclideanDistance } from '../utils/index.js'
 export interface PartitionConfig {
  maxNodesPerPartition: number
  partitionStrategy: 'semantic' | 'hash' // Simplified to focus on useful strategies
  semanticClusters?: number // Auto-configured based on dataset size
  autoTuneSemanticClusters?: boolean // Automatically adjust cluster count
 }
 export interface PartitionMetadata {
  id: string
  nodeCount: number
  bounds?: {
    centroid: Vector
    radius: number
  }
  strategy: string
  created: Date
 }
 /**
 * Partitioned HNSW Index that splits large datasets across multiple smaller indices
 * This enables efficient search across millions of vectors by reducing memory usage
 * and parallelizing search operations
 */
 export class PartitionedHNSWIndex {
  private partitions: Map<string, HNSWIndex> = new Map()
  private partitionMetadata: Map<string, PartitionMetadata> = new Map()
  private config: PartitionConfig
  private hnswConfig: HNSWConfig
  private distanceFunction: DistanceFunction
  private dimension: number | null = null
  private nextPartitionId = 0
  constructor(
    partitionConfig: Partial<PartitionConfig> = {},
    hnswConfig: Partial<HNSWConfig> = {},
    distanceFunction: DistanceFunction = euclideanDistance
  ) {
    this.config = {
      maxNodesPerPartition: 50000, // Optimal size for memory efficiency
      partitionStrategy: 'semantic', // Default to semantic for better performance
      semanticClusters: 8, // Auto-tuned based on dataset
      autoTuneSemanticClusters: true,
      ...partitionConfig
    }
    // Optimized HNSW parameters for large scale
    this.hnswConfig = {
      M: 32, // Higher connectivity for better recall
      efConstruction: 400, // Better build quality
      efSearch: 100, // Balance speed vs accuracy
      ml: 24, // Deeper hierarchy
      ...hnswConfig
    }
    this.distanceFunction = distanceFunction
  }
  /**
   * Add a vector to the partitioned index
   */
  public async addItem(item: VectorDocument): Promise<string> {
    if (this.dimension === null) {
      this.dimension = item.vector.length
    }
    // Determine which partition this item belongs to
    const partitionId = await this.selectPartition(item)
    // Get or create the partition
    let partition = this.partitions.get(partitionId)
    if (!partition) {
      partition = new HNSWIndex(
        this.hnswConfig,
        this.distanceFunction,
        { useParallelization: true }
      )
      this.partitions.set(partitionId, partition)
      // Initialize partition metadata
      this.partitionMetadata.set(partitionId, {
        id: partitionId,
        nodeCount: 0,
        strategy: this.config.partitionStrategy,
        created: new Date()
      })
    }
    // Add item to the selected partition
    await partition.addItem(item)
    // Update partition metadata
    const metadata = this.partitionMetadata.get(partitionId)!
    metadata.nodeCount = partition.size()
    // Update bounds for semantic strategy
    if (this.config.partitionStrategy === 'semantic') {
      this.updatePartitionBounds(partitionId, item.vector)
    }
    // Check if partition is getting too large and needs splitting
    if (metadata.nodeCount > this.config.maxNodesPerPartition * 1.2) {
      await this.splitPartition(partitionId)
    }
    return item.id
  }
  /**
   * Search across all partitions for nearest neighbors
   */
  public async search(
    queryVector: Vector,
    k: number = 10,
    searchScope?: {
      partitionIds?: string[]
      maxPartitions?: number
    }
  ): Promise<Array<[string, number]>> {
    if (this.partitions.size === 0) {
      return []
    }
    // Determine which partitions to search
    const partitionsToSearch = await this.selectSearchPartitions(queryVector, searchScope)
    // Search partitions in parallel
    const searchPromises = partitionsToSearch.map(async (partitionId) => {
      const partition = this.partitions.get(partitionId)
      if (!partition) return []
      // Search with higher k to get better global results
      const partitionK = Math.min(k * 2, partition.size())
      return partition.search(queryVector, partitionK)
    })
    const partitionResults = await Promise.all(searchPromises)
    // Merge and sort results from all partitions
    const allResults: Array<[string, number]> = []
    for (const results of partitionResults) {
      allResults.push(...results)
    }
    // Sort by distance and return top k
    allResults.sort((a, b) => a[1] - b[1])
    return allResults.slice(0, k)
  }
  /**
   * Select the appropriate partition for a new item
   * Automatically chooses semantic partitioning when beneficial, falls back to hash
   */
  private async selectPartition(item: VectorDocument): Promise<string> {
    // Auto-tune semantic clusters based on current dataset size
    if (this.config.autoTuneSemanticClusters && this.config.partitionStrategy === 'semantic') {
      this.autoTuneSemanticClusters()
    }
    switch (this.config.partitionStrategy) {
      case 'semantic':
        return await this.semanticPartition(item.vector)
      case 'hash':
      default:
        return this.hashPartition(item.id)
    }
  }
  /**
   * Hash-based partitioning for even distribution
   */
  private hashPartition(id: string): string {
    const hash = this.simpleHash(id)
    const existingPartitions = Array.from(this.partitions.keys())
    // Find partition with space, or create new one
    for (const partitionId of existingPartitions) {
      const metadata = this.partitionMetadata.get(partitionId)
      if (metadata && metadata.nodeCount < this.config.maxNodesPerPartition) {
        return partitionId
      }
    }
    // Create new partition
    return `partition_${this.nextPartitionId++}`
  }
  /**
   * Semantic clustering partitioning
   */
  private async semanticPartition(vector: Vector): Promise<string> {
    // Find closest partition centroid
    let closestPartition = ''
    let minDistance = Infinity
    for (const [partitionId, metadata] of this.partitionMetadata.entries()) {
      if (metadata.bounds?.centroid) {
        const distance = this.distanceFunction(vector, metadata.bounds.centroid)
        if (distance < minDistance) {
          minDistance = distance
          closestPartition = partitionId
        }
      }
    }
    // If no suitable partition found or it's full, create new one
    if (!closestPartition || 
        this.partitionMetadata.get(closestPartition)!.nodeCount >= this.config.maxNodesPerPartition) {
      closestPartition = `semantic_${this.nextPartitionId++}`
    }
    return closestPartition
  }
  /**
   * Auto-tune semantic clusters based on dataset size and performance
   */
  private autoTuneSemanticClusters(): void {
    const totalNodes = this.size()
    const currentPartitions = this.partitions.size
    // Optimal clusters based on dataset size
    let optimalClusters = Math.max(4, Math.min(32, Math.floor(totalNodes / 10000)))
    // Adjust based on current partition performance
    if (currentPartitions > 0) {
      const avgNodesPerPartition = totalNodes / currentPartitions
      if (avgNodesPerPartition > this.config.maxNodesPerPartition * 0.8) {
        // Partitions are getting full, increase clusters
        optimalClusters = Math.min(32, this.config.semanticClusters! + 2)
      } else if (avgNodesPerPartition < this.config.maxNodesPerPartition * 0.3 && currentPartitions > 4) {
        // Partitions are underutilized, decrease clusters
        optimalClusters = Math.max(4, this.config.semanticClusters! - 1)
      }
    }
    if (optimalClusters !== this.config.semanticClusters) {
      console.log(`Auto-tuning semantic clusters: ${this.config.semanticClusters} → ${optimalClusters}`)
      this.config.semanticClusters = optimalClusters
    }
  }
  /**
   * Select which partitions to search based on query
   */
  private async selectSearchPartitions(
    queryVector: Vector,
    searchScope?: {
      partitionIds?: string[]
      maxPartitions?: number
    }
  ): Promise<string[]> {
    if (searchScope?.partitionIds) {
      return searchScope.partitionIds.filter(id => this.partitions.has(id))
    }
    const maxPartitions = searchScope?.maxPartitions || Math.min(5, this.partitions.size)
    if (this.config.partitionStrategy === 'semantic') {
      // Search partitions with closest centroids
      const distances: Array<[string, number]> = []
      for (const [partitionId, metadata] of this.partitionMetadata.entries()) {
        if (metadata.bounds?.centroid) {
          const distance = this.distanceFunction(queryVector, metadata.bounds.centroid)
          distances.push([partitionId, distance])
        }
      }
      distances.sort((a, b) => a[1] - b[1])
      return distances.slice(0, maxPartitions).map(([id]) => id)
    }
    // For other strategies, search all partitions or random subset
    const allPartitionIds = Array.from(this.partitions.keys())
    if (allPartitionIds.length <= maxPartitions) {
      return allPartitionIds
    }
    // Return random subset
    const shuffled = [...allPartitionIds].sort(() => Math.random() - 0.5)
    return shuffled.slice(0, maxPartitions)
  }
  /**
   * Update partition bounds for semantic clustering
   */
  private updatePartitionBounds(partitionId: string, vector: Vector): void {
    const metadata = this.partitionMetadata.get(partitionId)!
    if (!metadata.bounds) {
      metadata.bounds = {
        centroid: [...vector],
        radius: 0
      }
      return
    }
    // Update centroid using incremental mean
    const { centroid } = metadata.bounds
    const nodeCount = metadata.nodeCount
    for (let i = 0; i < centroid.length; i++) {
      centroid[i] = (centroid[i] * (nodeCount - 1) + vector[i]) / nodeCount
    }
    // Update radius
    const distance = this.distanceFunction(vector, centroid)
    metadata.bounds.radius = Math.max(metadata.bounds.radius, distance)
  }
  /**
   * Split an overgrown partition into smaller partitions
   */
  private async splitPartition(partitionId: string): Promise<void> {
    const partition = this.partitions.get(partitionId)
    if (!partition) return
    console.log(`Splitting partition ${partitionId} with ${partition.size()} nodes`)
    // For now, we'll implement a simple strategy
    // In a full implementation, you'd want to analyze the data distribution
    // and create more intelligent splits
    // This is a placeholder - actual implementation would require
    // accessing the internal nodes of the HNSW index
  }
  /**
   * Simple hash function for consistent partitioning
   */
  private simpleHash(str: string): number {
    let hash = 0
    for (let i = 0; i < str.length; i++) {
      const char = str.charCodeAt(i)
      hash = ((hash << 5) - hash) + char
      hash = hash & hash // Convert to 32-bit integer
    }
    return Math.abs(hash)
  }
  /**
   * Get partition statistics
   */
  public getPartitionStats(): {
    totalPartitions: number
    totalNodes: number
    averageNodesPerPartition: number
    partitionDetails: PartitionMetadata[]
  } {
    const partitionDetails = Array.from(this.partitionMetadata.values())
    const totalNodes = partitionDetails.reduce((sum, p) => sum + p.nodeCount, 0)
    return {
      totalPartitions: partitionDetails.length,
      totalNodes,
      averageNodesPerPartition: totalNodes / partitionDetails.length || 0,
      partitionDetails
    }
  }
  /**
   * Remove an item from the index
   */
  public async removeItem(id: string): Promise<boolean> {
    // Find which partition contains this item
    for (const [partitionId, partition] of this.partitions.entries()) {
      if (await partition.removeItem(id)) {
        // Update metadata
        const metadata = this.partitionMetadata.get(partitionId)!
        metadata.nodeCount = partition.size()
        return true
      }
    }
    return false
  }
  /**
   * Clear all partitions
   */
  public clear(): void {
    for (const partition of this.partitions.values()) {
      partition.clear()
    }
    this.partitions.clear()
    this.partitionMetadata.clear()
    this.nextPartitionId = 0
  }
  /**
   * Get total size across all partitions
   */
  public size(): number {
    return Array.from(this.partitions.values()).reduce((sum, partition) => sum + partition.size(), 0)
  }
 }
--- a/src/hnsw/scaledHNSWSystem.ts
+++ b/src/hnsw/scaledHNSWSystem.ts
@ -1,745 +0,0 @@
 /**
 * Scaled HNSW System - Integration of All Optimization Strategies
 * Production-ready system for handling millions of vectors with sub-second search
 */
 import { Vector, VectorDocument, HNSWConfig } from '../coreTypes.js'
 import { PartitionedHNSWIndex, PartitionConfig } from './partitionedHNSWIndex.js'
 import { OptimizedHNSWIndex, OptimizedHNSWConfig } from './optimizedHNSWIndex.js'
 import { DistributedSearchSystem, SearchStrategy } from './distributedSearch.js'
 import { EnhancedCacheManager } from '../storage/enhancedCacheManager.js'
 import { BatchS3Operations } from '../storage/adapters/batchS3Operations.js'
 import { ReadOnlyOptimizations } from '../storage/readOnlyOptimizations.js'
 import { euclideanDistance } from '../utils/index.js'
 import { autoConfigureBrainy, AutoConfiguration } from '../utils/autoConfiguration.js'
 export interface ScaledHNSWConfig {
  // Required: Basic dataset expectations (can be auto-detected if not provided)
  expectedDatasetSize?: number // Auto-detected if not provided
  maxMemoryUsage?: number // Auto-detected based on environment
  targetSearchLatency?: number // Auto-configured based on environment
  // Storage configuration (optional - auto-detects S3 availability)
  s3Config?: {
    bucketName: string
    region: string
    endpoint?: string
    accessKeyId?: string // Falls back to env vars
    secretAccessKey?: string // Falls back to env vars
  }
  // Auto-configuration options
  autoConfigureEnvironment?: boolean // Default: true
  learningEnabled?: boolean // Default: true - adapts to performance
  // Manual overrides (optional - auto-configured if not provided)
  enablePartitioning?: boolean
  enableCompression?: boolean
  enableDistributedSearch?: boolean
  enablePredictiveCaching?: boolean
  // Advanced manual tuning (optional)
  partitionConfig?: Partial<PartitionConfig>
  hnswConfig?: Partial<OptimizedHNSWConfig>
  readOnlyMode?: boolean
 }
 /**
 * High-performance HNSW system with all optimizations integrated
 * Handles datasets from thousands to millions of vectors
 */
 export class ScaledHNSWSystem {
  private config: ScaledHNSWConfig & {
    expectedDatasetSize: number
    maxMemoryUsage: number
    targetSearchLatency: number
    autoConfigureEnvironment: boolean
    learningEnabled: boolean
    enablePartitioning: boolean
    enableCompression: boolean
    enableDistributedSearch: boolean
    enablePredictiveCaching: boolean
    readOnlyMode: boolean
  }
  private autoConfig: AutoConfiguration
  private partitionedIndex?: PartitionedHNSWIndex
  private distributedSearch?: DistributedSearchSystem
  private cacheManager?: EnhancedCacheManager<any>
  private batchOperations?: BatchS3Operations
  private readOnlyOptimizations?: ReadOnlyOptimizations
  // Performance monitoring and learning
  private performanceMetrics = {
    totalSearches: 0,
    averageSearchTime: 0,
    cacheHitRate: 0,
    compressionRatio: 0,
    memoryUsage: 0,
    indexSize: 0,
    lastLearningUpdate: Date.now()
  }
  constructor(config: ScaledHNSWConfig = {}) {
    this.autoConfig = AutoConfiguration.getInstance()
    // Set basic defaults - these will be overridden by auto-configuration
    this.config = {
      expectedDatasetSize: 100000,
      maxMemoryUsage: 4 * 1024 * 1024 * 1024,
      targetSearchLatency: 150,
      autoConfigureEnvironment: true,
      learningEnabled: true,
      enablePartitioning: true,
      enableCompression: true,
      enableDistributedSearch: true,
      enablePredictiveCaching: true,
      readOnlyMode: false,
      ...config
    }
    this.initializeOptimizedSystem()
  }
  /**
   * Initialize the optimized system based on configuration
   */
  private async initializeOptimizedSystem(): Promise<void> {
    console.log('Initializing Scaled HNSW System with auto-configuration...')
    // Auto-configure if enabled
    if (this.config.autoConfigureEnvironment) {
      const autoConfigResult = await this.autoConfig.detectAndConfigure({
        expectedDataSize: this.config.expectedDatasetSize,
        s3Available: !!this.config.s3Config,
        memoryBudget: this.config.maxMemoryUsage
      })
      console.log(`Detected environment: ${autoConfigResult.environment}`)
      console.log(`Available memory: ${(autoConfigResult.availableMemory / 1024 / 1024 / 1024).toFixed(1)}GB`)
      console.log(`CPU cores: ${autoConfigResult.cpuCores}`)
      // Override config with auto-detected values
      this.config = {
        ...this.config,
        expectedDatasetSize: autoConfigResult.recommendedConfig.expectedDatasetSize,
        maxMemoryUsage: autoConfigResult.recommendedConfig.maxMemoryUsage,
        targetSearchLatency: autoConfigResult.recommendedConfig.targetSearchLatency,
        enablePartitioning: autoConfigResult.recommendedConfig.enablePartitioning,
        enableCompression: autoConfigResult.recommendedConfig.enableCompression,
        enableDistributedSearch: autoConfigResult.recommendedConfig.enableDistributedSearch,
        enablePredictiveCaching: autoConfigResult.recommendedConfig.enablePredictiveCaching
      }
    }
    // Determine optimal configuration
    const optimizedConfig = this.calculateOptimalConfiguration()
    // Initialize partitioned index with semantic partitioning as default
    if (this.config.enablePartitioning) {
      this.partitionedIndex = new PartitionedHNSWIndex(
        {
          ...optimizedConfig.partitionConfig,
          partitionStrategy: 'semantic', // Always use semantic for better performance
          autoTuneSemanticClusters: true // Enable auto-tuning
        },
        optimizedConfig.hnswConfig,
        euclideanDistance
      )
      console.log('✓ Partitioned index initialized with semantic clustering')
    }
    // Initialize distributed search system
    if (this.config.enableDistributedSearch && this.partitionedIndex) {
      this.distributedSearch = new DistributedSearchSystem({
        maxConcurrentSearches: optimizedConfig.maxConcurrentSearches,
        searchTimeout: this.config.targetSearchLatency * 5,
        adaptivePartitionSelection: true,
        loadBalancing: true
      })
      console.log('✓ Distributed search system initialized')
    }
    // Initialize batch S3 operations
    if (this.config.s3Config) {
      // Create S3 client from config
      const { S3Client } = await import('@aws-sdk/client-s3')
      const s3Client = new S3Client({
        region: this.config.s3Config.region || 'us-east-1',
        endpoint: this.config.s3Config.endpoint,
        credentials: this.config.s3Config.accessKeyId && this.config.s3Config.secretAccessKey ? {
          accessKeyId: this.config.s3Config.accessKeyId,
          secretAccessKey: this.config.s3Config.secretAccessKey
        } : undefined // Will use default credentials from environment
      })
      this.batchOperations = new BatchS3Operations(
        s3Client,
        this.config.s3Config.bucketName,
        {
          maxConcurrency: 50,
          useS3Select: this.config.expectedDatasetSize > 100000
        }
      )
      console.log('✓ Batch S3 operations initialized')
    }
    // Initialize enhanced caching
    if (this.config.enablePredictiveCaching) {
      this.cacheManager = new EnhancedCacheManager({
        hotCacheMaxSize: optimizedConfig.hotCacheSize,
        warmCacheMaxSize: optimizedConfig.warmCacheSize,
        prefetchEnabled: true,
        prefetchStrategy: 'hybrid' as any, // Type casting for enum compatibility
        prefetchBatchSize: 50
      })
      if (this.batchOperations) {
        this.cacheManager.setStorageAdapters(null as any, this.batchOperations)
      }
      console.log('✓ Enhanced cache manager initialized')
    }
    // Initialize read-only optimizations
    if (this.config.readOnlyMode && this.config.enableCompression) {
      this.readOnlyOptimizations = new ReadOnlyOptimizations({
        compression: {
          vectorCompression: 'quantization' as any,
          metadataCompression: 'gzip' as any,
          quantizationType: 'scalar' as any,
          quantizationBits: 8
        },
        segmentSize: optimizedConfig.segmentSize,
        memoryMapped: true,
        cacheIndexInMemory: optimizedConfig.cacheIndexInMemory
      })
      console.log('✓ Read-only optimizations initialized')
    }
    console.log('Scaled HNSW System ready for', this.config.expectedDatasetSize, 'vectors')
  }
  /**
   * Calculate optimal configuration based on dataset size and constraints
   */
  private calculateOptimalConfiguration(): {
    partitionConfig: PartitionConfig
    hnswConfig: OptimizedHNSWConfig
    hotCacheSize: number
    warmCacheSize: number
    maxConcurrentSearches: number
    segmentSize: number
    cacheIndexInMemory: boolean
  } {
    const size = this.config.expectedDatasetSize
    const memoryBudget = this.config.maxMemoryUsage
    let config: any = {}
    if (size <= 10000) {
      // Small dataset - optimize for speed
      config = {
        partitionConfig: {
          maxNodesPerPartition: 10000,
          partitionStrategy: 'hash' as const
        },
        hnswConfig: {
          M: 16,
          efConstruction: 200,
          efSearch: 50,
          targetSearchLatency: this.config.targetSearchLatency
        },
        hotCacheSize: 1000,
        warmCacheSize: 5000,
        maxConcurrentSearches: 4,
        segmentSize: 5000,
        cacheIndexInMemory: true
      }
    } else if (size <= 100000) {
      // Medium dataset - balance performance and memory
      config = {
        partitionConfig: {
          maxNodesPerPartition: 25000,
          partitionStrategy: 'semantic' as const,
          semanticClusters: 8
        },
        hnswConfig: {
          M: 24,
          efConstruction: 300,
          efSearch: 75,
          targetSearchLatency: this.config.targetSearchLatency,
          dynamicParameterTuning: true
        },
        hotCacheSize: 2000,
        warmCacheSize: 15000,
        maxConcurrentSearches: 8,
        segmentSize: 10000,
        cacheIndexInMemory: memoryBudget > 2 * 1024 * 1024 * 1024 // 2GB
      }
    } else if (size <= 1000000) {
      // Large dataset - optimize for scale
      config = {
        partitionConfig: {
          maxNodesPerPartition: 50000,
          partitionStrategy: 'semantic' as const,
          semanticClusters: 16
        },
        hnswConfig: {
          M: 32,
          efConstruction: 400,
          efSearch: 100,
          targetSearchLatency: this.config.targetSearchLatency,
          dynamicParameterTuning: true,
          memoryBudget: memoryBudget
        },
        hotCacheSize: 5000,
        warmCacheSize: 25000,
        maxConcurrentSearches: 12,
        segmentSize: 20000,
        cacheIndexInMemory: memoryBudget > 8 * 1024 * 1024 * 1024 // 8GB
      }
    } else {
      // Very large dataset - maximum optimization
      config = {
        partitionConfig: {
          maxNodesPerPartition: 100000,
          partitionStrategy: 'hybrid' as const,
          semanticClusters: 32
        },
        hnswConfig: {
          M: 48,
          efConstruction: 500,
          efSearch: 150,
          targetSearchLatency: this.config.targetSearchLatency,
          dynamicParameterTuning: true,
          memoryBudget: memoryBudget,
          diskCacheEnabled: true
        },
        hotCacheSize: 10000,
        warmCacheSize: 50000,
        maxConcurrentSearches: 20,
        segmentSize: 50000,
        cacheIndexInMemory: false // Too large for memory
      }
    }
    return config
  }
  /**
   * Add vector to the scaled system
   */
  public async addVector(item: VectorDocument): Promise<string> {
    if (!this.partitionedIndex) {
      throw new Error('System not properly initialized')
    }
    const startTime = Date.now()
    const result = await this.partitionedIndex.addItem(item)
    // Update performance metrics
    this.performanceMetrics.indexSize = this.partitionedIndex.size()
    return result
  }
  /**
   * Bulk insert vectors with optimizations
   */
  public async bulkInsert(items: VectorDocument[]): Promise<string[]> {
    if (!this.partitionedIndex) {
      throw new Error('System not properly initialized')
    }
    console.log(`Starting optimized bulk insert of ${items.length} vectors`)
    const startTime = Date.now()
    // Sort items for optimal insertion order
    const sortedItems = this.optimizeInsertionOrder(items)
    const results: string[] = []
    const batchSize = this.calculateOptimalBatchSize(items.length)
    // Process in batches
    for (let i = 0; i < sortedItems.length; i += batchSize) {
      const batch = sortedItems.slice(i, i + batchSize)
      for (const item of batch) {
        const id = await this.partitionedIndex.addItem(item)
        results.push(id)
      }
      // Progress logging
      if (i % (batchSize * 10) === 0) {
        const progress = ((i / sortedItems.length) * 100).toFixed(1)
        console.log(`Bulk insert progress: ${progress}%`)
      }
    }
    const totalTime = Date.now() - startTime
    console.log(`Bulk insert completed: ${results.length} vectors in ${totalTime}ms`)
    return results
  }
  /**
   * High-performance vector search with all optimizations
   */
  public async search(
    queryVector: Vector,
    k: number = 10,
    options: {
      strategy?: SearchStrategy
      useCache?: boolean
      maxPartitions?: number
    } = {}
  ): Promise<Array<[string, number]>> {
    const startTime = Date.now()
    try {
      let results: Array<[string, number]>
      if (this.distributedSearch && this.partitionedIndex) {
        // Use distributed search for optimal performance
        results = await this.distributedSearch.distributedSearch(
          this.partitionedIndex,
          queryVector,
          k,
          options.strategy || SearchStrategy.ADAPTIVE
        )
      } else if (this.partitionedIndex) {
        // Fall back to partitioned search
        results = await this.partitionedIndex.search(
          queryVector,
          k,
          { maxPartitions: options.maxPartitions }
        )
      } else {
        throw new Error('No search system available')
      }
      // Update performance metrics and learn from performance
      const searchTime = Date.now() - startTime
      this.updateSearchMetrics(searchTime, results.length)
      // Adaptive learning - adjust configuration based on performance
      if (this.config.learningEnabled && this.shouldTriggerLearning()) {
        await this.adaptivelyLearnFromPerformance()
      }
      return results
    } catch (error) {
      console.error('Search failed:', error)
      throw error
    }
  }
  /**
   * Get system performance metrics
   */
  public getPerformanceMetrics(): typeof this.performanceMetrics & {
    partitionStats?: any
    cacheStats?: any
    compressionStats?: any
    distributedSearchStats?: any
  } {
    const metrics = { ...this.performanceMetrics }
    // Add subsystem metrics
    if (this.partitionedIndex) {
      (metrics as any).partitionStats = this.partitionedIndex.getPartitionStats()
    }
    if (this.cacheManager) {
      (metrics as any).cacheStats = this.cacheManager.getStats()
    }
    if (this.readOnlyOptimizations) {
      (metrics as any).compressionStats = this.readOnlyOptimizations.getCompressionStats()
    }
    if (this.distributedSearch) {
      (metrics as any).distributedSearchStats = this.distributedSearch.getSearchStats()
    }
    return metrics
  }
  /**
   * Optimize insertion order for better index quality
   */
  private optimizeInsertionOrder(items: VectorDocument[]): VectorDocument[] {
    if (items.length < 1000) {
      return items // Not worth optimizing small batches
    }
    // Simple clustering-based approach for better HNSW construction
    // In production, you might use more sophisticated clustering
    return items.sort(() => Math.random() - 0.5)
  }
  /**
   * Calculate optimal batch size based on system resources
   */
  private calculateOptimalBatchSize(totalItems: number): number {
    const memoryBudget = this.config.maxMemoryUsage
    const estimatedItemSize = 1000 // Rough estimate per item in bytes
    const maxBatch = Math.floor(memoryBudget * 0.1 / estimatedItemSize)
    const targetBatch = Math.min(1000, Math.max(100, maxBatch))
    return Math.min(targetBatch, totalItems)
  }
  /**
   * Update search performance metrics
   */
  private updateSearchMetrics(searchTime: number, resultCount: number): void {
    this.performanceMetrics.totalSearches++
    this.performanceMetrics.averageSearchTime = 
      (this.performanceMetrics.averageSearchTime + searchTime) / 2
    // Update other metrics
    if (this.cacheManager) {
      const cacheStats = this.cacheManager.getStats()
      const totalOps = cacheStats.hotCacheHits + cacheStats.hotCacheMisses + 
                      cacheStats.warmCacheHits + cacheStats.warmCacheMisses
      this.performanceMetrics.cacheHitRate = totalOps > 0 ? 
        (cacheStats.hotCacheHits + cacheStats.warmCacheHits) / totalOps : 0
    }
    if (this.readOnlyOptimizations) {
      const compressionStats = this.readOnlyOptimizations.getCompressionStats()
      this.performanceMetrics.compressionRatio = compressionStats.compressionRatio
    }
    // Estimate memory usage
    this.performanceMetrics.memoryUsage = this.estimateMemoryUsage()
  }
  /**
   * Estimate current memory usage
   */
  private estimateMemoryUsage(): number {
    let totalMemory = 0
    if (this.partitionedIndex) {
      // Rough estimate: 1KB per vector
      totalMemory += this.partitionedIndex.size() * 1024
    }
    if (this.cacheManager) {
      const cacheStats = this.cacheManager.getStats()
      totalMemory += (cacheStats.hotCacheSize + cacheStats.warmCacheSize) * 1024
    }
    return totalMemory
  }
  /**
   * Generate performance report
   */
  public generatePerformanceReport(): string {
    const metrics = this.getPerformanceMetrics()
    return `
 === Scaled HNSW System Performance Report ===
 Dataset Configuration:
 - Expected Size: ${this.config.expectedDatasetSize.toLocaleString()} vectors
 - Current Size: ${metrics.indexSize.toLocaleString()} vectors
 - Memory Budget: ${(this.config.maxMemoryUsage / 1024 / 1024 / 1024).toFixed(1)}GB
 - Target Latency: ${this.config.targetSearchLatency}ms
 Performance Metrics:
 - Total Searches: ${metrics.totalSearches.toLocaleString()}
 - Average Search Time: ${metrics.averageSearchTime.toFixed(1)}ms
 - Cache Hit Rate: ${(metrics.cacheHitRate * 100).toFixed(1)}%
 - Memory Usage: ${(metrics.memoryUsage / 1024 / 1024).toFixed(1)}MB
 - Compression Ratio: ${metrics.compressionRatio ? (metrics.compressionRatio * 100).toFixed(1) + '%' : 'N/A'}
 System Status: ${this.getSystemStatus()}
    `.trim()
  }
  /**
   * Get overall system status
   */
  private getSystemStatus(): string {
    const metrics = this.getPerformanceMetrics()
    if (metrics.averageSearchTime <= this.config.targetSearchLatency) {
      return '✅ OPTIMAL'
    } else if (metrics.averageSearchTime <= this.config.targetSearchLatency * 2) {
      return '⚠️  ACCEPTABLE'
    } else {
      return '❌ NEEDS OPTIMIZATION'
    }
  }
  /**
   * Check if adaptive learning should be triggered
   */
  private shouldTriggerLearning(): boolean {
    const timeSinceLastLearning = Date.now() - this.performanceMetrics.lastLearningUpdate
    const minLearningInterval = 30000 // 30 seconds
    const minSearches = 20 // Minimum searches before learning
    return timeSinceLastLearning > minLearningInterval && 
           this.performanceMetrics.totalSearches > minSearches &&
           this.performanceMetrics.totalSearches % 50 === 0 // Learn every 50 searches
  }
  /**
   * Adaptively learn from performance and adjust configuration
   */
  private async adaptivelyLearnFromPerformance(): Promise<void> {
    try {
      const currentMetrics = {
        averageSearchTime: this.performanceMetrics.averageSearchTime,
        memoryUsage: this.performanceMetrics.memoryUsage,
        cacheHitRate: this.performanceMetrics.cacheHitRate,
        errorRate: 0 // Could be tracked separately
      }
      const adjustments = await this.autoConfig.learnFromPerformance(currentMetrics)
      if (Object.keys(adjustments).length > 0) {
        console.log('🧠 Adaptive learning: Adjusting configuration based on performance')
        // Apply learned adjustments
        let configChanged = false
        if (adjustments.enableDistributedSearch !== undefined && 
            adjustments.enableDistributedSearch !== this.config.enableDistributedSearch) {
          this.config.enableDistributedSearch = adjustments.enableDistributedSearch
          configChanged = true
        }
        if (adjustments.enableCompression !== undefined && 
            adjustments.enableCompression !== this.config.enableCompression) {
          this.config.enableCompression = adjustments.enableCompression
          configChanged = true
        }
        if (adjustments.enablePredictiveCaching !== undefined && 
            adjustments.enablePredictiveCaching !== this.config.enablePredictiveCaching) {
          this.config.enablePredictiveCaching = adjustments.enablePredictiveCaching
          configChanged = true
        }
        // Apply partition adjustments
        if (adjustments.maxNodesPerPartition && 
            this.partitionedIndex && 
            adjustments.maxNodesPerPartition !== this.partitionedIndex.getPartitionStats().averageNodesPerPartition) {
          // This would require rebuilding the index in a real implementation
          console.log(`Learning suggests partition size: ${adjustments.maxNodesPerPartition}`)
        }
        if (configChanged) {
          console.log('✅ Configuration updated based on performance learning')
        }
      }
      this.performanceMetrics.lastLearningUpdate = Date.now()
    } catch (error) {
      console.warn('Adaptive learning failed:', error)
    }
  }
  /**
   * Update dataset analysis for better auto-configuration
   */
  public async updateDatasetAnalysis(vectorCount: number, vectorDimension?: number): Promise<void> {
    if (this.config.autoConfigureEnvironment) {
      const analysis = {
        estimatedSize: vectorCount,
        vectorDimension,
        accessPatterns: this.inferAccessPatterns()
      }
      await this.autoConfig.adaptToDataset(analysis)
      console.log(`📊 Dataset analysis updated: ${vectorCount} vectors${vectorDimension ? `, ${vectorDimension}D` : ''}`)
    }
  }
  /**
   * Infer access patterns from current metrics
   */
  private inferAccessPatterns(): 'read-heavy' | 'write-heavy' | 'balanced' {
    // Simple heuristic - in practice, this would track read/write ratios
    if (this.performanceMetrics.totalSearches > 100) {
      return 'read-heavy'
    }
    return 'balanced'
  }
  /**
   * Cleanup system resources
   */
  public cleanup(): void {
    this.distributedSearch?.cleanup()
    this.cacheManager?.clear()
    this.readOnlyOptimizations?.cleanup()
    this.partitionedIndex?.clear()
    this.autoConfig.resetCache()
    console.log('Scaled HNSW System cleaned up')
  }
 }
 // Export convenience factory functions
 /**
 * Create a fully auto-configured Brainy system - minimal setup required!
 * Just provide S3 config if you want persistence beyond the current session
 */
 export function createAutoBrainy(s3Config?: {
  bucketName: string
  region?: string
  accessKeyId?: string
  secretAccessKey?: string
 }): ScaledHNSWSystem {
  return new ScaledHNSWSystem({
    s3Config: s3Config ? {
      bucketName: s3Config.bucketName,
      region: s3Config.region || 'us-east-1',
      accessKeyId: s3Config.accessKeyId,
      secretAccessKey: s3Config.secretAccessKey
    } : undefined,
    autoConfigureEnvironment: true,
    learningEnabled: true
  })
 }
 /**
 * Create a Brainy system optimized for specific scenarios
 */
 export async function createQuickBrainy(
  scenario: 'small' | 'medium' | 'large' | 'enterprise',
  s3Config?: { bucketName: string; region?: string }
 ): Promise<ScaledHNSWSystem> {
  const { getQuickSetup } = await import('../utils/autoConfiguration.js')
  const quickConfig = await getQuickSetup(scenario)
  return new ScaledHNSWSystem({
    ...quickConfig,
    s3Config: s3Config && quickConfig.s3Required ? {
      bucketName: s3Config.bucketName,
      region: s3Config.region || 'us-east-1',
      accessKeyId: process.env.AWS_ACCESS_KEY_ID,
      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
    } : undefined,
    autoConfigureEnvironment: true,
    learningEnabled: true
  })
 }
 /**
 * Legacy factory function - still works but consider using createAutoBrainy() instead
 */
 export function createScaledHNSWSystem(config: ScaledHNSWConfig = {}): ScaledHNSWSystem {
  return new ScaledHNSWSystem(config)
 }
--- a/src/index.ts
+++ b/src/index.ts
@ -346,14 +346,10 @@ import type {
  StorageAdapter
 } from './coreTypes.js'
-// Export HNSW index and optimized version
+// Export HNSW index
 import { HNSWIndex } from './hnsw/hnswIndex.js'
 import {
  HNSWIndexOptimized,
  HNSWOptimizedConfig
 } from './hnsw/hnswIndexOptimized.js'
-export { HNSWIndex, HNSWIndexOptimized }
+export { HNSWIndex }
 export type {
  Vector,
@ -365,7 +361,6 @@ export type {
  HNSWNoun,
  HNSWVerb,
  HNSWConfig,
  HNSWOptimizedConfig,
  StorageAdapter
 }
--- a/src/triple/TripleIntelligenceSystem.ts
+++ b/src/triple/TripleIntelligenceSystem.ts
@ -14,7 +14,6 @@
 */
 import { HNSWIndex } from '../hnsw/hnswIndex.js'
 import { HNSWIndexOptimized } from '../hnsw/hnswIndexOptimized.js'
 import { TypeAwareHNSWIndex } from '../hnsw/typeAwareHNSWIndex.js'
 import { MetadataIndexManager } from '../utils/metadataIndex.js'
 import { Vector } from '../coreTypes.js'
@ -233,7 +232,7 @@ class QueryPlanner {
 */
 export class TripleIntelligenceSystem {
  private metadataIndex: MetadataIndexManager
-  private hnswIndex: HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex
+  private hnswIndex: HNSWIndex | TypeAwareHNSWIndex
  private graphIndex: GraphAdjacencyIndex
  private metrics: PerformanceMetrics
  private planner: QueryPlanner
@ -242,7 +241,7 @@ export class TripleIntelligenceSystem {
  constructor(
    metadataIndex: MetadataIndexManager,
-    hnswIndex: HNSWIndex | HNSWIndexOptimized | TypeAwareHNSWIndex,
+    hnswIndex: HNSWIndex | TypeAwareHNSWIndex,
    graphIndex: GraphAdjacencyIndex,
    embedder: (text: string) => Promise<Vector>,
    storage: any