Initial commit: Brainy - Multi-Dimensional AI Database

Open source vector database with HNSW indexing, graph relationships,
and metadata facets. Features CLI with professional augmentation registry
integration for discovering extensions and capabilities.

dist/hnsw/distributedSearch.d.ts

@@ -0,0 +1,118 @@
+/**
+ * Distributed Search System for Large-Scale HNSW Indices
+ * Implements parallel search across multiple partitions and instances
+ */
+import { Vector } from '../coreTypes.js';
+import { PartitionedHNSWIndex } from './partitionedHNSWIndex.js';
+interface DistributedSearchConfig {
+    maxConcurrentSearches?: number;
+    searchTimeout?: number;
+    resultMergeStrategy?: 'distance' | 'score' | 'hybrid';
+    adaptivePartitionSelection?: boolean;
+    redundantSearches?: number;
+    loadBalancing?: boolean;
+}
+export declare enum SearchStrategy {
+    BROADCAST = "broadcast",// Search all partitions
+    SELECTIVE = "selective",// Search subset of partitions
+    ADAPTIVE = "adaptive",// Dynamically adjust based on results
+    HIERARCHICAL = "hierarchical"
+}
+interface SearchWorker {
+    id: string;
+    busy: boolean;
+    tasksCompleted: number;
+    averageTaskTime: number;
+    lastTaskTime: number;
+}
+/**
+ * Distributed search coordinator for large-scale vector search
+ */
+export declare class DistributedSearchSystem {
+    private config;
+    private searchWorkers;
+    private searchQueue;
+    private activeSearches;
+    private partitionStats;
+    private searchStats;
+    constructor(config?: Partial<DistributedSearchConfig>);
+    /**
+     * Execute distributed search across multiple partitions
+     */
+    distributedSearch(partitionedIndex: PartitionedHNSWIndex, queryVector: Vector, k: number, strategy?: SearchStrategy): Promise<Array<[string, number]>>;
+    /**
+     * Select partitions to search based on strategy
+     */
+    private selectPartitions;
+    /**
+     * Adaptive partition selection based on historical performance
+     */
+    private adaptivePartitionSelection;
+    /**
+     * Select top-performing partitions
+     */
+    private selectTopPartitions;
+    /**
+     * Hierarchical partition selection for very large datasets
+     */
+    private hierarchicalPartitionSelection;
+    /**
+     * Create search tasks for parallel execution
+     */
+    private createSearchTasks;
+    /**
+     * Execute searches in parallel across selected partitions
+     */
+    private executeParallelSearches;
+    /**
+     * Execute search on a single partition
+     */
+    private executePartitionSearch;
+    /**
+     * Determine if search should use worker thread
+     */
+    private shouldUseWorkerThread;
+    /**
+     * Execute search in worker thread
+     */
+    private executeInWorkerThread;
+    /**
+     * Get available worker from pool
+     */
+    private getAvailableWorker;
+    /**
+     * Merge search results from multiple partitions
+     */
+    private mergeSearchResults;
+    /**
+     * Get partition quality score
+     */
+    private getPartitionQuality;
+    /**
+     * Update search statistics
+     */
+    private updateSearchStats;
+    /**
+     * Initialize worker thread pool
+     */
+    private initializeWorkerPool;
+    /**
+     * Generate unique search ID
+     */
+    private generateSearchId;
+    /**
+     * Get search performance statistics
+     */
+    getSearchStats(): typeof this.searchStats & {
+        workerStats: SearchWorker[];
+        partitionStats: Array<{
+            id: string;
+            stats: any;
+        }>;
+    };
+    /**
+     * Cleanup resources
+     */
+    cleanup(): void;
+}
+export {};

dist/hnsw/distributedSearch.js

@@ -0,0 +1,452 @@
+/**
+ * Distributed Search System for Large-Scale HNSW Indices
+ * Implements parallel search across multiple partitions and instances
+ */
+import { executeInThread } from '../utils/workerUtils.js';
+// Search coordination strategies
+export var SearchStrategy;
+(function (SearchStrategy) {
+    SearchStrategy["BROADCAST"] = "broadcast";
+    SearchStrategy["SELECTIVE"] = "selective";
+    SearchStrategy["ADAPTIVE"] = "adaptive";
+    SearchStrategy["HIERARCHICAL"] = "hierarchical"; // Multi-level search
+})(SearchStrategy || (SearchStrategy = {}));
+/**
+ * Distributed search coordinator for large-scale vector search
+ */
+export class DistributedSearchSystem {
+    constructor(config = {}) {
+        this.searchWorkers = new Map();
+        this.searchQueue = [];
+        this.activeSearches = new Map();
+        this.partitionStats = new Map();
+        // Performance monitoring
+        this.searchStats = {
+            totalSearches: 0,
+            averageLatency: 0,
+            parallelEfficiency: 0,
+            cacheHitRate: 0,
+            partitionUtilization: new Map()
+        };
+        this.config = {
+            maxConcurrentSearches: 10,
+            searchTimeout: 30000, // 30 seconds
+            resultMergeStrategy: 'hybrid',
+            adaptivePartitionSelection: true,
+            redundantSearches: 0,
+            loadBalancing: true,
+            ...config
+        };
+        this.initializeWorkerPool();
+    }
+    /**
+     * Execute distributed search across multiple partitions
+     */
+    async distributedSearch(partitionedIndex, queryVector, k, strategy = SearchStrategy.ADAPTIVE) {
+        const searchId = this.generateSearchId();
+        const startTime = Date.now();
+        try {
+            // Select partitions to search based on strategy
+            const partitionsToSearch = await this.selectPartitions(partitionedIndex, queryVector, strategy);
+            // Create search tasks
+            const searchTasks = this.createSearchTasks(partitionsToSearch, queryVector, k, searchId);
+            // Execute searches in parallel
+            const searchResults = await this.executeParallelSearches(partitionedIndex, searchTasks);
+            // Merge results from all partitions
+            const mergedResults = this.mergeSearchResults(searchResults, k);
+            // Update statistics
+            this.updateSearchStats(searchId, startTime, searchResults);
+            return mergedResults;
+        }
+        catch (error) {
+            console.error(`Distributed search ${searchId} failed:`, error);
+            throw error;
+        }
+    }
+    /**
+     * Select partitions to search based on strategy
+     */
+    async selectPartitions(partitionedIndex, queryVector, strategy) {
+        const stats = partitionedIndex.getPartitionStats();
+        const allPartitionIds = stats.partitionDetails.map(p => p.id);
+        switch (strategy) {
+            case SearchStrategy.BROADCAST:
+                return allPartitionIds;
+            case SearchStrategy.SELECTIVE:
+                return this.selectTopPartitions(allPartitionIds, 3);
+            case SearchStrategy.ADAPTIVE:
+                return await this.adaptivePartitionSelection(allPartitionIds, queryVector);
+            case SearchStrategy.HIERARCHICAL:
+                return this.hierarchicalPartitionSelection(allPartitionIds);
+            default:
+                return allPartitionIds;
+        }
+    }
+    /**
+     * Adaptive partition selection based on historical performance
+     */
+    async adaptivePartitionSelection(partitionIds, queryVector) {
+        const candidates = [];
+        for (const partitionId of partitionIds) {
+            const stats = this.partitionStats.get(partitionId);
+            let score = 1.0;
+            if (stats) {
+                // Score based on performance metrics
+                const speedScore = 1000 / Math.max(stats.averageSearchTime, 1);
+                const loadScore = Math.max(0, 1 - stats.load);
+                const qualityScore = stats.quality;
+                const recencyScore = Math.max(0, 1 - (Date.now() - stats.lastUsed) / 3600000);
+                score = speedScore * 0.3 + loadScore * 0.25 + qualityScore * 0.3 + recencyScore * 0.15;
+            }
+            candidates.push({ id: partitionId, score });
+        }
+        // Sort by score and select top partitions
+        candidates.sort((a, b) => b.score - a.score);
+        const selectedCount = Math.min(Math.ceil(partitionIds.length * 0.6), 8);
+        return candidates.slice(0, selectedCount).map(c => c.id);
+    }
+    /**
+     * Select top-performing partitions
+     */
+    selectTopPartitions(partitionIds, count) {
+        const withStats = partitionIds.map(id => ({
+            id,
+            stats: this.partitionStats.get(id)
+        }));
+        // Sort by average search time (faster is better)
+        withStats.sort((a, b) => {
+            const timeA = a.stats?.averageSearchTime || 1000;
+            const timeB = b.stats?.averageSearchTime || 1000;
+            return timeA - timeB;
+        });
+        return withStats.slice(0, count).map(p => p.id);
+    }
+    /**
+     * Hierarchical partition selection for very large datasets
+     */
+    hierarchicalPartitionSelection(partitionIds) {
+        // First level: select representative partitions
+        const firstLevel = partitionIds.filter((_, index) => index % 3 === 0);
+        // Could implement a two-phase search here:
+        // 1. Quick search on representative partitions
+        // 2. Detailed search on promising partitions
+        return firstLevel;
+    }
+    /**
+     * Create search tasks for parallel execution
+     */
+    createSearchTasks(partitionIds, queryVector, k, searchId) {
+        const tasks = [];
+        for (let i = 0; i < partitionIds.length; i++) {
+            const partitionId = partitionIds[i];
+            const stats = this.partitionStats.get(partitionId);
+            // Calculate priority based on partition performance
+            const priority = stats ? (1000 - stats.averageSearchTime) : 500;
+            tasks.push({
+                partitionId,
+                queryVector: [...queryVector], // Clone vector
+                k: Math.max(k * 2, 20), // Search for more results per partition
+                searchId,
+                priority
+            });
+            // Add redundant searches if configured
+            if (this.config.redundantSearches > 0 && i < this.config.redundantSearches) {
+                tasks.push({
+                    partitionId,
+                    queryVector: [...queryVector],
+                    k: Math.max(k * 2, 20),
+                    searchId: `${searchId}_redundant_${i}`,
+                    priority: priority - 100 // Lower priority for redundant searches
+                });
+            }
+        }
+        // Sort tasks by priority
+        tasks.sort((a, b) => b.priority - a.priority);
+        return tasks;
+    }
+    /**
+     * Execute searches in parallel across selected partitions
+     */
+    async executeParallelSearches(partitionedIndex, searchTasks) {
+        const results = [];
+        const semaphore = new Semaphore(this.config.maxConcurrentSearches);
+        // Execute tasks with controlled concurrency
+        const taskPromises = searchTasks.map(async (task) => {
+            await semaphore.acquire();
+            try {
+                const startTime = Date.now();
+                // Execute search with timeout
+                const searchPromise = this.executePartitionSearch(partitionedIndex, task);
+                const timeoutPromise = new Promise((_, reject) => {
+                    setTimeout(() => reject(new Error('Search timeout')), this.config.searchTimeout);
+                });
+                const result = await Promise.race([searchPromise, timeoutPromise]);
+                result.searchTime = Date.now() - startTime;
+                return result;
+            }
+            catch (error) {
+                return {
+                    partitionId: task.partitionId,
+                    results: [],
+                    searchTime: this.config.searchTimeout,
+                    nodesVisited: 0,
+                    error: error
+                };
+            }
+            finally {
+                semaphore.release();
+            }
+        });
+        // Wait for all searches to complete
+        const taskResults = await Promise.allSettled(taskPromises);
+        for (const result of taskResults) {
+            if (result.status === 'fulfilled') {
+                results.push(result.value);
+            }
+        }
+        return results;
+    }
+    /**
+     * Execute search on a single partition
+     */
+    async executePartitionSearch(partitionedIndex, task) {
+        try {
+            // Use thread pool for compute-intensive operations
+            if (this.shouldUseWorkerThread(task)) {
+                return await this.executeInWorkerThread(partitionedIndex, task);
+            }
+            // Execute search directly
+            const results = await partitionedIndex.search(task.queryVector, task.k, { partitionIds: [task.partitionId] });
+            return {
+                partitionId: task.partitionId,
+                results,
+                searchTime: 0, // Will be set by caller
+                nodesVisited: results.length // Approximation
+            };
+        }
+        catch (error) {
+            throw new Error(`Partition search failed: ${error}`);
+        }
+    }
+    /**
+     * Determine if search should use worker thread
+     */
+    shouldUseWorkerThread(task) {
+        // Use worker threads for high-dimensional vectors or large k
+        return task.queryVector.length > 512 || task.k > 100;
+    }
+    /**
+     * Execute search in worker thread
+     */
+    async executeInWorkerThread(partitionedIndex, task) {
+        const worker = this.getAvailableWorker();
+        if (!worker) {
+            // No available workers, execute synchronously
+            return this.executePartitionSearch(partitionedIndex, task);
+        }
+        try {
+            worker.busy = true;
+            const startTime = Date.now();
+            // Execute in thread (simplified - would need proper worker setup)
+            const searchFunction = `
+        return partitionedIndex.search(
+          task.queryVector,
+          task.k,
+          { partitionIds: [task.partitionId] }
+        )
+      `;
+            const results = await executeInThread(searchFunction, {
+                queryVector: task.queryVector,
+                k: task.k,
+                partitionId: task.partitionId
+            });
+            const searchTime = Date.now() - startTime;
+            worker.averageTaskTime = (worker.averageTaskTime + searchTime) / 2;
+            worker.tasksCompleted++;
+            return {
+                partitionId: task.partitionId,
+                results: results || [],
+                searchTime,
+                nodesVisited: results ? results.length : 0
+            };
+        }
+        finally {
+            worker.busy = false;
+            worker.lastTaskTime = Date.now();
+        }
+    }
+    /**
+     * Get available worker from pool
+     */
+    getAvailableWorker() {
+        for (const worker of this.searchWorkers.values()) {
+            if (!worker.busy) {
+                return worker;
+            }
+        }
+        return null;
+    }
+    /**
+     * Merge search results from multiple partitions
+     */
+    mergeSearchResults(partitionResults, k) {
+        const allResults = [];
+        const seenIds = new Set();
+        // Collect all unique results
+        for (const partitionResult of partitionResults) {
+            if (partitionResult.error) {
+                console.warn(`Partition ${partitionResult.partitionId} failed:`, partitionResult.error);
+                continue;
+            }
+            for (const [id, distance] of partitionResult.results) {
+                if (!seenIds.has(id)) {
+                    allResults.push([id, distance]);
+                    seenIds.add(id);
+                }
+            }
+        }
+        // Sort and return top k results
+        switch (this.config.resultMergeStrategy) {
+            case 'distance':
+                allResults.sort((a, b) => a[1] - b[1]);
+                break;
+            case 'score':
+                // Convert distance to score (1 / (1 + distance))
+                allResults.sort((a, b) => {
+                    const scoreA = 1 / (1 + a[1]);
+                    const scoreB = 1 / (1 + b[1]);
+                    return scoreB - scoreA;
+                });
+                break;
+            case 'hybrid':
+                // Weighted combination of distance and partition quality
+                allResults.sort((a, b) => {
+                    const qualityWeightA = this.getPartitionQuality(a[0]);
+                    const qualityWeightB = this.getPartitionQuality(b[0]);
+                    const adjustedDistanceA = a[1] / (qualityWeightA + 0.1);
+                    const adjustedDistanceB = b[1] / (qualityWeightB + 0.1);
+                    return adjustedDistanceA - adjustedDistanceB;
+                });
+                break;
+        }
+        return allResults.slice(0, k);
+    }
+    /**
+     * Get partition quality score
+     */
+    getPartitionQuality(nodeId) {
+        // This would require knowing which partition a node came from
+        // For now, return a default quality score
+        return 1.0;
+    }
+    /**
+     * Update search statistics
+     */
+    updateSearchStats(searchId, startTime, results) {
+        const totalTime = Date.now() - startTime;
+        const successfulSearches = results.filter(r => !r.error);
+        // Update global stats
+        this.searchStats.totalSearches++;
+        this.searchStats.averageLatency =
+            (this.searchStats.averageLatency + totalTime) / 2;
+        // Calculate parallel efficiency
+        const totalPartitionTime = results.reduce((sum, r) => sum + r.searchTime, 0);
+        this.searchStats.parallelEfficiency =
+            totalPartitionTime > 0 ? totalTime / totalPartitionTime : 0;
+        // Update partition statistics
+        for (const result of successfulSearches) {
+            let stats = this.partitionStats.get(result.partitionId);
+            if (!stats) {
+                stats = {
+                    averageSearchTime: result.searchTime,
+                    load: 0,
+                    quality: 1.0,
+                    lastUsed: Date.now()
+                };
+            }
+            else {
+                stats.averageSearchTime = (stats.averageSearchTime + result.searchTime) / 2;
+                stats.lastUsed = Date.now();
+            }
+            this.partitionStats.set(result.partitionId, stats);
+            this.searchStats.partitionUtilization.set(result.partitionId, (this.searchStats.partitionUtilization.get(result.partitionId) || 0) + 1);
+        }
+    }
+    /**
+     * Initialize worker thread pool
+     */
+    initializeWorkerPool() {
+        const workerCount = Math.min(navigator.hardwareConcurrency || 4, 8);
+        for (let i = 0; i < workerCount; i++) {
+            const worker = {
+                id: `worker_${i}`,
+                busy: false,
+                tasksCompleted: 0,
+                averageTaskTime: 0,
+                lastTaskTime: 0
+            };
+            this.searchWorkers.set(worker.id, worker);
+        }
+        console.log(`Initialized worker pool with ${workerCount} workers`);
+    }
+    /**
+     * Generate unique search ID
+     */
+    generateSearchId() {
+        return `search_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+    }
+    /**
+     * Get search performance statistics
+     */
+    getSearchStats() {
+        return {
+            ...this.searchStats,
+            workerStats: Array.from(this.searchWorkers.values()),
+            partitionStats: Array.from(this.partitionStats.entries()).map(([id, stats]) => ({
+                id,
+                stats
+            }))
+        };
+    }
+    /**
+     * Cleanup resources
+     */
+    cleanup() {
+        // Clear active searches
+        this.activeSearches.clear();
+        // Reset worker states
+        for (const worker of this.searchWorkers.values()) {
+            worker.busy = false;
+        }
+        // Clear statistics
+        this.partitionStats.clear();
+    }
+}
+/**
+ * Simple semaphore for concurrency control
+ */
+class Semaphore {
+    constructor(permits) {
+        this.waiting = [];
+        this.permits = permits;
+    }
+    async acquire() {
+        if (this.permits > 0) {
+            this.permits--;
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => {
+            this.waiting.push(resolve);
+        });
+    }
+    release() {
+        if (this.waiting.length > 0) {
+            const resolve = this.waiting.shift();
+            resolve();
+        }
+        else {
+            this.permits++;
+        }
+    }
+}
+//# sourceMappingURL=distributedSearch.js.map

dist/hnsw/hnswIndex.d.ts

@@ -0,0 +1,121 @@
+/**
+ * HNSW (Hierarchical Navigable Small World) Index implementation
+ * Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
+ */
+import { DistanceFunction, HNSWConfig, HNSWNoun, Vector, VectorDocument } from '../coreTypes.js';
+export declare class HNSWIndex {
+    private nouns;
+    private entryPointId;
+    private maxLevel;
+    private config;
+    private distanceFunction;
+    private dimension;
+    private useParallelization;
+    constructor(config?: Partial<HNSWConfig>, distanceFunction?: DistanceFunction, options?: {
+        useParallelization?: boolean;
+    });
+    /**
+     * Set whether to use parallelization for performance-critical operations
+     */
+    setUseParallelization(useParallelization: boolean): void;
+    /**
+     * Get whether parallelization is enabled
+     */
+    getUseParallelization(): boolean;
+    /**
+     * Calculate distances between a query vector and multiple vectors in parallel
+     * This is used to optimize performance for search operations
+     * Uses optimized batch processing for optimal performance
+     *
+     * @param queryVector The query vector
+     * @param vectors Array of vectors to compare against
+     * @returns Array of distances
+     */
+    private calculateDistancesInParallel;
+    /**
+     * Add a vector to the index
+     */
+    addItem(item: VectorDocument): Promise<string>;
+    /**
+     * Search for nearest neighbors
+     */
+    search(queryVector: Vector, k?: number, filter?: (id: string) => Promise<boolean>): Promise<Array<[string, number]>>;
+    /**
+     * Remove an item from the index
+     */
+    removeItem(id: string): boolean;
+    /**
+     * Get all nouns in the index
+     * @deprecated Use getNounsPaginated() instead for better scalability
+     */
+    getNouns(): Map<string, HNSWNoun>;
+    /**
+     * Get nouns with pagination
+     * @param options Pagination options
+     * @returns Object containing paginated nouns and pagination info
+     */
+    getNounsPaginated(options?: {
+        offset?: number;
+        limit?: number;
+        filter?: (noun: HNSWNoun) => boolean;
+    }): {
+        items: Map<string, HNSWNoun>;
+        totalCount: number;
+        hasMore: boolean;
+    };
+    /**
+     * Clear the index
+     */
+    clear(): void;
+    /**
+     * Get the size of the index
+     */
+    size(): number;
+    /**
+     * Get the distance function used by the index
+     */
+    getDistanceFunction(): DistanceFunction;
+    /**
+     * Get the entry point ID
+     */
+    getEntryPointId(): string | null;
+    /**
+     * Get the maximum level
+     */
+    getMaxLevel(): number;
+    /**
+     * Get the dimension
+     */
+    getDimension(): number | null;
+    /**
+     * Get the configuration
+     */
+    getConfig(): HNSWConfig;
+    /**
+     * Get index health metrics
+     */
+    getIndexHealth(): {
+        averageConnections: number;
+        layerDistribution: number[];
+        maxLayer: number;
+        totalNodes: number;
+    };
+    /**
+     * Search within a specific layer
+     * Returns a map of noun IDs to distances, sorted by distance
+     */
+    private searchLayer;
+    /**
+     * Select M nearest neighbors from the candidate set
+     */
+    private selectNeighbors;
+    /**
+     * Ensure a noun doesn't have too many connections at a given level
+     */
+    private pruneConnections;
+    /**
+     * Generate a random level for a new noun
+     * Uses the same distribution as in the original HNSW paper
+     */
+    private getRandomLevel;
+}

dist/hnsw/hnswIndex.js

@@ -0,0 +1,621 @@
+/**
+ * HNSW (Hierarchical Navigable Small World) Index implementation
+ * Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
+ */
+import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js';
+// Default HNSW parameters
+const DEFAULT_CONFIG = {
+    M: 16, // Max number of connections per noun
+    efConstruction: 200, // Size of a dynamic candidate list during construction
+    efSearch: 50, // Size of a dynamic candidate list during search
+    ml: 16 // Max level
+};
+export class HNSWIndex {
+    constructor(config = {}, distanceFunction = euclideanDistance, options = {}) {
+        this.nouns = new Map();
+        this.entryPointId = null;
+        this.maxLevel = 0;
+        this.dimension = null;
+        this.useParallelization = true; // Whether to use parallelization for performance-critical operations
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        this.distanceFunction = distanceFunction;
+        this.useParallelization =
+            options.useParallelization !== undefined
+                ? options.useParallelization
+                : true;
+    }
+    /**
+     * Set whether to use parallelization for performance-critical operations
+     */
+    setUseParallelization(useParallelization) {
+        this.useParallelization = useParallelization;
+    }
+    /**
+     * Get whether parallelization is enabled
+     */
+    getUseParallelization() {
+        return this.useParallelization;
+    }
+    /**
+     * Calculate distances between a query vector and multiple vectors in parallel
+     * This is used to optimize performance for search operations
+     * Uses optimized batch processing for optimal performance
+     *
+     * @param queryVector The query vector
+     * @param vectors Array of vectors to compare against
+     * @returns Array of distances
+     */
+    async calculateDistancesInParallel(queryVector, vectors) {
+        // If parallelization is disabled or there are very few vectors, use sequential processing
+        if (!this.useParallelization || vectors.length < 10) {
+            return vectors.map((item) => ({
+                id: item.id,
+                distance: this.distanceFunction(queryVector, item.vector)
+            }));
+        }
+        try {
+            // Extract just the vectors from the input array
+            const vectorsOnly = vectors.map((item) => item.vector);
+            // Use optimized batch distance calculation
+            const distances = await calculateDistancesBatch(queryVector, vectorsOnly, this.distanceFunction);
+            // Map the distances back to their IDs
+            return vectors.map((item, index) => ({
+                id: item.id,
+                distance: distances[index]
+            }));
+        }
+        catch (error) {
+            console.error('Error in batch distance calculation, falling back to sequential processing:', error);
+            // Fall back to sequential processing if batch calculation fails
+            return vectors.map((item) => ({
+                id: item.id,
+                distance: this.distanceFunction(queryVector, item.vector)
+            }));
+        }
+    }
+    /**
+     * Add a vector to the index
+     */
+    async addItem(item) {
+        // Check if item is defined
+        if (!item) {
+            throw new Error('Item is undefined or null');
+        }
+        const { id, vector } = item;
+        // Check if vector is defined
+        if (!vector) {
+            throw new Error('Vector is undefined or null');
+        }
+        // Set dimension on first insert
+        if (this.dimension === null) {
+            this.dimension = vector.length;
+        }
+        else if (vector.length !== this.dimension) {
+            throw new Error(`Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`);
+        }
+        // Generate random level for this noun
+        const nounLevel = this.getRandomLevel();
+        // Create new noun
+        const noun = {
+            id,
+            vector,
+            connections: new Map(),
+            level: nounLevel
+        };
+        // Initialize empty connection sets for each level
+        for (let level = 0; level <= nounLevel; level++) {
+            noun.connections.set(level, new Set());
+        }
+        // If this is the first noun, make it the entry point
+        if (this.nouns.size === 0) {
+            this.entryPointId = id;
+            this.maxLevel = nounLevel;
+            this.nouns.set(id, noun);
+            return id;
+        }
+        // Find entry point
+        if (!this.entryPointId) {
+            console.error('Entry point ID is null');
+            // If there's no entry point, this is the first noun, so we should have returned earlier
+            // This is a safety check
+            this.entryPointId = id;
+            this.maxLevel = nounLevel;
+            this.nouns.set(id, noun);
+            return id;
+        }
+        const entryPoint = this.nouns.get(this.entryPointId);
+        if (!entryPoint) {
+            console.error(`Entry point with ID ${this.entryPointId} not found`);
+            // If the entry point doesn't exist, treat this as the first noun
+            this.entryPointId = id;
+            this.maxLevel = nounLevel;
+            this.nouns.set(id, noun);
+            return id;
+        }
+        let currObj = entryPoint;
+        let currDist = this.distanceFunction(vector, entryPoint.vector);
+        // Traverse the graph from top to bottom to find the closest noun
+        for (let level = this.maxLevel; level > nounLevel; level--) {
+            let changed = true;
+            while (changed) {
+                changed = false;
+                // Check all neighbors at current level
+                const connections = currObj.connections.get(level) || new Set();
+                for (const neighborId of connections) {
+                    const neighbor = this.nouns.get(neighborId);
+                    if (!neighbor) {
+                        // Skip neighbors that don't exist (expected during rapid additions/deletions)
+                        continue;
+                    }
+                    const distToNeighbor = this.distanceFunction(vector, neighbor.vector);
+                    if (distToNeighbor < currDist) {
+                        currDist = distToNeighbor;
+                        currObj = neighbor;
+                        changed = true;
+                    }
+                }
+            }
+        }
+        // For each level from nounLevel down to 0
+        for (let level = Math.min(nounLevel, this.maxLevel); level >= 0; level--) {
+            // Find ef nearest elements using greedy search
+            const nearestNouns = await this.searchLayer(vector, currObj, this.config.efConstruction, level);
+            // Select M nearest neighbors
+            const neighbors = this.selectNeighbors(vector, nearestNouns, this.config.M);
+            // Add bidirectional connections
+            for (const [neighborId, _] of neighbors) {
+                const neighbor = this.nouns.get(neighborId);
+                if (!neighbor) {
+                    // Skip neighbors that don't exist (expected during rapid additions/deletions)
+                    continue;
+                }
+                noun.connections.get(level).add(neighborId);
+                // Add reverse connection
+                if (!neighbor.connections.has(level)) {
+                    neighbor.connections.set(level, new Set());
+                }
+                neighbor.connections.get(level).add(id);
+                // Ensure neighbor doesn't have too many connections
+                if (neighbor.connections.get(level).size > this.config.M) {
+                    this.pruneConnections(neighbor, level);
+                }
+            }
+            // Update entry point for the next level
+            if (nearestNouns.size > 0) {
+                const [nearestId, nearestDist] = [...nearestNouns][0];
+                if (nearestDist < currDist) {
+                    currDist = nearestDist;
+                    const nearestNoun = this.nouns.get(nearestId);
+                    if (!nearestNoun) {
+                        console.error(`Nearest noun with ID ${nearestId} not found in addItem`);
+                        // Keep the current object as is
+                    }
+                    else {
+                        currObj = nearestNoun;
+                    }
+                }
+            }
+        }
+        // Update max level and entry point if needed
+        if (nounLevel > this.maxLevel) {
+            this.maxLevel = nounLevel;
+            this.entryPointId = id;
+        }
+        // Add noun to the index
+        this.nouns.set(id, noun);
+        return id;
+    }
+    /**
+     * Search for nearest neighbors
+     */
+    async search(queryVector, k = 10, filter) {
+        if (this.nouns.size === 0) {
+            return [];
+        }
+        // Check if query vector is defined
+        if (!queryVector) {
+            throw new Error('Query vector is undefined or null');
+        }
+        if (this.dimension !== null && queryVector.length !== this.dimension) {
+            throw new Error(`Query vector dimension mismatch: expected ${this.dimension}, got ${queryVector.length}`);
+        }
+        // Start from the entry point
+        if (!this.entryPointId) {
+            console.error('Entry point ID is null');
+            return [];
+        }
+        const entryPoint = this.nouns.get(this.entryPointId);
+        if (!entryPoint) {
+            console.error(`Entry point with ID ${this.entryPointId} not found`);
+            return [];
+        }
+        let currObj = entryPoint;
+        let currDist = this.distanceFunction(queryVector, currObj.vector);
+        // Traverse the graph from top to bottom to find the closest noun
+        for (let level = this.maxLevel; level > 0; level--) {
+            let changed = true;
+            while (changed) {
+                changed = false;
+                // Check all neighbors at current level
+                const connections = currObj.connections.get(level) || new Set();
+                // If we have enough connections, use parallel distance calculation
+                if (this.useParallelization && connections.size >= 10) {
+                    // Prepare vectors for parallel calculation
+                    const vectors = [];
+                    for (const neighborId of connections) {
+                        const neighbor = this.nouns.get(neighborId);
+                        if (!neighbor)
+                            continue;
+                        vectors.push({ id: neighborId, vector: neighbor.vector });
+                    }
+                    // Calculate distances in parallel
+                    const distances = await this.calculateDistancesInParallel(queryVector, vectors);
+                    // Find the closest neighbor
+                    for (const { id, distance } of distances) {
+                        if (distance < currDist) {
+                            currDist = distance;
+                            const neighbor = this.nouns.get(id);
+                            if (neighbor) {
+                                currObj = neighbor;
+                                changed = true;
+                            }
+                        }
+                    }
+                }
+                else {
+                    // Use sequential processing for small number of connections
+                    for (const neighborId of connections) {
+                        const neighbor = this.nouns.get(neighborId);
+                        if (!neighbor) {
+                            // Skip neighbors that don't exist (expected during rapid additions/deletions)
+                            continue;
+                        }
+                        const distToNeighbor = this.distanceFunction(queryVector, neighbor.vector);
+                        if (distToNeighbor < currDist) {
+                            currDist = distToNeighbor;
+                            currObj = neighbor;
+                            changed = true;
+                        }
+                    }
+                }
+            }
+        }
+        // Search at level 0 with ef = k
+        // If we have a filter, increase ef to compensate for filtered results
+        const ef = filter ? Math.max(this.config.efSearch * 3, k * 3) : Math.max(this.config.efSearch, k);
+        const nearestNouns = await this.searchLayer(queryVector, currObj, ef, 0, filter);
+        // Convert to array and sort by distance
+        return [...nearestNouns].slice(0, k);
+    }
+    /**
+     * Remove an item from the index
+     */
+    removeItem(id) {
+        if (!this.nouns.has(id)) {
+            return false;
+        }
+        const noun = this.nouns.get(id);
+        // Remove connections to this noun from all neighbors
+        for (const [level, connections] of noun.connections.entries()) {
+            for (const neighborId of connections) {
+                const neighbor = this.nouns.get(neighborId);
+                if (!neighbor) {
+                    // Skip neighbors that don't exist (expected during rapid additions/deletions)
+                    continue;
+                }
+                if (neighbor.connections.has(level)) {
+                    neighbor.connections.get(level).delete(id);
+                    // Prune connections after removing this noun to ensure consistency
+                    this.pruneConnections(neighbor, level);
+                }
+            }
+        }
+        // Also check all other nouns for references to this noun and remove them
+        for (const [nounId, otherNoun] of this.nouns.entries()) {
+            if (nounId === id)
+                continue; // Skip the noun being removed
+            for (const [level, connections] of otherNoun.connections.entries()) {
+                if (connections.has(id)) {
+                    connections.delete(id);
+                    // Prune connections after removing this reference
+                    this.pruneConnections(otherNoun, level);
+                }
+            }
+        }
+        // Remove the noun
+        this.nouns.delete(id);
+        // If we removed the entry point, find a new one
+        if (this.entryPointId === id) {
+            if (this.nouns.size === 0) {
+                this.entryPointId = null;
+                this.maxLevel = 0;
+            }
+            else {
+                // Find the noun with the highest level
+                let maxLevel = 0;
+                let newEntryPointId = null;
+                for (const [nounId, noun] of this.nouns.entries()) {
+                    if (noun.connections.size === 0)
+                        continue; // Skip nouns with no connections
+                    const nounLevel = Math.max(...noun.connections.keys());
+                    if (nounLevel >= maxLevel) {
+                        maxLevel = nounLevel;
+                        newEntryPointId = nounId;
+                    }
+                }
+                this.entryPointId = newEntryPointId;
+                this.maxLevel = maxLevel;
+            }
+        }
+        return true;
+    }
+    /**
+     * Get all nouns in the index
+     * @deprecated Use getNounsPaginated() instead for better scalability
+     */
+    getNouns() {
+        return new Map(this.nouns);
+    }
+    /**
+     * Get nouns with pagination
+     * @param options Pagination options
+     * @returns Object containing paginated nouns and pagination info
+     */
+    getNounsPaginated(options = {}) {
+        const offset = options.offset || 0;
+        const limit = options.limit || 100;
+        const filter = options.filter || (() => true);
+        // Get all noun entries
+        const entries = [...this.nouns.entries()];
+        // Apply filter if provided
+        const filteredEntries = entries.filter(([_, noun]) => filter(noun));
+        // Get total count after filtering
+        const totalCount = filteredEntries.length;
+        // Apply pagination
+        const paginatedEntries = filteredEntries.slice(offset, offset + limit);
+        // Check if there are more items
+        const hasMore = offset + limit < totalCount;
+        // Create a new map with the paginated entries
+        const items = new Map(paginatedEntries);
+        return {
+            items,
+            totalCount,
+            hasMore
+        };
+    }
+    /**
+     * Clear the index
+     */
+    clear() {
+        this.nouns.clear();
+        this.entryPointId = null;
+        this.maxLevel = 0;
+    }
+    /**
+     * Get the size of the index
+     */
+    size() {
+        return this.nouns.size;
+    }
+    /**
+     * Get the distance function used by the index
+     */
+    getDistanceFunction() {
+        return this.distanceFunction;
+    }
+    /**
+     * Get the entry point ID
+     */
+    getEntryPointId() {
+        return this.entryPointId;
+    }
+    /**
+     * Get the maximum level
+     */
+    getMaxLevel() {
+        return this.maxLevel;
+    }
+    /**
+     * Get the dimension
+     */
+    getDimension() {
+        return this.dimension;
+    }
+    /**
+     * Get the configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+    /**
+     * Get index health metrics
+     */
+    getIndexHealth() {
+        let totalConnections = 0;
+        const layerCounts = new Array(this.maxLevel + 1).fill(0);
+        // Count connections and layer distribution
+        this.nouns.forEach(noun => {
+            // Count connections at each layer
+            for (let level = 0; level <= noun.level; level++) {
+                totalConnections += noun.connections.get(level)?.size || 0;
+                layerCounts[level]++;
+            }
+        });
+        const totalNodes = this.nouns.size;
+        const averageConnections = totalNodes > 0 ? totalConnections / totalNodes : 0;
+        return {
+            averageConnections,
+            layerDistribution: layerCounts,
+            maxLayer: this.maxLevel,
+            totalNodes
+        };
+    }
+    /**
+     * Search within a specific layer
+     * Returns a map of noun IDs to distances, sorted by distance
+     */
+    async searchLayer(queryVector, entryPoint, ef, level, filter) {
+        // Set of visited nouns
+        const visited = new Set([entryPoint.id]);
+        // Check if entry point passes filter
+        const entryPointDistance = this.distanceFunction(queryVector, entryPoint.vector);
+        const entryPointPasses = filter ? await filter(entryPoint.id) : true;
+        // Priority queue of candidates (closest first)
+        const candidates = new Map();
+        candidates.set(entryPoint.id, entryPointDistance);
+        // Priority queue of nearest neighbors found so far (closest first)
+        const nearest = new Map();
+        if (entryPointPasses) {
+            nearest.set(entryPoint.id, entryPointDistance);
+        }
+        // While there are candidates to explore
+        while (candidates.size > 0) {
+            // Get closest candidate
+            const [closestId, closestDist] = [...candidates][0];
+            candidates.delete(closestId);
+            // If this candidate is farther than the farthest in our result set, we're done
+            const farthestInNearest = [...nearest][nearest.size - 1];
+            if (nearest.size >= ef && closestDist > farthestInNearest[1]) {
+                break;
+            }
+            // Explore neighbors of the closest candidate
+            const noun = this.nouns.get(closestId);
+            if (!noun) {
+                console.error(`Noun with ID ${closestId} not found in searchLayer`);
+                continue;
+            }
+            const connections = noun.connections.get(level) || new Set();
+            // If we have enough connections and parallelization is enabled, use parallel distance calculation
+            if (this.useParallelization && connections.size >= 10) {
+                // Collect unvisited neighbors
+                const unvisitedNeighbors = [];
+                for (const neighborId of connections) {
+                    if (!visited.has(neighborId)) {
+                        visited.add(neighborId);
+                        const neighbor = this.nouns.get(neighborId);
+                        if (!neighbor)
+                            continue;
+                        unvisitedNeighbors.push({ id: neighborId, vector: neighbor.vector });
+                    }
+                }
+                if (unvisitedNeighbors.length > 0) {
+                    // Calculate distances in parallel
+                    const distances = await this.calculateDistancesInParallel(queryVector, unvisitedNeighbors);
+                    // Process the results
+                    for (const { id, distance } of distances) {
+                        // Apply filter if provided
+                        const passes = filter ? await filter(id) : true;
+                        // Always add to candidates for graph traversal
+                        candidates.set(id, distance);
+                        // Only add to nearest if it passes the filter
+                        if (passes) {
+                            // If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
+                            if (nearest.size < ef || distance < farthestInNearest[1]) {
+                                nearest.set(id, distance);
+                                // If we have more than ef neighbors, remove the farthest one
+                                if (nearest.size > ef) {
+                                    const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1]);
+                                    nearest.clear();
+                                    for (let i = 0; i < ef; i++) {
+                                        nearest.set(sortedNearest[i][0], sortedNearest[i][1]);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            else {
+                // Use sequential processing for small number of connections
+                for (const neighborId of connections) {
+                    if (!visited.has(neighborId)) {
+                        visited.add(neighborId);
+                        const neighbor = this.nouns.get(neighborId);
+                        if (!neighbor) {
+                            // Skip neighbors that don't exist (expected during rapid additions/deletions)
+                            continue;
+                        }
+                        const distToNeighbor = this.distanceFunction(queryVector, neighbor.vector);
+                        // Apply filter if provided
+                        const passes = filter ? await filter(neighborId) : true;
+                        // Always add to candidates for graph traversal
+                        candidates.set(neighborId, distToNeighbor);
+                        // Only add to nearest if it passes the filter
+                        if (passes) {
+                            // If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
+                            if (nearest.size < ef || distToNeighbor < farthestInNearest[1]) {
+                                nearest.set(neighborId, distToNeighbor);
+                                // If we have more than ef neighbors, remove the farthest one
+                                if (nearest.size > ef) {
+                                    const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1]);
+                                    nearest.clear();
+                                    for (let i = 0; i < ef; i++) {
+                                        nearest.set(sortedNearest[i][0], sortedNearest[i][1]);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        // Sort nearest by distance
+        return new Map([...nearest].sort((a, b) => a[1] - b[1]));
+    }
+    /**
+     * Select M nearest neighbors from the candidate set
+     */
+    selectNeighbors(queryVector, candidates, M) {
+        if (candidates.size <= M) {
+            return candidates;
+        }
+        // Simple heuristic: just take the M closest
+        const sortedCandidates = [...candidates].sort((a, b) => a[1] - b[1]);
+        const result = new Map();
+        for (let i = 0; i < Math.min(M, sortedCandidates.length); i++) {
+            result.set(sortedCandidates[i][0], sortedCandidates[i][1]);
+        }
+        return result;
+    }
+    /**
+     * Ensure a noun doesn't have too many connections at a given level
+     */
+    pruneConnections(noun, level) {
+        const connections = noun.connections.get(level);
+        if (connections.size <= this.config.M) {
+            return;
+        }
+        // Calculate distances to all neighbors
+        const distances = new Map();
+        const validNeighborIds = new Set();
+        for (const neighborId of connections) {
+            const neighbor = this.nouns.get(neighborId);
+            if (!neighbor) {
+                // Skip neighbors that don't exist (expected during rapid additions/deletions)
+                continue;
+            }
+            // Only add valid neighbors to the distances map
+            distances.set(neighborId, this.distanceFunction(noun.vector, neighbor.vector));
+            validNeighborIds.add(neighborId);
+        }
+        // Only proceed if we have valid neighbors
+        if (distances.size === 0) {
+            // If no valid neighbors, clear connections at this level
+            noun.connections.set(level, new Set());
+            return;
+        }
+        // Select M closest neighbors from valid ones
+        const selectedNeighbors = this.selectNeighbors(noun.vector, distances, this.config.M);
+        // Update connections with only valid neighbors
+        noun.connections.set(level, new Set(selectedNeighbors.keys()));
+    }
+    /**
+     * Generate a random level for a new noun
+     * Uses the same distribution as in the original HNSW paper
+     */
+    getRandomLevel() {
+        const r = Math.random();
+        return Math.floor(-Math.log(r) * (1.0 / Math.log(this.config.M)));
+    }
+}
+//# sourceMappingURL=hnswIndex.js.map

dist/hnsw/hnswIndexOptimized.d.ts

@@ -0,0 +1,178 @@
+/**
+ * Optimized HNSW (Hierarchical Navigable Small World) Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
+import { HNSWIndex } from './hnswIndex.js';
+import { StorageAdapter } from '../coreTypes.js';
+export interface HNSWOptimizedConfig extends HNSWConfig {
+    memoryThreshold?: number;
+    productQuantization?: {
+        enabled: boolean;
+        numSubvectors?: number;
+        numCentroids?: number;
+    };
+    useDiskBasedIndex?: boolean;
+}
+/**
+ * Product Quantization implementation
+ * Reduces vector dimensionality by splitting vectors into subvectors
+ * and quantizing each subvector to the nearest centroid
+ */
+declare class ProductQuantizer {
+    private numSubvectors;
+    private numCentroids;
+    private centroids;
+    private subvectorSize;
+    private initialized;
+    private dimension;
+    constructor(numSubvectors?: number, numCentroids?: number);
+    /**
+     * Initialize the product quantizer with training data
+     * @param vectors Training vectors to use for learning centroids
+     */
+    train(vectors: Vector[]): void;
+    /**
+     * Quantize a vector using product quantization
+     * @param vector Vector to quantize
+     * @returns Array of centroid indices, one for each subvector
+     */
+    quantize(vector: Vector): number[];
+    /**
+     * Reconstruct a vector from its quantized representation
+     * @param codes Array of centroid indices
+     * @returns Reconstructed vector
+     */
+    reconstruct(codes: number[]): Vector;
+    /**
+     * Compute squared Euclidean distance between two vectors
+     * @param a First vector
+     * @param b Second vector
+     * @returns Squared Euclidean distance
+     */
+    private euclideanDistanceSquared;
+    /**
+     * Implement k-means++ algorithm to initialize centroids
+     * @param vectors Vectors to cluster
+     * @param k Number of clusters
+     * @returns Array of centroids
+     */
+    private kMeansPlusPlus;
+    /**
+     * Get the centroids for each subvector
+     * @returns Array of centroid arrays
+     */
+    getCentroids(): Vector[][];
+    /**
+     * Set the centroids for each subvector
+     * @param centroids Array of centroid arrays
+     */
+    setCentroids(centroids: Vector[][]): void;
+    /**
+     * Get the dimension of the vectors
+     * @returns Dimension
+     */
+    getDimension(): number;
+    /**
+     * Set the dimension of the vectors
+     * @param dimension Dimension
+     */
+    setDimension(dimension: number): void;
+}
+/**
+ * Optimized HNSW Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+export declare class HNSWIndexOptimized extends HNSWIndex {
+    private optimizedConfig;
+    private productQuantizer;
+    private storage;
+    private useDiskBasedIndex;
+    private useProductQuantization;
+    private quantizedVectors;
+    private memoryUsage;
+    private vectorCount;
+    private memoryUpdateLock;
+    constructor(config: Partial<HNSWOptimizedConfig> | undefined, distanceFunction: DistanceFunction, storage?: StorageAdapter | null);
+    /**
+     * Thread-safe method to update memory usage
+     * @param memoryDelta Change in memory usage (can be negative)
+     * @param vectorCountDelta Change in vector count (can be negative)
+     */
+    private updateMemoryUsage;
+    /**
+     * Thread-safe method to get current memory usage
+     * @returns Current memory usage and vector count
+     */
+    private getMemoryUsageAsync;
+    /**
+     * Add a vector to the index
+     * Uses product quantization if enabled and memory threshold is exceeded
+     */
+    addItem(item: VectorDocument): Promise<string>;
+    /**
+     * Search for nearest neighbors
+     * Uses product quantization if enabled
+     */
+    search(queryVector: Vector, k?: number): Promise<Array<[string, number]>>;
+    /**
+     * Remove an item from the index
+     */
+    removeItem(id: string): boolean;
+    /**
+     * Clear the index
+     */
+    clear(): Promise<void>;
+    /**
+     * Initialize product quantizer with existing vectors
+     */
+    private initializeProductQuantizer;
+    /**
+     * Get the product quantizer
+     * @returns Product quantizer or null if not enabled
+     */
+    getProductQuantizer(): ProductQuantizer | null;
+    /**
+     * Get the optimized configuration
+     * @returns Optimized configuration
+     */
+    getOptimizedConfig(): HNSWOptimizedConfig;
+    /**
+     * Get the estimated memory usage
+     * @returns Estimated memory usage in bytes
+     */
+    getMemoryUsage(): number;
+    /**
+     * Set the storage adapter
+     * @param storage Storage adapter
+     */
+    setStorage(storage: StorageAdapter): void;
+    /**
+     * Get the storage adapter
+     * @returns Storage adapter or null if not set
+     */
+    getStorage(): StorageAdapter | null;
+    /**
+     * Set whether to use disk-based index
+     * @param useDiskBasedIndex Whether to use disk-based index
+     */
+    setUseDiskBasedIndex(useDiskBasedIndex: boolean): void;
+    /**
+     * Get whether disk-based index is used
+     * @returns Whether disk-based index is used
+     */
+    getUseDiskBasedIndex(): boolean;
+    /**
+     * Set whether to use product quantization
+     * @param useProductQuantization Whether to use product quantization
+     */
+    setUseProductQuantization(useProductQuantization: boolean): void;
+    /**
+     * Get whether product quantization is used
+     * @returns Whether product quantization is used
+     */
+    getUseProductQuantization(): boolean;
+}
+export {};

dist/hnsw/hnswIndexOptimized.js

@@ -0,0 +1,471 @@
+/**
+ * Optimized HNSW (Hierarchical Navigable Small World) Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+import { HNSWIndex } from './hnswIndex.js';
+// Default configuration for the optimized HNSW index
+const DEFAULT_OPTIMIZED_CONFIG = {
+    M: 16,
+    efConstruction: 200,
+    efSearch: 50,
+    ml: 16,
+    memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
+    productQuantization: {
+        enabled: false,
+        numSubvectors: 16,
+        numCentroids: 256
+    },
+    useDiskBasedIndex: false
+};
+/**
+ * Product Quantization implementation
+ * Reduces vector dimensionality by splitting vectors into subvectors
+ * and quantizing each subvector to the nearest centroid
+ */
+class ProductQuantizer {
+    constructor(numSubvectors = 16, numCentroids = 256) {
+        this.centroids = [];
+        this.subvectorSize = 0;
+        this.initialized = false;
+        this.dimension = 0;
+        this.numSubvectors = numSubvectors;
+        this.numCentroids = numCentroids;
+    }
+    /**
+     * Initialize the product quantizer with training data
+     * @param vectors Training vectors to use for learning centroids
+     */
+    train(vectors) {
+        if (vectors.length === 0) {
+            throw new Error('Cannot train product quantizer with empty vector set');
+        }
+        this.dimension = vectors[0].length;
+        this.subvectorSize = Math.ceil(this.dimension / this.numSubvectors);
+        // Initialize centroids for each subvector
+        for (let i = 0; i < this.numSubvectors; i++) {
+            // Extract subvectors from training data
+            const subvectors = vectors.map((vector) => {
+                const start = i * this.subvectorSize;
+                const end = Math.min(start + this.subvectorSize, this.dimension);
+                return vector.slice(start, end);
+            });
+            // Initialize centroids for this subvector using k-means++
+            this.centroids[i] = this.kMeansPlusPlus(subvectors, this.numCentroids);
+        }
+        this.initialized = true;
+    }
+    /**
+     * Quantize a vector using product quantization
+     * @param vector Vector to quantize
+     * @returns Array of centroid indices, one for each subvector
+     */
+    quantize(vector) {
+        if (!this.initialized) {
+            throw new Error('Product quantizer not initialized. Call train() first.');
+        }
+        if (vector.length !== this.dimension) {
+            throw new Error(`Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`);
+        }
+        const codes = [];
+        // Quantize each subvector
+        for (let i = 0; i < this.numSubvectors; i++) {
+            const start = i * this.subvectorSize;
+            const end = Math.min(start + this.subvectorSize, this.dimension);
+            const subvector = vector.slice(start, end);
+            // Find nearest centroid
+            let minDist = Number.MAX_VALUE;
+            let nearestCentroidIndex = 0;
+            for (let j = 0; j < this.centroids[i].length; j++) {
+                const centroid = this.centroids[i][j];
+                const dist = this.euclideanDistanceSquared(subvector, centroid);
+                if (dist < minDist) {
+                    minDist = dist;
+                    nearestCentroidIndex = j;
+                }
+            }
+            codes.push(nearestCentroidIndex);
+        }
+        return codes;
+    }
+    /**
+     * Reconstruct a vector from its quantized representation
+     * @param codes Array of centroid indices
+     * @returns Reconstructed vector
+     */
+    reconstruct(codes) {
+        if (!this.initialized) {
+            throw new Error('Product quantizer not initialized. Call train() first.');
+        }
+        if (codes.length !== this.numSubvectors) {
+            throw new Error(`Code length mismatch: expected ${this.numSubvectors}, got ${codes.length}`);
+        }
+        const reconstructed = [];
+        // Reconstruct each subvector
+        for (let i = 0; i < this.numSubvectors; i++) {
+            const centroidIndex = codes[i];
+            const centroid = this.centroids[i][centroidIndex];
+            // Add centroid components to reconstructed vector
+            for (const component of centroid) {
+                reconstructed.push(component);
+            }
+        }
+        // Trim to original dimension if needed
+        return reconstructed.slice(0, this.dimension);
+    }
+    /**
+     * Compute squared Euclidean distance between two vectors
+     * @param a First vector
+     * @param b Second vector
+     * @returns Squared Euclidean distance
+     */
+    euclideanDistanceSquared(a, b) {
+        let sum = 0;
+        const length = Math.min(a.length, b.length);
+        for (let i = 0; i < length; i++) {
+            const diff = a[i] - b[i];
+            sum += diff * diff;
+        }
+        return sum;
+    }
+    /**
+     * Implement k-means++ algorithm to initialize centroids
+     * @param vectors Vectors to cluster
+     * @param k Number of clusters
+     * @returns Array of centroids
+     */
+    kMeansPlusPlus(vectors, k) {
+        if (vectors.length < k) {
+            // If we have fewer vectors than centroids, use the vectors as centroids
+            return [...vectors];
+        }
+        const centroids = [];
+        // Choose first centroid randomly
+        const firstIndex = Math.floor(Math.random() * vectors.length);
+        centroids.push([...vectors[firstIndex]]);
+        // Choose remaining centroids
+        for (let i = 1; i < k; i++) {
+            // Compute distances to nearest centroid for each vector
+            const distances = vectors.map((vector) => {
+                let minDist = Number.MAX_VALUE;
+                for (const centroid of centroids) {
+                    const dist = this.euclideanDistanceSquared(vector, centroid);
+                    minDist = Math.min(minDist, dist);
+                }
+                return minDist;
+            });
+            // Compute sum of distances
+            const distSum = distances.reduce((sum, dist) => sum + dist, 0);
+            // Choose next centroid with probability proportional to distance
+            let r = Math.random() * distSum;
+            let nextIndex = 0;
+            for (let j = 0; j < distances.length; j++) {
+                r -= distances[j];
+                if (r <= 0) {
+                    nextIndex = j;
+                    break;
+                }
+            }
+            centroids.push([...vectors[nextIndex]]);
+        }
+        return centroids;
+    }
+    /**
+     * Get the centroids for each subvector
+     * @returns Array of centroid arrays
+     */
+    getCentroids() {
+        return this.centroids;
+    }
+    /**
+     * Set the centroids for each subvector
+     * @param centroids Array of centroid arrays
+     */
+    setCentroids(centroids) {
+        this.centroids = centroids;
+        this.numSubvectors = centroids.length;
+        this.numCentroids = centroids[0].length;
+        this.initialized = true;
+    }
+    /**
+     * Get the dimension of the vectors
+     * @returns Dimension
+     */
+    getDimension() {
+        return this.dimension;
+    }
+    /**
+     * Set the dimension of the vectors
+     * @param dimension Dimension
+     */
+    setDimension(dimension) {
+        this.dimension = dimension;
+        this.subvectorSize = Math.ceil(dimension / this.numSubvectors);
+    }
+}
+/**
+ * Optimized HNSW Index implementation
+ * Extends the base HNSW implementation with support for large datasets
+ * Uses product quantization for dimensionality reduction and disk-based storage when needed
+ */
+export class HNSWIndexOptimized extends HNSWIndex {
+    constructor(config = {}, distanceFunction, storage = null) {
+        // Initialize base HNSW index with standard config
+        super(config, distanceFunction);
+        this.productQuantizer = null;
+        this.storage = null;
+        this.useDiskBasedIndex = false;
+        this.useProductQuantization = false;
+        this.quantizedVectors = new Map();
+        this.memoryUsage = 0;
+        this.vectorCount = 0;
+        // Thread safety for memory usage tracking
+        this.memoryUpdateLock = Promise.resolve();
+        // Set optimized config
+        this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config };
+        // Set storage adapter
+        this.storage = storage;
+        // Initialize product quantizer if enabled
+        if (this.optimizedConfig.productQuantization?.enabled) {
+            this.useProductQuantization = true;
+            this.productQuantizer = new ProductQuantizer(this.optimizedConfig.productQuantization.numSubvectors, this.optimizedConfig.productQuantization.numCentroids);
+        }
+        // Set disk-based index flag
+        this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false;
+    }
+    /**
+     * Thread-safe method to update memory usage
+     * @param memoryDelta Change in memory usage (can be negative)
+     * @param vectorCountDelta Change in vector count (can be negative)
+     */
+    async updateMemoryUsage(memoryDelta, vectorCountDelta) {
+        this.memoryUpdateLock = this.memoryUpdateLock.then(async () => {
+            this.memoryUsage = Math.max(0, this.memoryUsage + memoryDelta);
+            this.vectorCount = Math.max(0, this.vectorCount + vectorCountDelta);
+        });
+        await this.memoryUpdateLock;
+    }
+    /**
+     * Thread-safe method to get current memory usage
+     * @returns Current memory usage and vector count
+     */
+    async getMemoryUsageAsync() {
+        await this.memoryUpdateLock;
+        return {
+            memoryUsage: this.memoryUsage,
+            vectorCount: this.vectorCount
+        };
+    }
+    /**
+     * Add a vector to the index
+     * Uses product quantization if enabled and memory threshold is exceeded
+     */
+    async addItem(item) {
+        // Check if item is defined
+        if (!item) {
+            throw new Error('Item is undefined or null');
+        }
+        const { id, vector } = item;
+        // Check if vector is defined
+        if (!vector) {
+            throw new Error('Vector is undefined or null');
+        }
+        // Estimate memory usage for this vector
+        const vectorMemory = vector.length * 8; // 8 bytes per number (Float64)
+        const connectionsMemory = this.optimizedConfig.M * this.optimizedConfig.ml * 16; // Estimate for connections
+        const totalMemory = vectorMemory + connectionsMemory;
+        // Update memory usage estimate (thread-safe)
+        await this.updateMemoryUsage(totalMemory, 1);
+        // Check if we should switch to product quantization
+        const currentMemoryUsage = await this.getMemoryUsageAsync();
+        if (this.useProductQuantization &&
+            currentMemoryUsage.memoryUsage > this.optimizedConfig.memoryThreshold &&
+            this.productQuantizer &&
+            !this.productQuantizer.getDimension()) {
+            // Initialize product quantizer with existing vectors
+            this.initializeProductQuantizer();
+        }
+        // If product quantization is active, quantize the vector
+        if (this.useProductQuantization &&
+            this.productQuantizer &&
+            this.productQuantizer.getDimension() > 0) {
+            // Quantize the vector
+            const codes = this.productQuantizer.quantize(vector);
+            // Store the quantized vector
+            this.quantizedVectors.set(id, codes);
+            // Reconstruct the vector for indexing
+            const reconstructedVector = this.productQuantizer.reconstruct(codes);
+            // Add the reconstructed vector to the index
+            return await super.addItem({ id, vector: reconstructedVector });
+        }
+        // If disk-based index is active and storage is available, store the vector
+        if (this.useDiskBasedIndex && this.storage) {
+            // Create a noun object
+            const noun = {
+                id,
+                vector,
+                connections: new Map(),
+                level: 0
+            };
+            // Store the noun
+            this.storage.saveNoun(noun).catch((error) => {
+                console.error(`Failed to save noun ${id} to storage:`, error);
+            });
+        }
+        // Add the vector to the in-memory index
+        return await super.addItem(item);
+    }
+    /**
+     * Search for nearest neighbors
+     * Uses product quantization if enabled
+     */
+    async search(queryVector, k = 10) {
+        // Check if query vector is defined
+        if (!queryVector) {
+            throw new Error('Query vector is undefined or null');
+        }
+        // If product quantization is active, quantize the query vector
+        if (this.useProductQuantization &&
+            this.productQuantizer &&
+            this.productQuantizer.getDimension() > 0) {
+            // Quantize the query vector
+            const codes = this.productQuantizer.quantize(queryVector);
+            // Reconstruct the query vector
+            const reconstructedVector = this.productQuantizer.reconstruct(codes);
+            // Search with the reconstructed vector
+            return await super.search(reconstructedVector, k);
+        }
+        // Otherwise, use the standard search
+        return await super.search(queryVector, k);
+    }
+    /**
+     * Remove an item from the index
+     */
+    removeItem(id) {
+        // If product quantization is active, remove the quantized vector
+        if (this.useProductQuantization) {
+            this.quantizedVectors.delete(id);
+        }
+        // If disk-based index is active and storage is available, remove the vector from storage
+        if (this.useDiskBasedIndex && this.storage) {
+            this.storage.deleteNoun(id).catch((error) => {
+                console.error(`Failed to delete noun ${id} from storage:`, error);
+            });
+        }
+        // Update memory usage estimate (async operation, but don't block removal)
+        this.getMemoryUsageAsync().then((currentMemoryUsage) => {
+            if (currentMemoryUsage.vectorCount > 0) {
+                const memoryPerVector = currentMemoryUsage.memoryUsage / currentMemoryUsage.vectorCount;
+                this.updateMemoryUsage(-memoryPerVector, -1);
+            }
+        }).catch((error) => {
+            console.error('Failed to update memory usage after removal:', error);
+        });
+        // Remove the item from the in-memory index
+        return super.removeItem(id);
+    }
+    /**
+     * Clear the index
+     */
+    async clear() {
+        // Clear product quantization data
+        if (this.useProductQuantization) {
+            this.quantizedVectors.clear();
+            this.productQuantizer = new ProductQuantizer(this.optimizedConfig.productQuantization.numSubvectors, this.optimizedConfig.productQuantization.numCentroids);
+        }
+        // Reset memory usage (thread-safe)
+        const currentMemoryUsage = await this.getMemoryUsageAsync();
+        await this.updateMemoryUsage(-currentMemoryUsage.memoryUsage, -currentMemoryUsage.vectorCount);
+        // Clear the in-memory index
+        super.clear();
+    }
+    /**
+     * Initialize product quantizer with existing vectors
+     */
+    initializeProductQuantizer() {
+        if (!this.productQuantizer) {
+            return;
+        }
+        // Get all vectors from the index
+        const nouns = super.getNouns();
+        const vectors = [];
+        // Extract vectors
+        for (const [_, noun] of nouns) {
+            vectors.push(noun.vector);
+        }
+        // Train the product quantizer
+        if (vectors.length > 0) {
+            this.productQuantizer.train(vectors);
+            // Quantize all existing vectors
+            for (const [id, noun] of nouns) {
+                const codes = this.productQuantizer.quantize(noun.vector);
+                this.quantizedVectors.set(id, codes);
+            }
+            console.log(`Initialized product quantizer with ${vectors.length} vectors`);
+        }
+    }
+    /**
+     * Get the product quantizer
+     * @returns Product quantizer or null if not enabled
+     */
+    getProductQuantizer() {
+        return this.productQuantizer;
+    }
+    /**
+     * Get the optimized configuration
+     * @returns Optimized configuration
+     */
+    getOptimizedConfig() {
+        return { ...this.optimizedConfig };
+    }
+    /**
+     * Get the estimated memory usage
+     * @returns Estimated memory usage in bytes
+     */
+    getMemoryUsage() {
+        return this.memoryUsage;
+    }
+    /**
+     * Set the storage adapter
+     * @param storage Storage adapter
+     */
+    setStorage(storage) {
+        this.storage = storage;
+    }
+    /**
+     * Get the storage adapter
+     * @returns Storage adapter or null if not set
+     */
+    getStorage() {
+        return this.storage;
+    }
+    /**
+     * Set whether to use disk-based index
+     * @param useDiskBasedIndex Whether to use disk-based index
+     */
+    setUseDiskBasedIndex(useDiskBasedIndex) {
+        this.useDiskBasedIndex = useDiskBasedIndex;
+    }
+    /**
+     * Get whether disk-based index is used
+     * @returns Whether disk-based index is used
+     */
+    getUseDiskBasedIndex() {
+        return this.useDiskBasedIndex;
+    }
+    /**
+     * Set whether to use product quantization
+     * @param useProductQuantization Whether to use product quantization
+     */
+    setUseProductQuantization(useProductQuantization) {
+        this.useProductQuantization = useProductQuantization;
+    }
+    /**
+     * Get whether product quantization is used
+     * @returns Whether product quantization is used
+     */
+    getUseProductQuantization() {
+        return this.useProductQuantization;
+    }
+}
+//# sourceMappingURL=hnswIndexOptimized.js.map

dist/hnsw/optimizedHNSWIndex.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Optimized HNSW Index for Large-Scale Vector Search
+ * Implements dynamic parameter tuning and performance optimizations
+ */
+import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
+import { HNSWIndex } from './hnswIndex.js';
+export interface OptimizedHNSWConfig extends HNSWConfig {
+    dynamicParameterTuning?: boolean;
+    targetSearchLatency?: number;
+    targetRecall?: number;
+    maxNodes?: number;
+    memoryBudget?: number;
+    diskCacheEnabled?: boolean;
+    compressionEnabled?: boolean;
+    performanceTracking?: boolean;
+    adaptiveEfSearch?: boolean;
+    levelMultiplier?: number;
+    seedConnections?: number;
+    pruningStrategy?: 'simple' | 'diverse' | 'hybrid';
+}
+interface PerformanceMetrics {
+    averageSearchTime: number;
+    averageRecall: number;
+    memoryUsage: number;
+    indexSize: number;
+    apiCalls: number;
+    cacheHitRate: number;
+}
+interface DynamicParameters {
+    efSearch: number;
+    efConstruction: number;
+    M: number;
+    ml: number;
+}
+/**
+ * Optimized HNSW Index with dynamic parameter tuning for large datasets
+ */
+export declare class OptimizedHNSWIndex extends HNSWIndex {
+    private optimizedConfig;
+    private performanceMetrics;
+    private dynamicParams;
+    private searchHistory;
+    private parameterTuningInterval?;
+    constructor(config?: Partial<OptimizedHNSWConfig>, distanceFunction?: DistanceFunction);
+    /**
+     * Optimized search with dynamic parameter adjustment
+     */
+    search(queryVector: Vector, k?: number, filter?: (id: string) => Promise<boolean>): Promise<Array<[string, number]>>;
+    /**
+     * Dynamically adjust efSearch based on performance requirements
+     */
+    private adjustEfSearch;
+    /**
+     * Record search performance metrics
+     */
+    private recordSearchMetrics;
+    /**
+     * Check memory usage and trigger optimizations
+     */
+    private checkMemoryUsage;
+    /**
+     * Compress index to reduce memory usage (placeholder)
+     */
+    private compressIndex;
+    /**
+     * Start automatic parameter tuning
+     */
+    private startParameterTuning;
+    /**
+     * Automatic parameter tuning based on performance metrics
+     */
+    private tuneParameters;
+    /**
+     * Get optimized configuration recommendations for current dataset size
+     */
+    getOptimizedConfig(): OptimizedHNSWConfig;
+    /**
+     * Get current performance metrics
+     */
+    getPerformanceMetrics(): PerformanceMetrics & {
+        currentParams: DynamicParameters;
+        searchHistorySize: number;
+    };
+    /**
+     * Apply optimized bulk insertion strategy
+     */
+    bulkInsert(items: VectorDocument[]): Promise<string[]>;
+    /**
+     * Optimize insertion order to improve index quality
+     */
+    private optimizeInsertionOrder;
+    /**
+     * Cleanup resources
+     */
+    destroy(): void;
+}
+export {};

dist/hnsw/optimizedHNSWIndex.js

@@ -0,0 +1,313 @@
+/**
+ * Optimized HNSW Index for Large-Scale Vector Search
+ * Implements dynamic parameter tuning and performance optimizations
+ */
+import { HNSWIndex } from './hnswIndex.js';
+import { euclideanDistance } from '../utils/index.js';
+/**
+ * Optimized HNSW Index with dynamic parameter tuning for large datasets
+ */
+export class OptimizedHNSWIndex extends HNSWIndex {
+    constructor(config = {}, distanceFunction = euclideanDistance) {
+        // Set optimized defaults for large scale
+        const defaultConfig = {
+            M: 32, // Higher connectivity for better recall
+            efConstruction: 400, // Better build quality
+            efSearch: 100, // Dynamic - will be tuned
+            ml: 24, // Deeper hierarchy
+            useDiskBasedIndex: false, // Added missing property
+            dynamicParameterTuning: true,
+            targetSearchLatency: 100, // 100ms target
+            targetRecall: 0.95, // 95% recall target
+            maxNodes: 1000000, // 1M node limit
+            memoryBudget: 8 * 1024 * 1024 * 1024, // 8GB
+            diskCacheEnabled: true,
+            compressionEnabled: false, // Disabled by default for compatibility
+            performanceTracking: true,
+            adaptiveEfSearch: true,
+            levelMultiplier: 16,
+            seedConnections: 8,
+            pruningStrategy: 'hybrid'
+        };
+        const mergedConfig = { ...defaultConfig, ...config };
+        // Initialize parent with base config
+        super({
+            M: mergedConfig.M,
+            efConstruction: mergedConfig.efConstruction,
+            efSearch: mergedConfig.efSearch,
+            ml: mergedConfig.ml
+        }, distanceFunction, { useParallelization: true });
+        this.searchHistory = [];
+        this.optimizedConfig = mergedConfig;
+        // Initialize dynamic parameters
+        this.dynamicParams = {
+            efSearch: mergedConfig.efSearch,
+            efConstruction: mergedConfig.efConstruction,
+            M: mergedConfig.M,
+            ml: mergedConfig.ml
+        };
+        // Initialize performance metrics
+        this.performanceMetrics = {
+            averageSearchTime: 0,
+            averageRecall: 0,
+            memoryUsage: 0,
+            indexSize: 0,
+            apiCalls: 0,
+            cacheHitRate: 0
+        };
+        // Start parameter tuning if enabled
+        if (this.optimizedConfig.dynamicParameterTuning) {
+            this.startParameterTuning();
+        }
+    }
+    /**
+     * Optimized search with dynamic parameter adjustment
+     */
+    async search(queryVector, k = 10, filter) {
+        const startTime = Date.now();
+        // Adjust efSearch dynamically based on k and performance history
+        if (this.optimizedConfig.adaptiveEfSearch) {
+            this.adjustEfSearch(k);
+        }
+        // Check memory usage and trigger optimizations if needed
+        if (this.optimizedConfig.performanceTracking) {
+            this.checkMemoryUsage();
+        }
+        // Perform the search with current parameters
+        const originalConfig = this.getConfig();
+        // Temporarily update search parameters
+        const tempConfig = {
+            ...originalConfig,
+            efSearch: this.dynamicParams.efSearch
+        };
+        // Use the parent's search method with optimized parameters
+        let results;
+        try {
+            // This is a simplified approach - in practice, we'd need to modify
+            // the parent class to accept runtime parameter changes
+            results = await super.search(queryVector, k, filter);
+        }
+        catch (error) {
+            console.error('Optimized search failed, falling back to default:', error);
+            results = await super.search(queryVector, k, filter);
+        }
+        // Record performance metrics
+        const searchTime = Date.now() - startTime;
+        this.recordSearchMetrics(searchTime, k, results.length);
+        return results;
+    }
+    /**
+     * Dynamically adjust efSearch based on performance requirements
+     */
+    adjustEfSearch(k) {
+        const recentSearches = this.searchHistory.slice(-10);
+        if (recentSearches.length < 3) {
+            // Not enough data, use heuristic
+            this.dynamicParams.efSearch = Math.max(k * 2, 50);
+            return;
+        }
+        const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
+        const targetLatency = this.optimizedConfig.targetSearchLatency;
+        // Adjust efSearch based on latency performance
+        if (averageLatency > targetLatency * 1.2) {
+            // Too slow, reduce efSearch
+            this.dynamicParams.efSearch = Math.max(Math.floor(this.dynamicParams.efSearch * 0.9), k);
+        }
+        else if (averageLatency < targetLatency * 0.8) {
+            // Fast enough, can increase efSearch for better recall
+            this.dynamicParams.efSearch = Math.min(Math.floor(this.dynamicParams.efSearch * 1.1), 500 // Maximum efSearch
+            );
+        }
+        // Ensure efSearch is at least k
+        this.dynamicParams.efSearch = Math.max(this.dynamicParams.efSearch, k);
+    }
+    /**
+     * Record search performance metrics
+     */
+    recordSearchMetrics(latency, k, resultCount) {
+        if (!this.optimizedConfig.performanceTracking) {
+            return;
+        }
+        // Add to search history
+        this.searchHistory.push({
+            latency,
+            k,
+            timestamp: Date.now()
+        });
+        // Keep only recent history (last 100 searches)
+        if (this.searchHistory.length > 100) {
+            this.searchHistory.shift();
+        }
+        // Update performance metrics
+        const recentSearches = this.searchHistory.slice(-20);
+        this.performanceMetrics.averageSearchTime =
+            recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
+        // Estimate recall (simplified - would need ground truth for accurate measurement)
+        this.performanceMetrics.averageRecall = Math.min(resultCount / k, 1.0);
+    }
+    /**
+     * Check memory usage and trigger optimizations
+     */
+    checkMemoryUsage() {
+        // Estimate memory usage (simplified)
+        const estimatedMemory = this.size() * 1000; // Rough estimate per node
+        this.performanceMetrics.memoryUsage = estimatedMemory;
+        if (estimatedMemory > this.optimizedConfig.memoryBudget * 0.9) {
+            console.warn('Memory usage approaching limit, consider index partitioning');
+            // Could trigger automatic partitioning or compression here
+            if (this.optimizedConfig.compressionEnabled) {
+                this.compressIndex();
+            }
+        }
+    }
+    /**
+     * Compress index to reduce memory usage (placeholder)
+     */
+    compressIndex() {
+        console.log('Index compression not implemented yet');
+        // This would implement vector quantization or other compression techniques
+    }
+    /**
+     * Start automatic parameter tuning
+     */
+    startParameterTuning() {
+        this.parameterTuningInterval = setInterval(() => {
+            this.tuneParameters();
+        }, 30000); // Tune every 30 seconds
+    }
+    /**
+     * Automatic parameter tuning based on performance metrics
+     */
+    tuneParameters() {
+        if (this.searchHistory.length < 10) {
+            return; // Not enough data
+        }
+        const recentSearches = this.searchHistory.slice(-20);
+        const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
+        // Tune based on performance vs targets
+        const latencyRatio = averageLatency / this.optimizedConfig.targetSearchLatency;
+        const recallRatio = this.performanceMetrics.averageRecall / this.optimizedConfig.targetRecall;
+        // Adjust M (connectivity) for long-term performance
+        if (this.size() > 10000) { // Only tune for larger indices
+            if (recallRatio < 0.95 && latencyRatio < 1.5) {
+                // Recall is low but we have latency budget, increase M
+                this.dynamicParams.M = Math.min(this.dynamicParams.M + 2, 64);
+            }
+            else if (latencyRatio > 1.2 && recallRatio > 1.0) {
+                // Latency is high but recall is good, can reduce M
+                this.dynamicParams.M = Math.max(this.dynamicParams.M - 2, 16);
+            }
+        }
+        console.log(`Parameter tuning: efSearch=${this.dynamicParams.efSearch}, M=${this.dynamicParams.M}, latency=${averageLatency.toFixed(1)}ms`);
+    }
+    /**
+     * Get optimized configuration recommendations for current dataset size
+     */
+    getOptimizedConfig() {
+        const currentSize = this.size();
+        let recommendedConfig = {};
+        if (currentSize < 10000) {
+            // Small dataset - optimize for speed
+            recommendedConfig = {
+                M: 16,
+                efConstruction: 200,
+                efSearch: 50,
+                ml: 16
+            };
+        }
+        else if (currentSize < 100000) {
+            // Medium dataset - balance speed and recall
+            recommendedConfig = {
+                M: 24,
+                efConstruction: 300,
+                efSearch: 75,
+                ml: 20
+            };
+        }
+        else if (currentSize < 1000000) {
+            // Large dataset - optimize for recall
+            recommendedConfig = {
+                M: 32,
+                efConstruction: 400,
+                efSearch: 100,
+                ml: 24
+            };
+        }
+        else {
+            // Very large dataset - maximum quality
+            recommendedConfig = {
+                M: 48,
+                efConstruction: 500,
+                efSearch: 150,
+                ml: 28
+            };
+        }
+        return {
+            ...this.optimizedConfig,
+            ...recommendedConfig
+        };
+    }
+    /**
+     * Get current performance metrics
+     */
+    getPerformanceMetrics() {
+        return {
+            ...this.performanceMetrics,
+            currentParams: { ...this.dynamicParams },
+            searchHistorySize: this.searchHistory.length
+        };
+    }
+    /**
+     * Apply optimized bulk insertion strategy
+     */
+    async bulkInsert(items) {
+        console.log(`Starting optimized bulk insert of ${items.length} items`);
+        // Sort items to optimize insertion order (by vector similarity)
+        const sortedItems = this.optimizeInsertionOrder(items);
+        // Temporarily adjust construction parameters for bulk operations
+        const originalEfConstruction = this.dynamicParams.efConstruction;
+        this.dynamicParams.efConstruction = Math.min(this.dynamicParams.efConstruction * 1.5, 800);
+        const results = [];
+        const batchSize = 100;
+        try {
+            // Process in batches to manage memory
+            for (let i = 0; i < sortedItems.length; i += batchSize) {
+                const batch = sortedItems.slice(i, i + batchSize);
+                for (const item of batch) {
+                    const id = await this.addItem(item);
+                    results.push(id);
+                }
+                // Periodic memory check
+                if (i % (batchSize * 10) === 0) {
+                    this.checkMemoryUsage();
+                }
+            }
+        }
+        finally {
+            // Restore original construction parameters
+            this.dynamicParams.efConstruction = originalEfConstruction;
+        }
+        console.log(`Completed bulk insert of ${results.length} items`);
+        return results;
+    }
+    /**
+     * Optimize insertion order to improve index quality
+     */
+    optimizeInsertionOrder(items) {
+        if (items.length < 100) {
+            return items; // Not worth optimizing small batches
+        }
+        // Simple clustering-based ordering
+        // In practice, you might use more sophisticated methods
+        return items.sort(() => Math.random() - 0.5); // Shuffle for now
+    }
+    /**
+     * Cleanup resources
+     */
+    destroy() {
+        if (this.parameterTuningInterval) {
+            clearInterval(this.parameterTuningInterval);
+        }
+    }
+}
+//# sourceMappingURL=optimizedHNSWIndex.js.map

dist/hnsw/partitionedHNSWIndex.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Partitioned HNSW Index for Large-Scale Vector Search
+ * Implements sharding strategies to handle millions of vectors efficiently
+ */
+import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
+export interface PartitionConfig {
+    maxNodesPerPartition: number;
+    partitionStrategy: 'semantic' | 'hash';
+    semanticClusters?: number;
+    autoTuneSemanticClusters?: boolean;
+}
+export interface PartitionMetadata {
+    id: string;
+    nodeCount: number;
+    bounds?: {
+        centroid: Vector;
+        radius: number;
+    };
+    strategy: string;
+    created: Date;
+}
+/**
+ * Partitioned HNSW Index that splits large datasets across multiple smaller indices
+ * This enables efficient search across millions of vectors by reducing memory usage
+ * and parallelizing search operations
+ */
+export declare class PartitionedHNSWIndex {
+    private partitions;
+    private partitionMetadata;
+    private config;
+    private hnswConfig;
+    private distanceFunction;
+    private dimension;
+    private nextPartitionId;
+    constructor(partitionConfig?: Partial<PartitionConfig>, hnswConfig?: Partial<HNSWConfig>, distanceFunction?: DistanceFunction);
+    /**
+     * Add a vector to the partitioned index
+     */
+    addItem(item: VectorDocument): Promise<string>;
+    /**
+     * Search across all partitions for nearest neighbors
+     */
+    search(queryVector: Vector, k?: number, searchScope?: {
+        partitionIds?: string[];
+        maxPartitions?: number;
+    }): Promise<Array<[string, number]>>;
+    /**
+     * Select the appropriate partition for a new item
+     * Automatically chooses semantic partitioning when beneficial, falls back to hash
+     */
+    private selectPartition;
+    /**
+     * Hash-based partitioning for even distribution
+     */
+    private hashPartition;
+    /**
+     * Semantic clustering partitioning
+     */
+    private semanticPartition;
+    /**
+     * Auto-tune semantic clusters based on dataset size and performance
+     */
+    private autoTuneSemanticClusters;
+    /**
+     * Select which partitions to search based on query
+     */
+    private selectSearchPartitions;
+    /**
+     * Update partition bounds for semantic clustering
+     */
+    private updatePartitionBounds;
+    /**
+     * Split an overgrown partition into smaller partitions
+     */
+    private splitPartition;
+    /**
+     * Simple hash function for consistent partitioning
+     */
+    private simpleHash;
+    /**
+     * Get partition statistics
+     */
+    getPartitionStats(): {
+        totalPartitions: number;
+        totalNodes: number;
+        averageNodesPerPartition: number;
+        partitionDetails: PartitionMetadata[];
+    };
+    /**
+     * Remove an item from the index
+     */
+    removeItem(id: string): Promise<boolean>;
+    /**
+     * Clear all partitions
+     */
+    clear(): void;
+    /**
+     * Get total size across all partitions
+     */
+    size(): number;
+}

dist/hnsw/partitionedHNSWIndex.js

@@ -0,0 +1,304 @@
+/**
+ * Partitioned HNSW Index for Large-Scale Vector Search
+ * Implements sharding strategies to handle millions of vectors efficiently
+ */
+import { HNSWIndex } from './hnswIndex.js';
+import { euclideanDistance } from '../utils/index.js';
+/**
+ * Partitioned HNSW Index that splits large datasets across multiple smaller indices
+ * This enables efficient search across millions of vectors by reducing memory usage
+ * and parallelizing search operations
+ */
+export class PartitionedHNSWIndex {
+    constructor(partitionConfig = {}, hnswConfig = {}, distanceFunction = euclideanDistance) {
+        this.partitions = new Map();
+        this.partitionMetadata = new Map();
+        this.dimension = null;
+        this.nextPartitionId = 0;
+        this.config = {
+            maxNodesPerPartition: 50000, // Optimal size for memory efficiency
+            partitionStrategy: 'semantic', // Default to semantic for better performance
+            semanticClusters: 8, // Auto-tuned based on dataset
+            autoTuneSemanticClusters: true,
+            ...partitionConfig
+        };
+        // Optimized HNSW parameters for large scale
+        this.hnswConfig = {
+            M: 32, // Higher connectivity for better recall
+            efConstruction: 400, // Better build quality
+            efSearch: 100, // Balance speed vs accuracy
+            ml: 24, // Deeper hierarchy
+            ...hnswConfig
+        };
+        this.distanceFunction = distanceFunction;
+    }
+    /**
+     * Add a vector to the partitioned index
+     */
+    async addItem(item) {
+        if (this.dimension === null) {
+            this.dimension = item.vector.length;
+        }
+        // Determine which partition this item belongs to
+        const partitionId = await this.selectPartition(item);
+        // Get or create the partition
+        let partition = this.partitions.get(partitionId);
+        if (!partition) {
+            partition = new HNSWIndex(this.hnswConfig, this.distanceFunction, { useParallelization: true });
+            this.partitions.set(partitionId, partition);
+            // Initialize partition metadata
+            this.partitionMetadata.set(partitionId, {
+                id: partitionId,
+                nodeCount: 0,
+                strategy: this.config.partitionStrategy,
+                created: new Date()
+            });
+        }
+        // Add item to the selected partition
+        await partition.addItem(item);
+        // Update partition metadata
+        const metadata = this.partitionMetadata.get(partitionId);
+        metadata.nodeCount = partition.size();
+        // Update bounds for semantic strategy
+        if (this.config.partitionStrategy === 'semantic') {
+            this.updatePartitionBounds(partitionId, item.vector);
+        }
+        // Check if partition is getting too large and needs splitting
+        if (metadata.nodeCount > this.config.maxNodesPerPartition * 1.2) {
+            await this.splitPartition(partitionId);
+        }
+        return item.id;
+    }
+    /**
+     * Search across all partitions for nearest neighbors
+     */
+    async search(queryVector, k = 10, searchScope) {
+        if (this.partitions.size === 0) {
+            return [];
+        }
+        // Determine which partitions to search
+        const partitionsToSearch = await this.selectSearchPartitions(queryVector, searchScope);
+        // Search partitions in parallel
+        const searchPromises = partitionsToSearch.map(async (partitionId) => {
+            const partition = this.partitions.get(partitionId);
+            if (!partition)
+                return [];
+            // Search with higher k to get better global results
+            const partitionK = Math.min(k * 2, partition.size());
+            return partition.search(queryVector, partitionK);
+        });
+        const partitionResults = await Promise.all(searchPromises);
+        // Merge and sort results from all partitions
+        const allResults = [];
+        for (const results of partitionResults) {
+            allResults.push(...results);
+        }
+        // Sort by distance and return top k
+        allResults.sort((a, b) => a[1] - b[1]);
+        return allResults.slice(0, k);
+    }
+    /**
+     * Select the appropriate partition for a new item
+     * Automatically chooses semantic partitioning when beneficial, falls back to hash
+     */
+    async selectPartition(item) {
+        // Auto-tune semantic clusters based on current dataset size
+        if (this.config.autoTuneSemanticClusters && this.config.partitionStrategy === 'semantic') {
+            this.autoTuneSemanticClusters();
+        }
+        switch (this.config.partitionStrategy) {
+            case 'semantic':
+                return await this.semanticPartition(item.vector);
+            case 'hash':
+            default:
+                return this.hashPartition(item.id);
+        }
+    }
+    /**
+     * Hash-based partitioning for even distribution
+     */
+    hashPartition(id) {
+        const hash = this.simpleHash(id);
+        const existingPartitions = Array.from(this.partitions.keys());
+        // Find partition with space, or create new one
+        for (const partitionId of existingPartitions) {
+            const metadata = this.partitionMetadata.get(partitionId);
+            if (metadata && metadata.nodeCount < this.config.maxNodesPerPartition) {
+                return partitionId;
+            }
+        }
+        // Create new partition
+        return `partition_${this.nextPartitionId++}`;
+    }
+    /**
+     * Semantic clustering partitioning
+     */
+    async semanticPartition(vector) {
+        // Find closest partition centroid
+        let closestPartition = '';
+        let minDistance = Infinity;
+        for (const [partitionId, metadata] of this.partitionMetadata.entries()) {
+            if (metadata.bounds?.centroid) {
+                const distance = this.distanceFunction(vector, metadata.bounds.centroid);
+                if (distance < minDistance) {
+                    minDistance = distance;
+                    closestPartition = partitionId;
+                }
+            }
+        }
+        // If no suitable partition found or it's full, create new one
+        if (!closestPartition ||
+            this.partitionMetadata.get(closestPartition).nodeCount >= this.config.maxNodesPerPartition) {
+            closestPartition = `semantic_${this.nextPartitionId++}`;
+        }
+        return closestPartition;
+    }
+    /**
+     * Auto-tune semantic clusters based on dataset size and performance
+     */
+    autoTuneSemanticClusters() {
+        const totalNodes = this.size();
+        const currentPartitions = this.partitions.size;
+        // Optimal clusters based on dataset size
+        let optimalClusters = Math.max(4, Math.min(32, Math.floor(totalNodes / 10000)));
+        // Adjust based on current partition performance
+        if (currentPartitions > 0) {
+            const avgNodesPerPartition = totalNodes / currentPartitions;
+            if (avgNodesPerPartition > this.config.maxNodesPerPartition * 0.8) {
+                // Partitions are getting full, increase clusters
+                optimalClusters = Math.min(32, this.config.semanticClusters + 2);
+            }
+            else if (avgNodesPerPartition < this.config.maxNodesPerPartition * 0.3 && currentPartitions > 4) {
+                // Partitions are underutilized, decrease clusters
+                optimalClusters = Math.max(4, this.config.semanticClusters - 1);
+            }
+        }
+        if (optimalClusters !== this.config.semanticClusters) {
+            console.log(`Auto-tuning semantic clusters: ${this.config.semanticClusters} → ${optimalClusters}`);
+            this.config.semanticClusters = optimalClusters;
+        }
+    }
+    /**
+     * Select which partitions to search based on query
+     */
+    async selectSearchPartitions(queryVector, searchScope) {
+        if (searchScope?.partitionIds) {
+            return searchScope.partitionIds.filter(id => this.partitions.has(id));
+        }
+        const maxPartitions = searchScope?.maxPartitions || Math.min(5, this.partitions.size);
+        if (this.config.partitionStrategy === 'semantic') {
+            // Search partitions with closest centroids
+            const distances = [];
+            for (const [partitionId, metadata] of this.partitionMetadata.entries()) {
+                if (metadata.bounds?.centroid) {
+                    const distance = this.distanceFunction(queryVector, metadata.bounds.centroid);
+                    distances.push([partitionId, distance]);
+                }
+            }
+            distances.sort((a, b) => a[1] - b[1]);
+            return distances.slice(0, maxPartitions).map(([id]) => id);
+        }
+        // For other strategies, search all partitions or random subset
+        const allPartitionIds = Array.from(this.partitions.keys());
+        if (allPartitionIds.length <= maxPartitions) {
+            return allPartitionIds;
+        }
+        // Return random subset
+        const shuffled = [...allPartitionIds].sort(() => Math.random() - 0.5);
+        return shuffled.slice(0, maxPartitions);
+    }
+    /**
+     * Update partition bounds for semantic clustering
+     */
+    updatePartitionBounds(partitionId, vector) {
+        const metadata = this.partitionMetadata.get(partitionId);
+        if (!metadata.bounds) {
+            metadata.bounds = {
+                centroid: [...vector],
+                radius: 0
+            };
+            return;
+        }
+        // Update centroid using incremental mean
+        const { centroid } = metadata.bounds;
+        const nodeCount = metadata.nodeCount;
+        for (let i = 0; i < centroid.length; i++) {
+            centroid[i] = (centroid[i] * (nodeCount - 1) + vector[i]) / nodeCount;
+        }
+        // Update radius
+        const distance = this.distanceFunction(vector, centroid);
+        metadata.bounds.radius = Math.max(metadata.bounds.radius, distance);
+    }
+    /**
+     * Split an overgrown partition into smaller partitions
+     */
+    async splitPartition(partitionId) {
+        const partition = this.partitions.get(partitionId);
+        if (!partition)
+            return;
+        console.log(`Splitting partition ${partitionId} with ${partition.size()} nodes`);
+        // For now, we'll implement a simple strategy
+        // In a full implementation, you'd want to analyze the data distribution
+        // and create more intelligent splits
+        // This is a placeholder - actual implementation would require
+        // accessing the internal nodes of the HNSW index
+    }
+    /**
+     * Simple hash function for consistent partitioning
+     */
+    simpleHash(str) {
+        let hash = 0;
+        for (let i = 0; i < str.length; i++) {
+            const char = str.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash = hash & hash; // Convert to 32-bit integer
+        }
+        return Math.abs(hash);
+    }
+    /**
+     * Get partition statistics
+     */
+    getPartitionStats() {
+        const partitionDetails = Array.from(this.partitionMetadata.values());
+        const totalNodes = partitionDetails.reduce((sum, p) => sum + p.nodeCount, 0);
+        return {
+            totalPartitions: partitionDetails.length,
+            totalNodes,
+            averageNodesPerPartition: totalNodes / partitionDetails.length || 0,
+            partitionDetails
+        };
+    }
+    /**
+     * Remove an item from the index
+     */
+    async removeItem(id) {
+        // Find which partition contains this item
+        for (const [partitionId, partition] of this.partitions.entries()) {
+            if (partition.removeItem(id)) {
+                // Update metadata
+                const metadata = this.partitionMetadata.get(partitionId);
+                metadata.nodeCount = partition.size();
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Clear all partitions
+     */
+    clear() {
+        for (const partition of this.partitions.values()) {
+            partition.clear();
+        }
+        this.partitions.clear();
+        this.partitionMetadata.clear();
+        this.nextPartitionId = 0;
+    }
+    /**
+     * Get total size across all partitions
+     */
+    size() {
+        return Array.from(this.partitions.values()).reduce((sum, partition) => sum + partition.size(), 0);
+    }
+}
+//# sourceMappingURL=partitionedHNSWIndex.js.map

dist/hnsw/scaledHNSWSystem.d.ts

@@ -0,0 +1,142 @@
+/**
+ * Scaled HNSW System - Integration of All Optimization Strategies
+ * Production-ready system for handling millions of vectors with sub-second search
+ */
+import { Vector, VectorDocument } from '../coreTypes.js';
+import { PartitionConfig } from './partitionedHNSWIndex.js';
+import { OptimizedHNSWConfig } from './optimizedHNSWIndex.js';
+import { SearchStrategy } from './distributedSearch.js';
+export interface ScaledHNSWConfig {
+    expectedDatasetSize?: number;
+    maxMemoryUsage?: number;
+    targetSearchLatency?: number;
+    s3Config?: {
+        bucketName: string;
+        region: string;
+        endpoint?: string;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+    };
+    autoConfigureEnvironment?: boolean;
+    learningEnabled?: boolean;
+    enablePartitioning?: boolean;
+    enableCompression?: boolean;
+    enableDistributedSearch?: boolean;
+    enablePredictiveCaching?: boolean;
+    partitionConfig?: Partial<PartitionConfig>;
+    hnswConfig?: Partial<OptimizedHNSWConfig>;
+    readOnlyMode?: boolean;
+}
+/**
+ * High-performance HNSW system with all optimizations integrated
+ * Handles datasets from thousands to millions of vectors
+ */
+export declare class ScaledHNSWSystem {
+    private config;
+    private autoConfig;
+    private partitionedIndex?;
+    private distributedSearch?;
+    private cacheManager?;
+    private batchOperations?;
+    private readOnlyOptimizations?;
+    private performanceMetrics;
+    constructor(config?: ScaledHNSWConfig);
+    /**
+     * Initialize the optimized system based on configuration
+     */
+    private initializeOptimizedSystem;
+    /**
+     * Calculate optimal configuration based on dataset size and constraints
+     */
+    private calculateOptimalConfiguration;
+    /**
+     * Add vector to the scaled system
+     */
+    addVector(item: VectorDocument): Promise<string>;
+    /**
+     * Bulk insert vectors with optimizations
+     */
+    bulkInsert(items: VectorDocument[]): Promise<string[]>;
+    /**
+     * High-performance vector search with all optimizations
+     */
+    search(queryVector: Vector, k?: number, options?: {
+        strategy?: SearchStrategy;
+        useCache?: boolean;
+        maxPartitions?: number;
+    }): Promise<Array<[string, number]>>;
+    /**
+     * Get system performance metrics
+     */
+    getPerformanceMetrics(): typeof this.performanceMetrics & {
+        partitionStats?: any;
+        cacheStats?: any;
+        compressionStats?: any;
+        distributedSearchStats?: any;
+    };
+    /**
+     * Optimize insertion order for better index quality
+     */
+    private optimizeInsertionOrder;
+    /**
+     * Calculate optimal batch size based on system resources
+     */
+    private calculateOptimalBatchSize;
+    /**
+     * Update search performance metrics
+     */
+    private updateSearchMetrics;
+    /**
+     * Estimate current memory usage
+     */
+    private estimateMemoryUsage;
+    /**
+     * Generate performance report
+     */
+    generatePerformanceReport(): string;
+    /**
+     * Get overall system status
+     */
+    private getSystemStatus;
+    /**
+     * Check if adaptive learning should be triggered
+     */
+    private shouldTriggerLearning;
+    /**
+     * Adaptively learn from performance and adjust configuration
+     */
+    private adaptivelyLearnFromPerformance;
+    /**
+     * Update dataset analysis for better auto-configuration
+     */
+    updateDatasetAnalysis(vectorCount: number, vectorDimension?: number): Promise<void>;
+    /**
+     * Infer access patterns from current metrics
+     */
+    private inferAccessPatterns;
+    /**
+     * Cleanup system resources
+     */
+    cleanup(): void;
+}
+/**
+ * Create a fully auto-configured Brainy system - minimal setup required!
+ * Just provide S3 config if you want persistence beyond the current session
+ */
+export declare function createAutoBrainy(s3Config?: {
+    bucketName: string;
+    region?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+}): ScaledHNSWSystem;
+/**
+ * Create a Brainy system optimized for specific scenarios
+ */
+export declare function createQuickBrainy(scenario: 'small' | 'medium' | 'large' | 'enterprise', s3Config?: {
+    bucketName: string;
+    region?: string;
+}): Promise<ScaledHNSWSystem>;
+/**
+ * Legacy factory function - still works but consider using createAutoBrainy() instead
+ */
+export declare function createScaledHNSWSystem(config?: ScaledHNSWConfig): ScaledHNSWSystem;

dist/hnsw/scaledHNSWSystem.js

@@ -0,0 +1,559 @@
+/**
+ * Scaled HNSW System - Integration of All Optimization Strategies
+ * Production-ready system for handling millions of vectors with sub-second search
+ */
+import { PartitionedHNSWIndex } from './partitionedHNSWIndex.js';
+import { DistributedSearchSystem, SearchStrategy } from './distributedSearch.js';
+import { EnhancedCacheManager } from '../storage/enhancedCacheManager.js';
+import { BatchS3Operations } from '../storage/adapters/batchS3Operations.js';
+import { ReadOnlyOptimizations } from '../storage/readOnlyOptimizations.js';
+import { euclideanDistance } from '../utils/index.js';
+import { AutoConfiguration } from '../utils/autoConfiguration.js';
+/**
+ * High-performance HNSW system with all optimizations integrated
+ * Handles datasets from thousands to millions of vectors
+ */
+export class ScaledHNSWSystem {
+    constructor(config = {}) {
+        // Performance monitoring and learning
+        this.performanceMetrics = {
+            totalSearches: 0,
+            averageSearchTime: 0,
+            cacheHitRate: 0,
+            compressionRatio: 0,
+            memoryUsage: 0,
+            indexSize: 0,
+            lastLearningUpdate: Date.now()
+        };
+        this.autoConfig = AutoConfiguration.getInstance();
+        // Set basic defaults - these will be overridden by auto-configuration
+        this.config = {
+            expectedDatasetSize: 100000,
+            maxMemoryUsage: 4 * 1024 * 1024 * 1024,
+            targetSearchLatency: 150,
+            autoConfigureEnvironment: true,
+            learningEnabled: true,
+            enablePartitioning: true,
+            enableCompression: true,
+            enableDistributedSearch: true,
+            enablePredictiveCaching: true,
+            readOnlyMode: false,
+            ...config
+        };
+        this.initializeOptimizedSystem();
+    }
+    /**
+     * Initialize the optimized system based on configuration
+     */
+    async initializeOptimizedSystem() {
+        console.log('Initializing Scaled HNSW System with auto-configuration...');
+        // Auto-configure if enabled
+        if (this.config.autoConfigureEnvironment) {
+            const autoConfigResult = await this.autoConfig.detectAndConfigure({
+                expectedDataSize: this.config.expectedDatasetSize,
+                s3Available: !!this.config.s3Config,
+                memoryBudget: this.config.maxMemoryUsage
+            });
+            console.log(`Detected environment: ${autoConfigResult.environment}`);
+            console.log(`Available memory: ${(autoConfigResult.availableMemory / 1024 / 1024 / 1024).toFixed(1)}GB`);
+            console.log(`CPU cores: ${autoConfigResult.cpuCores}`);
+            // Override config with auto-detected values
+            this.config = {
+                ...this.config,
+                expectedDatasetSize: autoConfigResult.recommendedConfig.expectedDatasetSize,
+                maxMemoryUsage: autoConfigResult.recommendedConfig.maxMemoryUsage,
+                targetSearchLatency: autoConfigResult.recommendedConfig.targetSearchLatency,
+                enablePartitioning: autoConfigResult.recommendedConfig.enablePartitioning,
+                enableCompression: autoConfigResult.recommendedConfig.enableCompression,
+                enableDistributedSearch: autoConfigResult.recommendedConfig.enableDistributedSearch,
+                enablePredictiveCaching: autoConfigResult.recommendedConfig.enablePredictiveCaching
+            };
+        }
+        // Determine optimal configuration
+        const optimizedConfig = this.calculateOptimalConfiguration();
+        // Initialize partitioned index with semantic partitioning as default
+        if (this.config.enablePartitioning) {
+            this.partitionedIndex = new PartitionedHNSWIndex({
+                ...optimizedConfig.partitionConfig,
+                partitionStrategy: 'semantic', // Always use semantic for better performance
+                autoTuneSemanticClusters: true // Enable auto-tuning
+            }, optimizedConfig.hnswConfig, euclideanDistance);
+            console.log('✓ Partitioned index initialized with semantic clustering');
+        }
+        // Initialize distributed search system
+        if (this.config.enableDistributedSearch && this.partitionedIndex) {
+            this.distributedSearch = new DistributedSearchSystem({
+                maxConcurrentSearches: optimizedConfig.maxConcurrentSearches,
+                searchTimeout: this.config.targetSearchLatency * 5,
+                adaptivePartitionSelection: true,
+                loadBalancing: true
+            });
+            console.log('✓ Distributed search system initialized');
+        }
+        // Initialize batch S3 operations
+        if (this.config.s3Config) {
+            this.batchOperations = new BatchS3Operations(null, // Would be initialized with actual S3 client
+            this.config.s3Config.bucketName, {
+                maxConcurrency: 50,
+                useS3Select: this.config.expectedDatasetSize > 100000
+            });
+            console.log('✓ Batch S3 operations initialized');
+        }
+        // Initialize enhanced caching
+        if (this.config.enablePredictiveCaching) {
+            this.cacheManager = new EnhancedCacheManager({
+                hotCacheMaxSize: optimizedConfig.hotCacheSize,
+                warmCacheMaxSize: optimizedConfig.warmCacheSize,
+                prefetchEnabled: true,
+                prefetchStrategy: 'hybrid', // Type casting for enum compatibility
+                prefetchBatchSize: 50
+            });
+            if (this.batchOperations) {
+                this.cacheManager.setStorageAdapters(null, this.batchOperations);
+            }
+            console.log('✓ Enhanced cache manager initialized');
+        }
+        // Initialize read-only optimizations
+        if (this.config.readOnlyMode && this.config.enableCompression) {
+            this.readOnlyOptimizations = new ReadOnlyOptimizations({
+                compression: {
+                    vectorCompression: 'quantization',
+                    metadataCompression: 'gzip',
+                    quantizationType: 'scalar',
+                    quantizationBits: 8
+                },
+                segmentSize: optimizedConfig.segmentSize,
+                memoryMapped: true,
+                cacheIndexInMemory: optimizedConfig.cacheIndexInMemory
+            });
+            console.log('✓ Read-only optimizations initialized');
+        }
+        console.log('Scaled HNSW System ready for', this.config.expectedDatasetSize, 'vectors');
+    }
+    /**
+     * Calculate optimal configuration based on dataset size and constraints
+     */
+    calculateOptimalConfiguration() {
+        const size = this.config.expectedDatasetSize;
+        const memoryBudget = this.config.maxMemoryUsage;
+        let config = {};
+        if (size <= 10000) {
+            // Small dataset - optimize for speed
+            config = {
+                partitionConfig: {
+                    maxNodesPerPartition: 10000,
+                    partitionStrategy: 'hash'
+                },
+                hnswConfig: {
+                    M: 16,
+                    efConstruction: 200,
+                    efSearch: 50,
+                    targetSearchLatency: this.config.targetSearchLatency
+                },
+                hotCacheSize: 1000,
+                warmCacheSize: 5000,
+                maxConcurrentSearches: 4,
+                segmentSize: 5000,
+                cacheIndexInMemory: true
+            };
+        }
+        else if (size <= 100000) {
+            // Medium dataset - balance performance and memory
+            config = {
+                partitionConfig: {
+                    maxNodesPerPartition: 25000,
+                    partitionStrategy: 'semantic',
+                    semanticClusters: 8
+                },
+                hnswConfig: {
+                    M: 24,
+                    efConstruction: 300,
+                    efSearch: 75,
+                    targetSearchLatency: this.config.targetSearchLatency,
+                    dynamicParameterTuning: true
+                },
+                hotCacheSize: 2000,
+                warmCacheSize: 15000,
+                maxConcurrentSearches: 8,
+                segmentSize: 10000,
+                cacheIndexInMemory: memoryBudget > 2 * 1024 * 1024 * 1024 // 2GB
+            };
+        }
+        else if (size <= 1000000) {
+            // Large dataset - optimize for scale
+            config = {
+                partitionConfig: {
+                    maxNodesPerPartition: 50000,
+                    partitionStrategy: 'semantic',
+                    semanticClusters: 16
+                },
+                hnswConfig: {
+                    M: 32,
+                    efConstruction: 400,
+                    efSearch: 100,
+                    targetSearchLatency: this.config.targetSearchLatency,
+                    dynamicParameterTuning: true,
+                    memoryBudget: memoryBudget
+                },
+                hotCacheSize: 5000,
+                warmCacheSize: 25000,
+                maxConcurrentSearches: 12,
+                segmentSize: 20000,
+                cacheIndexInMemory: memoryBudget > 8 * 1024 * 1024 * 1024 // 8GB
+            };
+        }
+        else {
+            // Very large dataset - maximum optimization
+            config = {
+                partitionConfig: {
+                    maxNodesPerPartition: 100000,
+                    partitionStrategy: 'hybrid',
+                    semanticClusters: 32
+                },
+                hnswConfig: {
+                    M: 48,
+                    efConstruction: 500,
+                    efSearch: 150,
+                    targetSearchLatency: this.config.targetSearchLatency,
+                    dynamicParameterTuning: true,
+                    memoryBudget: memoryBudget,
+                    diskCacheEnabled: true
+                },
+                hotCacheSize: 10000,
+                warmCacheSize: 50000,
+                maxConcurrentSearches: 20,
+                segmentSize: 50000,
+                cacheIndexInMemory: false // Too large for memory
+            };
+        }
+        return config;
+    }
+    /**
+     * Add vector to the scaled system
+     */
+    async addVector(item) {
+        if (!this.partitionedIndex) {
+            throw new Error('System not properly initialized');
+        }
+        const startTime = Date.now();
+        const result = await this.partitionedIndex.addItem(item);
+        // Update performance metrics
+        this.performanceMetrics.indexSize = this.partitionedIndex.size();
+        return result;
+    }
+    /**
+     * Bulk insert vectors with optimizations
+     */
+    async bulkInsert(items) {
+        if (!this.partitionedIndex) {
+            throw new Error('System not properly initialized');
+        }
+        console.log(`Starting optimized bulk insert of ${items.length} vectors`);
+        const startTime = Date.now();
+        // Sort items for optimal insertion order
+        const sortedItems = this.optimizeInsertionOrder(items);
+        const results = [];
+        const batchSize = this.calculateOptimalBatchSize(items.length);
+        // Process in batches
+        for (let i = 0; i < sortedItems.length; i += batchSize) {
+            const batch = sortedItems.slice(i, i + batchSize);
+            for (const item of batch) {
+                const id = await this.partitionedIndex.addItem(item);
+                results.push(id);
+            }
+            // Progress logging
+            if (i % (batchSize * 10) === 0) {
+                const progress = ((i / sortedItems.length) * 100).toFixed(1);
+                console.log(`Bulk insert progress: ${progress}%`);
+            }
+        }
+        const totalTime = Date.now() - startTime;
+        console.log(`Bulk insert completed: ${results.length} vectors in ${totalTime}ms`);
+        return results;
+    }
+    /**
+     * High-performance vector search with all optimizations
+     */
+    async search(queryVector, k = 10, options = {}) {
+        const startTime = Date.now();
+        try {
+            let results;
+            if (this.distributedSearch && this.partitionedIndex) {
+                // Use distributed search for optimal performance
+                results = await this.distributedSearch.distributedSearch(this.partitionedIndex, queryVector, k, options.strategy || SearchStrategy.ADAPTIVE);
+            }
+            else if (this.partitionedIndex) {
+                // Fall back to partitioned search
+                results = await this.partitionedIndex.search(queryVector, k, { maxPartitions: options.maxPartitions });
+            }
+            else {
+                throw new Error('No search system available');
+            }
+            // Update performance metrics and learn from performance
+            const searchTime = Date.now() - startTime;
+            this.updateSearchMetrics(searchTime, results.length);
+            // Adaptive learning - adjust configuration based on performance
+            if (this.config.learningEnabled && this.shouldTriggerLearning()) {
+                await this.adaptivelyLearnFromPerformance();
+            }
+            return results;
+        }
+        catch (error) {
+            console.error('Search failed:', error);
+            throw error;
+        }
+    }
+    /**
+     * Get system performance metrics
+     */
+    getPerformanceMetrics() {
+        const metrics = { ...this.performanceMetrics };
+        // Add subsystem metrics
+        if (this.partitionedIndex) {
+            metrics.partitionStats = this.partitionedIndex.getPartitionStats();
+        }
+        if (this.cacheManager) {
+            metrics.cacheStats = this.cacheManager.getStats();
+        }
+        if (this.readOnlyOptimizations) {
+            metrics.compressionStats = this.readOnlyOptimizations.getCompressionStats();
+        }
+        if (this.distributedSearch) {
+            metrics.distributedSearchStats = this.distributedSearch.getSearchStats();
+        }
+        return metrics;
+    }
+    /**
+     * Optimize insertion order for better index quality
+     */
+    optimizeInsertionOrder(items) {
+        if (items.length < 1000) {
+            return items; // Not worth optimizing small batches
+        }
+        // Simple clustering-based approach for better HNSW construction
+        // In production, you might use more sophisticated clustering
+        return items.sort(() => Math.random() - 0.5);
+    }
+    /**
+     * Calculate optimal batch size based on system resources
+     */
+    calculateOptimalBatchSize(totalItems) {
+        const memoryBudget = this.config.maxMemoryUsage;
+        const estimatedItemSize = 1000; // Rough estimate per item in bytes
+        const maxBatch = Math.floor(memoryBudget * 0.1 / estimatedItemSize);
+        const targetBatch = Math.min(1000, Math.max(100, maxBatch));
+        return Math.min(targetBatch, totalItems);
+    }
+    /**
+     * Update search performance metrics
+     */
+    updateSearchMetrics(searchTime, resultCount) {
+        this.performanceMetrics.totalSearches++;
+        this.performanceMetrics.averageSearchTime =
+            (this.performanceMetrics.averageSearchTime + searchTime) / 2;
+        // Update other metrics
+        if (this.cacheManager) {
+            const cacheStats = this.cacheManager.getStats();
+            const totalOps = cacheStats.hotCacheHits + cacheStats.hotCacheMisses +
+                cacheStats.warmCacheHits + cacheStats.warmCacheMisses;
+            this.performanceMetrics.cacheHitRate = totalOps > 0 ?
+                (cacheStats.hotCacheHits + cacheStats.warmCacheHits) / totalOps : 0;
+        }
+        if (this.readOnlyOptimizations) {
+            const compressionStats = this.readOnlyOptimizations.getCompressionStats();
+            this.performanceMetrics.compressionRatio = compressionStats.compressionRatio;
+        }
+        // Estimate memory usage
+        this.performanceMetrics.memoryUsage = this.estimateMemoryUsage();
+    }
+    /**
+     * Estimate current memory usage
+     */
+    estimateMemoryUsage() {
+        let totalMemory = 0;
+        if (this.partitionedIndex) {
+            // Rough estimate: 1KB per vector
+            totalMemory += this.partitionedIndex.size() * 1024;
+        }
+        if (this.cacheManager) {
+            const cacheStats = this.cacheManager.getStats();
+            totalMemory += (cacheStats.hotCacheSize + cacheStats.warmCacheSize) * 1024;
+        }
+        return totalMemory;
+    }
+    /**
+     * Generate performance report
+     */
+    generatePerformanceReport() {
+        const metrics = this.getPerformanceMetrics();
+        return `
+=== Scaled HNSW System Performance Report ===
+
+Dataset Configuration:
+- Expected Size: ${this.config.expectedDatasetSize.toLocaleString()} vectors
+- Current Size: ${metrics.indexSize.toLocaleString()} vectors
+- Memory Budget: ${(this.config.maxMemoryUsage / 1024 / 1024 / 1024).toFixed(1)}GB
+- Target Latency: ${this.config.targetSearchLatency}ms
+
+Performance Metrics:
+- Total Searches: ${metrics.totalSearches.toLocaleString()}
+- Average Search Time: ${metrics.averageSearchTime.toFixed(1)}ms
+- Cache Hit Rate: ${(metrics.cacheHitRate * 100).toFixed(1)}%
+- Memory Usage: ${(metrics.memoryUsage / 1024 / 1024).toFixed(1)}MB
+- Compression Ratio: ${metrics.compressionRatio ? (metrics.compressionRatio * 100).toFixed(1) + '%' : 'N/A'}
+
+System Status: ${this.getSystemStatus()}
+    `.trim();
+    }
+    /**
+     * Get overall system status
+     */
+    getSystemStatus() {
+        const metrics = this.getPerformanceMetrics();
+        if (metrics.averageSearchTime <= this.config.targetSearchLatency) {
+            return '✅ OPTIMAL';
+        }
+        else if (metrics.averageSearchTime <= this.config.targetSearchLatency * 2) {
+            return '⚠️  ACCEPTABLE';
+        }
+        else {
+            return '❌ NEEDS OPTIMIZATION';
+        }
+    }
+    /**
+     * Check if adaptive learning should be triggered
+     */
+    shouldTriggerLearning() {
+        const timeSinceLastLearning = Date.now() - this.performanceMetrics.lastLearningUpdate;
+        const minLearningInterval = 30000; // 30 seconds
+        const minSearches = 20; // Minimum searches before learning
+        return timeSinceLastLearning > minLearningInterval &&
+            this.performanceMetrics.totalSearches > minSearches &&
+            this.performanceMetrics.totalSearches % 50 === 0; // Learn every 50 searches
+    }
+    /**
+     * Adaptively learn from performance and adjust configuration
+     */
+    async adaptivelyLearnFromPerformance() {
+        try {
+            const currentMetrics = {
+                averageSearchTime: this.performanceMetrics.averageSearchTime,
+                memoryUsage: this.performanceMetrics.memoryUsage,
+                cacheHitRate: this.performanceMetrics.cacheHitRate,
+                errorRate: 0 // Could be tracked separately
+            };
+            const adjustments = await this.autoConfig.learnFromPerformance(currentMetrics);
+            if (Object.keys(adjustments).length > 0) {
+                console.log('🧠 Adaptive learning: Adjusting configuration based on performance');
+                // Apply learned adjustments
+                let configChanged = false;
+                if (adjustments.enableDistributedSearch !== undefined &&
+                    adjustments.enableDistributedSearch !== this.config.enableDistributedSearch) {
+                    this.config.enableDistributedSearch = adjustments.enableDistributedSearch;
+                    configChanged = true;
+                }
+                if (adjustments.enableCompression !== undefined &&
+                    adjustments.enableCompression !== this.config.enableCompression) {
+                    this.config.enableCompression = adjustments.enableCompression;
+                    configChanged = true;
+                }
+                if (adjustments.enablePredictiveCaching !== undefined &&
+                    adjustments.enablePredictiveCaching !== this.config.enablePredictiveCaching) {
+                    this.config.enablePredictiveCaching = adjustments.enablePredictiveCaching;
+                    configChanged = true;
+                }
+                // Apply partition adjustments
+                if (adjustments.maxNodesPerPartition &&
+                    this.partitionedIndex &&
+                    adjustments.maxNodesPerPartition !== this.partitionedIndex.getPartitionStats().averageNodesPerPartition) {
+                    // This would require rebuilding the index in a real implementation
+                    console.log(`Learning suggests partition size: ${adjustments.maxNodesPerPartition}`);
+                }
+                if (configChanged) {
+                    console.log('✅ Configuration updated based on performance learning');
+                }
+            }
+            this.performanceMetrics.lastLearningUpdate = Date.now();
+        }
+        catch (error) {
+            console.warn('Adaptive learning failed:', error);
+        }
+    }
+    /**
+     * Update dataset analysis for better auto-configuration
+     */
+    async updateDatasetAnalysis(vectorCount, vectorDimension) {
+        if (this.config.autoConfigureEnvironment) {
+            const analysis = {
+                estimatedSize: vectorCount,
+                vectorDimension,
+                accessPatterns: this.inferAccessPatterns()
+            };
+            await this.autoConfig.adaptToDataset(analysis);
+            console.log(`📊 Dataset analysis updated: ${vectorCount} vectors${vectorDimension ? `, ${vectorDimension}D` : ''}`);
+        }
+    }
+    /**
+     * Infer access patterns from current metrics
+     */
+    inferAccessPatterns() {
+        // Simple heuristic - in practice, this would track read/write ratios
+        if (this.performanceMetrics.totalSearches > 100) {
+            return 'read-heavy';
+        }
+        return 'balanced';
+    }
+    /**
+     * Cleanup system resources
+     */
+    cleanup() {
+        this.distributedSearch?.cleanup();
+        this.cacheManager?.clear();
+        this.readOnlyOptimizations?.cleanup();
+        this.partitionedIndex?.clear();
+        this.autoConfig.resetCache();
+        console.log('Scaled HNSW System cleaned up');
+    }
+}
+// Export convenience factory functions
+/**
+ * Create a fully auto-configured Brainy system - minimal setup required!
+ * Just provide S3 config if you want persistence beyond the current session
+ */
+export function createAutoBrainy(s3Config) {
+    return new ScaledHNSWSystem({
+        s3Config: s3Config ? {
+            bucketName: s3Config.bucketName,
+            region: s3Config.region || 'us-east-1',
+            accessKeyId: s3Config.accessKeyId,
+            secretAccessKey: s3Config.secretAccessKey
+        } : undefined,
+        autoConfigureEnvironment: true,
+        learningEnabled: true
+    });
+}
+/**
+ * Create a Brainy system optimized for specific scenarios
+ */
+export async function createQuickBrainy(scenario, s3Config) {
+    const { getQuickSetup } = await import('../utils/autoConfiguration.js');
+    const quickConfig = await getQuickSetup(scenario);
+    return new ScaledHNSWSystem({
+        ...quickConfig,
+        s3Config: s3Config && quickConfig.s3Required ? {
+            bucketName: s3Config.bucketName,
+            region: s3Config.region || 'us-east-1',
+            accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+            secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
+        } : undefined,
+        autoConfigureEnvironment: true,
+        learningEnabled: true
+    });
+}
+/**
+ * Legacy factory function - still works but consider using createAutoBrainy() instead
+ */
+export function createScaledHNSWSystem(config = {}) {
+    return new ScaledHNSWSystem(config);
+}
+//# sourceMappingURL=scaledHNSWSystem.js.map