chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
118
.recovery-workspace/dist-backup-20250910-141917/hnsw/distributedSearch.d.ts
vendored
Normal file
118
.recovery-workspace/dist-backup-20250910-141917/hnsw/distributedSearch.d.ts
vendored
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
/**
|
||||
* Distributed Search System for Large-Scale HNSW Indices
|
||||
* Implements parallel search across multiple partitions and instances
|
||||
*/
|
||||
import { Vector } from '../coreTypes.js';
|
||||
import { PartitionedHNSWIndex } from './partitionedHNSWIndex.js';
|
||||
interface DistributedSearchConfig {
|
||||
maxConcurrentSearches?: number;
|
||||
searchTimeout?: number;
|
||||
resultMergeStrategy?: 'distance' | 'score' | 'hybrid';
|
||||
adaptivePartitionSelection?: boolean;
|
||||
redundantSearches?: number;
|
||||
loadBalancing?: boolean;
|
||||
}
|
||||
export declare enum SearchStrategy {
|
||||
BROADCAST = "broadcast",// Search all partitions
|
||||
SELECTIVE = "selective",// Search subset of partitions
|
||||
ADAPTIVE = "adaptive",// Dynamically adjust based on results
|
||||
HIERARCHICAL = "hierarchical"
|
||||
}
|
||||
interface SearchWorker {
|
||||
id: string;
|
||||
busy: boolean;
|
||||
tasksCompleted: number;
|
||||
averageTaskTime: number;
|
||||
lastTaskTime: number;
|
||||
}
|
||||
/**
|
||||
* Distributed search coordinator for large-scale vector search
|
||||
*/
|
||||
export declare class DistributedSearchSystem {
|
||||
private config;
|
||||
private searchWorkers;
|
||||
private searchQueue;
|
||||
private activeSearches;
|
||||
private partitionStats;
|
||||
private searchStats;
|
||||
constructor(config?: Partial<DistributedSearchConfig>);
|
||||
/**
|
||||
* Execute distributed search across multiple partitions
|
||||
*/
|
||||
distributedSearch(partitionedIndex: PartitionedHNSWIndex, queryVector: Vector, k: number, strategy?: SearchStrategy): Promise<Array<[string, number]>>;
|
||||
/**
|
||||
* Select partitions to search based on strategy
|
||||
*/
|
||||
private selectPartitions;
|
||||
/**
|
||||
* Adaptive partition selection based on historical performance
|
||||
*/
|
||||
private adaptivePartitionSelection;
|
||||
/**
|
||||
* Select top-performing partitions
|
||||
*/
|
||||
private selectTopPartitions;
|
||||
/**
|
||||
* Hierarchical partition selection for very large datasets
|
||||
*/
|
||||
private hierarchicalPartitionSelection;
|
||||
/**
|
||||
* Create search tasks for parallel execution
|
||||
*/
|
||||
private createSearchTasks;
|
||||
/**
|
||||
* Execute searches in parallel across selected partitions
|
||||
*/
|
||||
private executeParallelSearches;
|
||||
/**
|
||||
* Execute search on a single partition
|
||||
*/
|
||||
private executePartitionSearch;
|
||||
/**
|
||||
* Determine if search should use worker thread
|
||||
*/
|
||||
private shouldUseWorkerThread;
|
||||
/**
|
||||
* Execute search in worker thread
|
||||
*/
|
||||
private executeInWorkerThread;
|
||||
/**
|
||||
* Get available worker from pool
|
||||
*/
|
||||
private getAvailableWorker;
|
||||
/**
|
||||
* Merge search results from multiple partitions
|
||||
*/
|
||||
private mergeSearchResults;
|
||||
/**
|
||||
* Get partition quality score
|
||||
*/
|
||||
private getPartitionQuality;
|
||||
/**
|
||||
* Update search statistics
|
||||
*/
|
||||
private updateSearchStats;
|
||||
/**
|
||||
* Initialize worker thread pool
|
||||
*/
|
||||
private initializeWorkerPool;
|
||||
/**
|
||||
* Generate unique search ID
|
||||
*/
|
||||
private generateSearchId;
|
||||
/**
|
||||
* Get search performance statistics
|
||||
*/
|
||||
getSearchStats(): typeof this.searchStats & {
|
||||
workerStats: SearchWorker[];
|
||||
partitionStats: Array<{
|
||||
id: string;
|
||||
stats: any;
|
||||
}>;
|
||||
};
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
cleanup(): void;
|
||||
}
|
||||
export {};
|
||||
|
|
@ -0,0 +1,452 @@
|
|||
/**
|
||||
* Distributed Search System for Large-Scale HNSW Indices
|
||||
* Implements parallel search across multiple partitions and instances
|
||||
*/
|
||||
import { executeInThread } from '../utils/workerUtils.js';
|
||||
// Search coordination strategies
|
||||
export var SearchStrategy;
|
||||
(function (SearchStrategy) {
|
||||
SearchStrategy["BROADCAST"] = "broadcast";
|
||||
SearchStrategy["SELECTIVE"] = "selective";
|
||||
SearchStrategy["ADAPTIVE"] = "adaptive";
|
||||
SearchStrategy["HIERARCHICAL"] = "hierarchical"; // Multi-level search
|
||||
})(SearchStrategy || (SearchStrategy = {}));
|
||||
/**
|
||||
* Distributed search coordinator for large-scale vector search
|
||||
*/
|
||||
export class DistributedSearchSystem {
|
||||
constructor(config = {}) {
|
||||
this.searchWorkers = new Map();
|
||||
this.searchQueue = [];
|
||||
this.activeSearches = new Map();
|
||||
this.partitionStats = new Map();
|
||||
// Performance monitoring
|
||||
this.searchStats = {
|
||||
totalSearches: 0,
|
||||
averageLatency: 0,
|
||||
parallelEfficiency: 0,
|
||||
cacheHitRate: 0,
|
||||
partitionUtilization: new Map()
|
||||
};
|
||||
this.config = {
|
||||
maxConcurrentSearches: 10,
|
||||
searchTimeout: 30000, // 30 seconds
|
||||
resultMergeStrategy: 'hybrid',
|
||||
adaptivePartitionSelection: true,
|
||||
redundantSearches: 0,
|
||||
loadBalancing: true,
|
||||
...config
|
||||
};
|
||||
this.initializeWorkerPool();
|
||||
}
|
||||
/**
|
||||
* Execute distributed search across multiple partitions
|
||||
*/
|
||||
async distributedSearch(partitionedIndex, queryVector, k, strategy = SearchStrategy.ADAPTIVE) {
|
||||
const searchId = this.generateSearchId();
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
// Select partitions to search based on strategy
|
||||
const partitionsToSearch = await this.selectPartitions(partitionedIndex, queryVector, strategy);
|
||||
// Create search tasks
|
||||
const searchTasks = this.createSearchTasks(partitionsToSearch, queryVector, k, searchId);
|
||||
// Execute searches in parallel
|
||||
const searchResults = await this.executeParallelSearches(partitionedIndex, searchTasks);
|
||||
// Merge results from all partitions
|
||||
const mergedResults = this.mergeSearchResults(searchResults, k);
|
||||
// Update statistics
|
||||
this.updateSearchStats(searchId, startTime, searchResults);
|
||||
return mergedResults;
|
||||
}
|
||||
catch (error) {
|
||||
console.error(`Distributed search ${searchId} failed:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Select partitions to search based on strategy
|
||||
*/
|
||||
async selectPartitions(partitionedIndex, queryVector, strategy) {
|
||||
const stats = partitionedIndex.getPartitionStats();
|
||||
const allPartitionIds = stats.partitionDetails.map(p => p.id);
|
||||
switch (strategy) {
|
||||
case SearchStrategy.BROADCAST:
|
||||
return allPartitionIds;
|
||||
case SearchStrategy.SELECTIVE:
|
||||
return this.selectTopPartitions(allPartitionIds, 3);
|
||||
case SearchStrategy.ADAPTIVE:
|
||||
return await this.adaptivePartitionSelection(allPartitionIds, queryVector);
|
||||
case SearchStrategy.HIERARCHICAL:
|
||||
return this.hierarchicalPartitionSelection(allPartitionIds);
|
||||
default:
|
||||
return allPartitionIds;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Adaptive partition selection based on historical performance
|
||||
*/
|
||||
async adaptivePartitionSelection(partitionIds, queryVector) {
|
||||
const candidates = [];
|
||||
for (const partitionId of partitionIds) {
|
||||
const stats = this.partitionStats.get(partitionId);
|
||||
let score = 1.0;
|
||||
if (stats) {
|
||||
// Score based on performance metrics
|
||||
const speedScore = 1000 / Math.max(stats.averageSearchTime, 1);
|
||||
const loadScore = Math.max(0, 1 - stats.load);
|
||||
const qualityScore = stats.quality;
|
||||
const recencyScore = Math.max(0, 1 - (Date.now() - stats.lastUsed) / 3600000);
|
||||
score = speedScore * 0.3 + loadScore * 0.25 + qualityScore * 0.3 + recencyScore * 0.15;
|
||||
}
|
||||
candidates.push({ id: partitionId, score });
|
||||
}
|
||||
// Sort by score and select top partitions
|
||||
candidates.sort((a, b) => b.score - a.score);
|
||||
const selectedCount = Math.min(Math.ceil(partitionIds.length * 0.6), 8);
|
||||
return candidates.slice(0, selectedCount).map(c => c.id);
|
||||
}
|
||||
/**
|
||||
* Select top-performing partitions
|
||||
*/
|
||||
selectTopPartitions(partitionIds, count) {
|
||||
const withStats = partitionIds.map(id => ({
|
||||
id,
|
||||
stats: this.partitionStats.get(id)
|
||||
}));
|
||||
// Sort by average search time (faster is better)
|
||||
withStats.sort((a, b) => {
|
||||
const timeA = a.stats?.averageSearchTime || 1000;
|
||||
const timeB = b.stats?.averageSearchTime || 1000;
|
||||
return timeA - timeB;
|
||||
});
|
||||
return withStats.slice(0, count).map(p => p.id);
|
||||
}
|
||||
/**
|
||||
* Hierarchical partition selection for very large datasets
|
||||
*/
|
||||
hierarchicalPartitionSelection(partitionIds) {
|
||||
// First level: select representative partitions
|
||||
const firstLevel = partitionIds.filter((_, index) => index % 3 === 0);
|
||||
// Could implement a two-phase search here:
|
||||
// 1. Quick search on representative partitions
|
||||
// 2. Detailed search on promising partitions
|
||||
return firstLevel;
|
||||
}
|
||||
/**
|
||||
* Create search tasks for parallel execution
|
||||
*/
|
||||
createSearchTasks(partitionIds, queryVector, k, searchId) {
|
||||
const tasks = [];
|
||||
for (let i = 0; i < partitionIds.length; i++) {
|
||||
const partitionId = partitionIds[i];
|
||||
const stats = this.partitionStats.get(partitionId);
|
||||
// Calculate priority based on partition performance
|
||||
const priority = stats ? (1000 - stats.averageSearchTime) : 500;
|
||||
tasks.push({
|
||||
partitionId,
|
||||
queryVector: [...queryVector], // Clone vector
|
||||
k: Math.max(k * 2, 20), // Search for more results per partition
|
||||
searchId,
|
||||
priority
|
||||
});
|
||||
// Add redundant searches if configured
|
||||
if (this.config.redundantSearches > 0 && i < this.config.redundantSearches) {
|
||||
tasks.push({
|
||||
partitionId,
|
||||
queryVector: [...queryVector],
|
||||
k: Math.max(k * 2, 20),
|
||||
searchId: `${searchId}_redundant_${i}`,
|
||||
priority: priority - 100 // Lower priority for redundant searches
|
||||
});
|
||||
}
|
||||
}
|
||||
// Sort tasks by priority
|
||||
tasks.sort((a, b) => b.priority - a.priority);
|
||||
return tasks;
|
||||
}
|
||||
/**
|
||||
* Execute searches in parallel across selected partitions
|
||||
*/
|
||||
async executeParallelSearches(partitionedIndex, searchTasks) {
|
||||
const results = [];
|
||||
const semaphore = new Semaphore(this.config.maxConcurrentSearches);
|
||||
// Execute tasks with controlled concurrency
|
||||
const taskPromises = searchTasks.map(async (task) => {
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
// Execute search with timeout
|
||||
const searchPromise = this.executePartitionSearch(partitionedIndex, task);
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error('Search timeout')), this.config.searchTimeout);
|
||||
});
|
||||
const result = await Promise.race([searchPromise, timeoutPromise]);
|
||||
result.searchTime = Date.now() - startTime;
|
||||
return result;
|
||||
}
|
||||
catch (error) {
|
||||
return {
|
||||
partitionId: task.partitionId,
|
||||
results: [],
|
||||
searchTime: this.config.searchTimeout,
|
||||
nodesVisited: 0,
|
||||
error: error
|
||||
};
|
||||
}
|
||||
finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
// Wait for all searches to complete
|
||||
const taskResults = await Promise.allSettled(taskPromises);
|
||||
for (const result of taskResults) {
|
||||
if (result.status === 'fulfilled') {
|
||||
results.push(result.value);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Execute search on a single partition
|
||||
*/
|
||||
async executePartitionSearch(partitionedIndex, task) {
|
||||
try {
|
||||
// Use thread pool for compute-intensive operations
|
||||
if (this.shouldUseWorkerThread(task)) {
|
||||
return await this.executeInWorkerThread(partitionedIndex, task);
|
||||
}
|
||||
// Execute search directly
|
||||
const results = await partitionedIndex.search(task.queryVector, task.k, { partitionIds: [task.partitionId] });
|
||||
return {
|
||||
partitionId: task.partitionId,
|
||||
results,
|
||||
searchTime: 0, // Will be set by caller
|
||||
nodesVisited: results.length // Approximation
|
||||
};
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Partition search failed: ${error}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Determine if search should use worker thread
|
||||
*/
|
||||
shouldUseWorkerThread(task) {
|
||||
// Use worker threads for high-dimensional vectors or large k
|
||||
return task.queryVector.length > 512 || task.k > 100;
|
||||
}
|
||||
/**
|
||||
* Execute search in worker thread
|
||||
*/
|
||||
async executeInWorkerThread(partitionedIndex, task) {
|
||||
const worker = this.getAvailableWorker();
|
||||
if (!worker) {
|
||||
// No available workers, execute synchronously
|
||||
return this.executePartitionSearch(partitionedIndex, task);
|
||||
}
|
||||
try {
|
||||
worker.busy = true;
|
||||
const startTime = Date.now();
|
||||
// Execute in thread (simplified - would need proper worker setup)
|
||||
const searchFunction = `
|
||||
return partitionedIndex.search(
|
||||
task.queryVector,
|
||||
task.k,
|
||||
{ partitionIds: [task.partitionId] }
|
||||
)
|
||||
`;
|
||||
const results = await executeInThread(searchFunction, {
|
||||
queryVector: task.queryVector,
|
||||
k: task.k,
|
||||
partitionId: task.partitionId
|
||||
});
|
||||
const searchTime = Date.now() - startTime;
|
||||
worker.averageTaskTime = (worker.averageTaskTime + searchTime) / 2;
|
||||
worker.tasksCompleted++;
|
||||
return {
|
||||
partitionId: task.partitionId,
|
||||
results: results || [],
|
||||
searchTime,
|
||||
nodesVisited: results ? results.length : 0
|
||||
};
|
||||
}
|
||||
finally {
|
||||
worker.busy = false;
|
||||
worker.lastTaskTime = Date.now();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get available worker from pool
|
||||
*/
|
||||
getAvailableWorker() {
|
||||
for (const worker of this.searchWorkers.values()) {
|
||||
if (!worker.busy) {
|
||||
return worker;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Merge search results from multiple partitions
|
||||
*/
|
||||
mergeSearchResults(partitionResults, k) {
|
||||
const allResults = [];
|
||||
const seenIds = new Set();
|
||||
// Collect all unique results
|
||||
for (const partitionResult of partitionResults) {
|
||||
if (partitionResult.error) {
|
||||
console.warn(`Partition ${partitionResult.partitionId} failed:`, partitionResult.error);
|
||||
continue;
|
||||
}
|
||||
for (const [id, distance] of partitionResult.results) {
|
||||
if (!seenIds.has(id)) {
|
||||
allResults.push([id, distance]);
|
||||
seenIds.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort and return top k results
|
||||
switch (this.config.resultMergeStrategy) {
|
||||
case 'distance':
|
||||
allResults.sort((a, b) => a[1] - b[1]);
|
||||
break;
|
||||
case 'score':
|
||||
// Convert distance to score (1 / (1 + distance))
|
||||
allResults.sort((a, b) => {
|
||||
const scoreA = 1 / (1 + a[1]);
|
||||
const scoreB = 1 / (1 + b[1]);
|
||||
return scoreB - scoreA;
|
||||
});
|
||||
break;
|
||||
case 'hybrid':
|
||||
// Weighted combination of distance and partition quality
|
||||
allResults.sort((a, b) => {
|
||||
const qualityWeightA = this.getPartitionQuality(a[0]);
|
||||
const qualityWeightB = this.getPartitionQuality(b[0]);
|
||||
const adjustedDistanceA = a[1] / (qualityWeightA + 0.1);
|
||||
const adjustedDistanceB = b[1] / (qualityWeightB + 0.1);
|
||||
return adjustedDistanceA - adjustedDistanceB;
|
||||
});
|
||||
break;
|
||||
}
|
||||
return allResults.slice(0, k);
|
||||
}
|
||||
/**
|
||||
* Get partition quality score
|
||||
*/
|
||||
getPartitionQuality(nodeId) {
|
||||
// This would require knowing which partition a node came from
|
||||
// For now, return a default quality score
|
||||
return 1.0;
|
||||
}
|
||||
/**
|
||||
* Update search statistics
|
||||
*/
|
||||
updateSearchStats(searchId, startTime, results) {
|
||||
const totalTime = Date.now() - startTime;
|
||||
const successfulSearches = results.filter(r => !r.error);
|
||||
// Update global stats
|
||||
this.searchStats.totalSearches++;
|
||||
this.searchStats.averageLatency =
|
||||
(this.searchStats.averageLatency + totalTime) / 2;
|
||||
// Calculate parallel efficiency
|
||||
const totalPartitionTime = results.reduce((sum, r) => sum + r.searchTime, 0);
|
||||
this.searchStats.parallelEfficiency =
|
||||
totalPartitionTime > 0 ? totalTime / totalPartitionTime : 0;
|
||||
// Update partition statistics
|
||||
for (const result of successfulSearches) {
|
||||
let stats = this.partitionStats.get(result.partitionId);
|
||||
if (!stats) {
|
||||
stats = {
|
||||
averageSearchTime: result.searchTime,
|
||||
load: 0,
|
||||
quality: 1.0,
|
||||
lastUsed: Date.now()
|
||||
};
|
||||
}
|
||||
else {
|
||||
stats.averageSearchTime = (stats.averageSearchTime + result.searchTime) / 2;
|
||||
stats.lastUsed = Date.now();
|
||||
}
|
||||
this.partitionStats.set(result.partitionId, stats);
|
||||
this.searchStats.partitionUtilization.set(result.partitionId, (this.searchStats.partitionUtilization.get(result.partitionId) || 0) + 1);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Initialize worker thread pool
|
||||
*/
|
||||
initializeWorkerPool() {
|
||||
const workerCount = Math.min(navigator.hardwareConcurrency || 4, 8);
|
||||
for (let i = 0; i < workerCount; i++) {
|
||||
const worker = {
|
||||
id: `worker_${i}`,
|
||||
busy: false,
|
||||
tasksCompleted: 0,
|
||||
averageTaskTime: 0,
|
||||
lastTaskTime: 0
|
||||
};
|
||||
this.searchWorkers.set(worker.id, worker);
|
||||
}
|
||||
console.log(`Initialized worker pool with ${workerCount} workers`);
|
||||
}
|
||||
/**
|
||||
* Generate unique search ID
|
||||
*/
|
||||
generateSearchId() {
|
||||
return `search_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
||||
}
|
||||
/**
|
||||
* Get search performance statistics
|
||||
*/
|
||||
getSearchStats() {
|
||||
return {
|
||||
...this.searchStats,
|
||||
workerStats: Array.from(this.searchWorkers.values()),
|
||||
partitionStats: Array.from(this.partitionStats.entries()).map(([id, stats]) => ({
|
||||
id,
|
||||
stats
|
||||
}))
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
cleanup() {
|
||||
// Clear active searches
|
||||
this.activeSearches.clear();
|
||||
// Reset worker states
|
||||
for (const worker of this.searchWorkers.values()) {
|
||||
worker.busy = false;
|
||||
}
|
||||
// Clear statistics
|
||||
this.partitionStats.clear();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Simple semaphore for concurrency control
|
||||
*/
|
||||
class Semaphore {
|
||||
constructor(permits) {
|
||||
this.waiting = [];
|
||||
this.permits = permits;
|
||||
}
|
||||
async acquire() {
|
||||
if (this.permits > 0) {
|
||||
this.permits--;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
this.waiting.push(resolve);
|
||||
});
|
||||
}
|
||||
release() {
|
||||
if (this.waiting.length > 0) {
|
||||
const resolve = this.waiting.shift();
|
||||
resolve();
|
||||
}
|
||||
else {
|
||||
this.permits++;
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=distributedSearch.js.map
|
||||
File diff suppressed because one or more lines are too long
134
.recovery-workspace/dist-backup-20250910-141917/hnsw/hnswIndex.d.ts
vendored
Normal file
134
.recovery-workspace/dist-backup-20250910-141917/hnsw/hnswIndex.d.ts
vendored
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
/**
|
||||
* HNSW (Hierarchical Navigable Small World) Index implementation
|
||||
* Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
|
||||
*/
|
||||
import { DistanceFunction, HNSWConfig, HNSWNoun, Vector, VectorDocument } from '../coreTypes.js';
|
||||
export declare class HNSWIndex {
|
||||
private nouns;
|
||||
private entryPointId;
|
||||
private maxLevel;
|
||||
private config;
|
||||
private distanceFunction;
|
||||
private dimension;
|
||||
private useParallelization;
|
||||
constructor(config?: Partial<HNSWConfig>, distanceFunction?: DistanceFunction, options?: {
|
||||
useParallelization?: boolean;
|
||||
});
|
||||
/**
|
||||
* Set whether to use parallelization for performance-critical operations
|
||||
*/
|
||||
setUseParallelization(useParallelization: boolean): void;
|
||||
/**
|
||||
* Get whether parallelization is enabled
|
||||
*/
|
||||
getUseParallelization(): boolean;
|
||||
/**
|
||||
* Calculate distances between a query vector and multiple vectors in parallel
|
||||
* This is used to optimize performance for search operations
|
||||
* Uses optimized batch processing for optimal performance
|
||||
*
|
||||
* @param queryVector The query vector
|
||||
* @param vectors Array of vectors to compare against
|
||||
* @returns Array of distances
|
||||
*/
|
||||
private calculateDistancesInParallel;
|
||||
/**
|
||||
* Add a vector to the index
|
||||
*/
|
||||
addItem(item: VectorDocument): Promise<string>;
|
||||
/**
|
||||
* Search for nearest neighbors
|
||||
*/
|
||||
search(queryVector: Vector, k?: number, filter?: (id: string) => Promise<boolean>): Promise<Array<[string, number]>>;
|
||||
/**
|
||||
* Remove an item from the index
|
||||
*/
|
||||
removeItem(id: string): boolean;
|
||||
/**
|
||||
* Get all nouns in the index
|
||||
* @deprecated Use getNounsPaginated() instead for better scalability
|
||||
*/
|
||||
getNouns(): Map<string, HNSWNoun>;
|
||||
/**
|
||||
* Get nouns with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Object containing paginated nouns and pagination info
|
||||
*/
|
||||
getNounsPaginated(options?: {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
filter?: (noun: HNSWNoun) => boolean;
|
||||
}): {
|
||||
items: Map<string, HNSWNoun>;
|
||||
totalCount: number;
|
||||
hasMore: boolean;
|
||||
};
|
||||
/**
|
||||
* Clear the index
|
||||
*/
|
||||
clear(): void;
|
||||
/**
|
||||
* Get the size of the index
|
||||
*/
|
||||
size(): number;
|
||||
/**
|
||||
* Get the distance function used by the index
|
||||
*/
|
||||
getDistanceFunction(): DistanceFunction;
|
||||
/**
|
||||
* Get the entry point ID
|
||||
*/
|
||||
getEntryPointId(): string | null;
|
||||
/**
|
||||
* Get the maximum level
|
||||
*/
|
||||
getMaxLevel(): number;
|
||||
/**
|
||||
* Get the dimension
|
||||
*/
|
||||
getDimension(): number | null;
|
||||
/**
|
||||
* Get the configuration
|
||||
*/
|
||||
getConfig(): HNSWConfig;
|
||||
/**
|
||||
* Get all nodes at a specific level for clustering
|
||||
* This enables O(n) clustering using HNSW's natural hierarchy
|
||||
*/
|
||||
getNodesAtLevel(level: number): HNSWNoun[];
|
||||
/**
|
||||
* Get level statistics for understanding the hierarchy
|
||||
*/
|
||||
getLevelStats(): Array<{
|
||||
level: number;
|
||||
nodeCount: number;
|
||||
avgConnections: number;
|
||||
}>;
|
||||
/**
|
||||
* Get index health metrics
|
||||
*/
|
||||
getIndexHealth(): {
|
||||
averageConnections: number;
|
||||
layerDistribution: number[];
|
||||
maxLayer: number;
|
||||
totalNodes: number;
|
||||
};
|
||||
/**
|
||||
* Search within a specific layer
|
||||
* Returns a map of noun IDs to distances, sorted by distance
|
||||
*/
|
||||
private searchLayer;
|
||||
/**
|
||||
* Select M nearest neighbors from the candidate set
|
||||
*/
|
||||
private selectNeighbors;
|
||||
/**
|
||||
* Ensure a noun doesn't have too many connections at a given level
|
||||
*/
|
||||
private pruneConnections;
|
||||
/**
|
||||
* Generate a random level for a new noun
|
||||
* Uses the same distribution as in the original HNSW paper
|
||||
*/
|
||||
private getRandomLevel;
|
||||
}
|
||||
|
|
@ -0,0 +1,656 @@
|
|||
/**
|
||||
* HNSW (Hierarchical Navigable Small World) Index implementation
|
||||
* Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
|
||||
*/
|
||||
import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js';
|
||||
// Default HNSW parameters
|
||||
const DEFAULT_CONFIG = {
|
||||
M: 16, // Max number of connections per noun
|
||||
efConstruction: 200, // Size of a dynamic candidate list during construction
|
||||
efSearch: 50, // Size of a dynamic candidate list during search
|
||||
ml: 16 // Max level
|
||||
};
|
||||
export class HNSWIndex {
|
||||
constructor(config = {}, distanceFunction = euclideanDistance, options = {}) {
|
||||
this.nouns = new Map();
|
||||
this.entryPointId = null;
|
||||
this.maxLevel = 0;
|
||||
this.dimension = null;
|
||||
this.useParallelization = true; // Whether to use parallelization for performance-critical operations
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
this.distanceFunction = distanceFunction;
|
||||
this.useParallelization =
|
||||
options.useParallelization !== undefined
|
||||
? options.useParallelization
|
||||
: true;
|
||||
}
|
||||
/**
|
||||
* Set whether to use parallelization for performance-critical operations
|
||||
*/
|
||||
setUseParallelization(useParallelization) {
|
||||
this.useParallelization = useParallelization;
|
||||
}
|
||||
/**
|
||||
* Get whether parallelization is enabled
|
||||
*/
|
||||
getUseParallelization() {
|
||||
return this.useParallelization;
|
||||
}
|
||||
/**
|
||||
* Calculate distances between a query vector and multiple vectors in parallel
|
||||
* This is used to optimize performance for search operations
|
||||
* Uses optimized batch processing for optimal performance
|
||||
*
|
||||
* @param queryVector The query vector
|
||||
* @param vectors Array of vectors to compare against
|
||||
* @returns Array of distances
|
||||
*/
|
||||
async calculateDistancesInParallel(queryVector, vectors) {
|
||||
// If parallelization is disabled or there are very few vectors, use sequential processing
|
||||
if (!this.useParallelization || vectors.length < 10) {
|
||||
return vectors.map((item) => ({
|
||||
id: item.id,
|
||||
distance: this.distanceFunction(queryVector, item.vector)
|
||||
}));
|
||||
}
|
||||
try {
|
||||
// Extract just the vectors from the input array
|
||||
const vectorsOnly = vectors.map((item) => item.vector);
|
||||
// Use optimized batch distance calculation
|
||||
const distances = await calculateDistancesBatch(queryVector, vectorsOnly, this.distanceFunction);
|
||||
// Map the distances back to their IDs
|
||||
return vectors.map((item, index) => ({
|
||||
id: item.id,
|
||||
distance: distances[index]
|
||||
}));
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error in batch distance calculation, falling back to sequential processing:', error);
|
||||
// Fall back to sequential processing if batch calculation fails
|
||||
return vectors.map((item) => ({
|
||||
id: item.id,
|
||||
distance: this.distanceFunction(queryVector, item.vector)
|
||||
}));
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Add a vector to the index
|
||||
*/
|
||||
async addItem(item) {
|
||||
// Check if item is defined
|
||||
if (!item) {
|
||||
throw new Error('Item is undefined or null');
|
||||
}
|
||||
const { id, vector } = item;
|
||||
// Check if vector is defined
|
||||
if (!vector) {
|
||||
throw new Error('Vector is undefined or null');
|
||||
}
|
||||
// Set dimension on first insert
|
||||
if (this.dimension === null) {
|
||||
this.dimension = vector.length;
|
||||
}
|
||||
else if (vector.length !== this.dimension) {
|
||||
throw new Error(`Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`);
|
||||
}
|
||||
// Generate random level for this noun
|
||||
const nounLevel = this.getRandomLevel();
|
||||
// Create new noun
|
||||
const noun = {
|
||||
id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: nounLevel
|
||||
};
|
||||
// Initialize empty connection sets for each level
|
||||
for (let level = 0; level <= nounLevel; level++) {
|
||||
noun.connections.set(level, new Set());
|
||||
}
|
||||
// If this is the first noun, make it the entry point
|
||||
if (this.nouns.size === 0) {
|
||||
this.entryPointId = id;
|
||||
this.maxLevel = nounLevel;
|
||||
this.nouns.set(id, noun);
|
||||
return id;
|
||||
}
|
||||
// Find entry point
|
||||
if (!this.entryPointId) {
|
||||
console.error('Entry point ID is null');
|
||||
// If there's no entry point, this is the first noun, so we should have returned earlier
|
||||
// This is a safety check
|
||||
this.entryPointId = id;
|
||||
this.maxLevel = nounLevel;
|
||||
this.nouns.set(id, noun);
|
||||
return id;
|
||||
}
|
||||
const entryPoint = this.nouns.get(this.entryPointId);
|
||||
if (!entryPoint) {
|
||||
console.error(`Entry point with ID ${this.entryPointId} not found`);
|
||||
// If the entry point doesn't exist, treat this as the first noun
|
||||
this.entryPointId = id;
|
||||
this.maxLevel = nounLevel;
|
||||
this.nouns.set(id, noun);
|
||||
return id;
|
||||
}
|
||||
let currObj = entryPoint;
|
||||
let currDist = this.distanceFunction(vector, entryPoint.vector);
|
||||
// Traverse the graph from top to bottom to find the closest noun
|
||||
for (let level = this.maxLevel; level > nounLevel; level--) {
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
// Check all neighbors at current level
|
||||
const connections = currObj.connections.get(level) || new Set();
|
||||
for (const neighborId of connections) {
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor) {
|
||||
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
||||
continue;
|
||||
}
|
||||
const distToNeighbor = this.distanceFunction(vector, neighbor.vector);
|
||||
if (distToNeighbor < currDist) {
|
||||
currDist = distToNeighbor;
|
||||
currObj = neighbor;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// For each level from nounLevel down to 0
|
||||
for (let level = Math.min(nounLevel, this.maxLevel); level >= 0; level--) {
|
||||
// Find ef nearest elements using greedy search
|
||||
const nearestNouns = await this.searchLayer(vector, currObj, this.config.efConstruction, level);
|
||||
// Select M nearest neighbors
|
||||
const neighbors = this.selectNeighbors(vector, nearestNouns, this.config.M);
|
||||
// Add bidirectional connections
|
||||
for (const [neighborId, _] of neighbors) {
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor) {
|
||||
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
||||
continue;
|
||||
}
|
||||
noun.connections.get(level).add(neighborId);
|
||||
// Add reverse connection
|
||||
if (!neighbor.connections.has(level)) {
|
||||
neighbor.connections.set(level, new Set());
|
||||
}
|
||||
neighbor.connections.get(level).add(id);
|
||||
// Ensure neighbor doesn't have too many connections
|
||||
if (neighbor.connections.get(level).size > this.config.M) {
|
||||
this.pruneConnections(neighbor, level);
|
||||
}
|
||||
}
|
||||
// Update entry point for the next level
|
||||
if (nearestNouns.size > 0) {
|
||||
const [nearestId, nearestDist] = [...nearestNouns][0];
|
||||
if (nearestDist < currDist) {
|
||||
currDist = nearestDist;
|
||||
const nearestNoun = this.nouns.get(nearestId);
|
||||
if (!nearestNoun) {
|
||||
console.error(`Nearest noun with ID ${nearestId} not found in addItem`);
|
||||
// Keep the current object as is
|
||||
}
|
||||
else {
|
||||
currObj = nearestNoun;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Update max level and entry point if needed
|
||||
if (nounLevel > this.maxLevel) {
|
||||
this.maxLevel = nounLevel;
|
||||
this.entryPointId = id;
|
||||
}
|
||||
// Add noun to the index
|
||||
this.nouns.set(id, noun);
|
||||
return id;
|
||||
}
|
||||
/**
|
||||
* Search for nearest neighbors
|
||||
*/
|
||||
async search(queryVector, k = 10, filter) {
|
||||
if (this.nouns.size === 0) {
|
||||
return [];
|
||||
}
|
||||
// Check if query vector is defined
|
||||
if (!queryVector) {
|
||||
throw new Error('Query vector is undefined or null');
|
||||
}
|
||||
if (this.dimension !== null && queryVector.length !== this.dimension) {
|
||||
throw new Error(`Query vector dimension mismatch: expected ${this.dimension}, got ${queryVector.length}`);
|
||||
}
|
||||
// Start from the entry point
|
||||
if (!this.entryPointId) {
|
||||
console.error('Entry point ID is null');
|
||||
return [];
|
||||
}
|
||||
const entryPoint = this.nouns.get(this.entryPointId);
|
||||
if (!entryPoint) {
|
||||
console.error(`Entry point with ID ${this.entryPointId} not found`);
|
||||
return [];
|
||||
}
|
||||
let currObj = entryPoint;
|
||||
let currDist = this.distanceFunction(queryVector, currObj.vector);
|
||||
// Traverse the graph from top to bottom to find the closest noun
|
||||
for (let level = this.maxLevel; level > 0; level--) {
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
// Check all neighbors at current level
|
||||
const connections = currObj.connections.get(level) || new Set();
|
||||
// If we have enough connections, use parallel distance calculation
|
||||
if (this.useParallelization && connections.size >= 10) {
|
||||
// Prepare vectors for parallel calculation
|
||||
const vectors = [];
|
||||
for (const neighborId of connections) {
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor)
|
||||
continue;
|
||||
vectors.push({ id: neighborId, vector: neighbor.vector });
|
||||
}
|
||||
// Calculate distances in parallel
|
||||
const distances = await this.calculateDistancesInParallel(queryVector, vectors);
|
||||
// Find the closest neighbor
|
||||
for (const { id, distance } of distances) {
|
||||
if (distance < currDist) {
|
||||
currDist = distance;
|
||||
const neighbor = this.nouns.get(id);
|
||||
if (neighbor) {
|
||||
currObj = neighbor;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Use sequential processing for small number of connections
|
||||
for (const neighborId of connections) {
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor) {
|
||||
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
||||
continue;
|
||||
}
|
||||
const distToNeighbor = this.distanceFunction(queryVector, neighbor.vector);
|
||||
if (distToNeighbor < currDist) {
|
||||
currDist = distToNeighbor;
|
||||
currObj = neighbor;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Search at level 0 with ef = k
|
||||
// If we have a filter, increase ef to compensate for filtered results
|
||||
const ef = filter ? Math.max(this.config.efSearch * 3, k * 3) : Math.max(this.config.efSearch, k);
|
||||
const nearestNouns = await this.searchLayer(queryVector, currObj, ef, 0, filter);
|
||||
// Convert to array and sort by distance
|
||||
return [...nearestNouns].slice(0, k);
|
||||
}
|
||||
/**
|
||||
* Remove an item from the index
|
||||
*/
|
||||
removeItem(id) {
|
||||
if (!this.nouns.has(id)) {
|
||||
return false;
|
||||
}
|
||||
const noun = this.nouns.get(id);
|
||||
// Remove connections to this noun from all neighbors
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
for (const neighborId of connections) {
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor) {
|
||||
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
||||
continue;
|
||||
}
|
||||
if (neighbor.connections.has(level)) {
|
||||
neighbor.connections.get(level).delete(id);
|
||||
// Prune connections after removing this noun to ensure consistency
|
||||
this.pruneConnections(neighbor, level);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also check all other nouns for references to this noun and remove them
|
||||
for (const [nounId, otherNoun] of this.nouns.entries()) {
|
||||
if (nounId === id)
|
||||
continue; // Skip the noun being removed
|
||||
for (const [level, connections] of otherNoun.connections.entries()) {
|
||||
if (connections.has(id)) {
|
||||
connections.delete(id);
|
||||
// Prune connections after removing this reference
|
||||
this.pruneConnections(otherNoun, level);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Remove the noun
|
||||
this.nouns.delete(id);
|
||||
// If we removed the entry point, find a new one
|
||||
if (this.entryPointId === id) {
|
||||
if (this.nouns.size === 0) {
|
||||
this.entryPointId = null;
|
||||
this.maxLevel = 0;
|
||||
}
|
||||
else {
|
||||
// Find the noun with the highest level
|
||||
let maxLevel = 0;
|
||||
let newEntryPointId = null;
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
if (noun.connections.size === 0)
|
||||
continue; // Skip nouns with no connections
|
||||
const nounLevel = Math.max(...noun.connections.keys());
|
||||
if (nounLevel >= maxLevel) {
|
||||
maxLevel = nounLevel;
|
||||
newEntryPointId = nounId;
|
||||
}
|
||||
}
|
||||
this.entryPointId = newEntryPointId;
|
||||
this.maxLevel = maxLevel;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Get all nouns in the index
|
||||
* @deprecated Use getNounsPaginated() instead for better scalability
|
||||
*/
|
||||
getNouns() {
|
||||
return new Map(this.nouns);
|
||||
}
|
||||
/**
|
||||
* Get nouns with pagination
|
||||
* @param options Pagination options
|
||||
* @returns Object containing paginated nouns and pagination info
|
||||
*/
|
||||
getNounsPaginated(options = {}) {
|
||||
const offset = options.offset || 0;
|
||||
const limit = options.limit || 100;
|
||||
const filter = options.filter || (() => true);
|
||||
// Get all noun entries
|
||||
const entries = [...this.nouns.entries()];
|
||||
// Apply filter if provided
|
||||
const filteredEntries = entries.filter(([_, noun]) => filter(noun));
|
||||
// Get total count after filtering
|
||||
const totalCount = filteredEntries.length;
|
||||
// Apply pagination
|
||||
const paginatedEntries = filteredEntries.slice(offset, offset + limit);
|
||||
// Check if there are more items
|
||||
const hasMore = offset + limit < totalCount;
|
||||
// Create a new map with the paginated entries
|
||||
const items = new Map(paginatedEntries);
|
||||
return {
|
||||
items,
|
||||
totalCount,
|
||||
hasMore
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Clear the index
|
||||
*/
|
||||
clear() {
|
||||
this.nouns.clear();
|
||||
this.entryPointId = null;
|
||||
this.maxLevel = 0;
|
||||
}
|
||||
/**
|
||||
* Get the size of the index
|
||||
*/
|
||||
size() {
|
||||
return this.nouns.size;
|
||||
}
|
||||
/**
|
||||
* Get the distance function used by the index
|
||||
*/
|
||||
getDistanceFunction() {
|
||||
return this.distanceFunction;
|
||||
}
|
||||
/**
|
||||
* Get the entry point ID
|
||||
*/
|
||||
getEntryPointId() {
|
||||
return this.entryPointId;
|
||||
}
|
||||
/**
|
||||
* Get the maximum level
|
||||
*/
|
||||
getMaxLevel() {
|
||||
return this.maxLevel;
|
||||
}
|
||||
/**
|
||||
* Get the dimension
|
||||
*/
|
||||
getDimension() {
|
||||
return this.dimension;
|
||||
}
|
||||
/**
|
||||
* Get the configuration
|
||||
*/
|
||||
getConfig() {
|
||||
return { ...this.config };
|
||||
}
|
||||
/**
|
||||
* Get all nodes at a specific level for clustering
|
||||
* This enables O(n) clustering using HNSW's natural hierarchy
|
||||
*/
|
||||
getNodesAtLevel(level) {
|
||||
const nodesAtLevel = [];
|
||||
for (const noun of this.nouns.values()) {
|
||||
// A noun exists at level L if it has connections at that level or higher
|
||||
if (noun.level >= level) {
|
||||
nodesAtLevel.push(noun);
|
||||
}
|
||||
}
|
||||
return nodesAtLevel;
|
||||
}
|
||||
/**
|
||||
* Get level statistics for understanding the hierarchy
|
||||
*/
|
||||
getLevelStats() {
|
||||
const levelStats = new Map();
|
||||
for (const noun of this.nouns.values()) {
|
||||
for (let level = 0; level <= noun.level; level++) {
|
||||
if (!levelStats.has(level)) {
|
||||
levelStats.set(level, { count: 0, totalConnections: 0 });
|
||||
}
|
||||
const stats = levelStats.get(level);
|
||||
stats.count++;
|
||||
stats.totalConnections += noun.connections.get(level)?.size || 0;
|
||||
}
|
||||
}
|
||||
return Array.from(levelStats.entries()).map(([level, stats]) => ({
|
||||
level,
|
||||
nodeCount: stats.count,
|
||||
avgConnections: stats.count > 0 ? stats.totalConnections / stats.count : 0
|
||||
})).sort((a, b) => a.level - b.level);
|
||||
}
|
||||
/**
|
||||
* Get index health metrics
|
||||
*/
|
||||
getIndexHealth() {
|
||||
let totalConnections = 0;
|
||||
const layerCounts = new Array(this.maxLevel + 1).fill(0);
|
||||
// Count connections and layer distribution
|
||||
this.nouns.forEach(noun => {
|
||||
// Count connections at each layer
|
||||
for (let level = 0; level <= noun.level; level++) {
|
||||
totalConnections += noun.connections.get(level)?.size || 0;
|
||||
layerCounts[level]++;
|
||||
}
|
||||
});
|
||||
const totalNodes = this.nouns.size;
|
||||
const averageConnections = totalNodes > 0 ? totalConnections / totalNodes : 0;
|
||||
return {
|
||||
averageConnections,
|
||||
layerDistribution: layerCounts,
|
||||
maxLayer: this.maxLevel,
|
||||
totalNodes
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Search within a specific layer
|
||||
* Returns a map of noun IDs to distances, sorted by distance
|
||||
*/
|
||||
async searchLayer(queryVector, entryPoint, ef, level, filter) {
|
||||
// Set of visited nouns
|
||||
const visited = new Set([entryPoint.id]);
|
||||
// Check if entry point passes filter
|
||||
const entryPointDistance = this.distanceFunction(queryVector, entryPoint.vector);
|
||||
const entryPointPasses = filter ? await filter(entryPoint.id) : true;
|
||||
// Priority queue of candidates (closest first)
|
||||
const candidates = new Map();
|
||||
candidates.set(entryPoint.id, entryPointDistance);
|
||||
// Priority queue of nearest neighbors found so far (closest first)
|
||||
const nearest = new Map();
|
||||
if (entryPointPasses) {
|
||||
nearest.set(entryPoint.id, entryPointDistance);
|
||||
}
|
||||
// While there are candidates to explore
|
||||
while (candidates.size > 0) {
|
||||
// Get closest candidate
|
||||
const [closestId, closestDist] = [...candidates][0];
|
||||
candidates.delete(closestId);
|
||||
// If this candidate is farther than the farthest in our result set, we're done
|
||||
const farthestInNearest = [...nearest][nearest.size - 1];
|
||||
if (nearest.size >= ef && closestDist > farthestInNearest[1]) {
|
||||
break;
|
||||
}
|
||||
// Explore neighbors of the closest candidate
|
||||
const noun = this.nouns.get(closestId);
|
||||
if (!noun) {
|
||||
console.error(`Noun with ID ${closestId} not found in searchLayer`);
|
||||
continue;
|
||||
}
|
||||
const connections = noun.connections.get(level) || new Set();
|
||||
// If we have enough connections and parallelization is enabled, use parallel distance calculation
|
||||
if (this.useParallelization && connections.size >= 10) {
|
||||
// Collect unvisited neighbors
|
||||
const unvisitedNeighbors = [];
|
||||
for (const neighborId of connections) {
|
||||
if (!visited.has(neighborId)) {
|
||||
visited.add(neighborId);
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor)
|
||||
continue;
|
||||
unvisitedNeighbors.push({ id: neighborId, vector: neighbor.vector });
|
||||
}
|
||||
}
|
||||
if (unvisitedNeighbors.length > 0) {
|
||||
// Calculate distances in parallel
|
||||
const distances = await this.calculateDistancesInParallel(queryVector, unvisitedNeighbors);
|
||||
// Process the results
|
||||
for (const { id, distance } of distances) {
|
||||
// Apply filter if provided
|
||||
const passes = filter ? await filter(id) : true;
|
||||
// Always add to candidates for graph traversal
|
||||
candidates.set(id, distance);
|
||||
// Only add to nearest if it passes the filter
|
||||
if (passes) {
|
||||
// If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
|
||||
if (nearest.size < ef || distance < farthestInNearest[1]) {
|
||||
nearest.set(id, distance);
|
||||
// If we have more than ef neighbors, remove the farthest one
|
||||
if (nearest.size > ef) {
|
||||
const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1]);
|
||||
nearest.clear();
|
||||
for (let i = 0; i < ef; i++) {
|
||||
nearest.set(sortedNearest[i][0], sortedNearest[i][1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Use sequential processing for small number of connections
|
||||
for (const neighborId of connections) {
|
||||
if (!visited.has(neighborId)) {
|
||||
visited.add(neighborId);
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor) {
|
||||
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
||||
continue;
|
||||
}
|
||||
const distToNeighbor = this.distanceFunction(queryVector, neighbor.vector);
|
||||
// Apply filter if provided
|
||||
const passes = filter ? await filter(neighborId) : true;
|
||||
// Always add to candidates for graph traversal
|
||||
candidates.set(neighborId, distToNeighbor);
|
||||
// Only add to nearest if it passes the filter
|
||||
if (passes) {
|
||||
// If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
|
||||
if (nearest.size < ef || distToNeighbor < farthestInNearest[1]) {
|
||||
nearest.set(neighborId, distToNeighbor);
|
||||
// If we have more than ef neighbors, remove the farthest one
|
||||
if (nearest.size > ef) {
|
||||
const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1]);
|
||||
nearest.clear();
|
||||
for (let i = 0; i < ef; i++) {
|
||||
nearest.set(sortedNearest[i][0], sortedNearest[i][1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort nearest by distance
|
||||
return new Map([...nearest].sort((a, b) => a[1] - b[1]));
|
||||
}
|
||||
/**
|
||||
* Select M nearest neighbors from the candidate set
|
||||
*/
|
||||
selectNeighbors(queryVector, candidates, M) {
|
||||
if (candidates.size <= M) {
|
||||
return candidates;
|
||||
}
|
||||
// Simple heuristic: just take the M closest
|
||||
const sortedCandidates = [...candidates].sort((a, b) => a[1] - b[1]);
|
||||
const result = new Map();
|
||||
for (let i = 0; i < Math.min(M, sortedCandidates.length); i++) {
|
||||
result.set(sortedCandidates[i][0], sortedCandidates[i][1]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Ensure a noun doesn't have too many connections at a given level
|
||||
*/
|
||||
pruneConnections(noun, level) {
|
||||
const connections = noun.connections.get(level);
|
||||
if (connections.size <= this.config.M) {
|
||||
return;
|
||||
}
|
||||
// Calculate distances to all neighbors
|
||||
const distances = new Map();
|
||||
const validNeighborIds = new Set();
|
||||
for (const neighborId of connections) {
|
||||
const neighbor = this.nouns.get(neighborId);
|
||||
if (!neighbor) {
|
||||
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
||||
continue;
|
||||
}
|
||||
// Only add valid neighbors to the distances map
|
||||
distances.set(neighborId, this.distanceFunction(noun.vector, neighbor.vector));
|
||||
validNeighborIds.add(neighborId);
|
||||
}
|
||||
// Only proceed if we have valid neighbors
|
||||
if (distances.size === 0) {
|
||||
// If no valid neighbors, clear connections at this level
|
||||
noun.connections.set(level, new Set());
|
||||
return;
|
||||
}
|
||||
// Select M closest neighbors from valid ones
|
||||
const selectedNeighbors = this.selectNeighbors(noun.vector, distances, this.config.M);
|
||||
// Update connections with only valid neighbors
|
||||
noun.connections.set(level, new Set(selectedNeighbors.keys()));
|
||||
}
|
||||
/**
|
||||
* Generate a random level for a new noun
|
||||
* Uses the same distribution as in the original HNSW paper
|
||||
*/
|
||||
getRandomLevel() {
|
||||
const r = Math.random();
|
||||
return Math.floor(-Math.log(r) * (1.0 / Math.log(this.config.M)));
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=hnswIndex.js.map
|
||||
File diff suppressed because one or more lines are too long
179
.recovery-workspace/dist-backup-20250910-141917/hnsw/hnswIndexOptimized.d.ts
vendored
Normal file
179
.recovery-workspace/dist-backup-20250910-141917/hnsw/hnswIndexOptimized.d.ts
vendored
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
/**
|
||||
* Optimized HNSW (Hierarchical Navigable Small World) Index implementation
|
||||
* Extends the base HNSW implementation with support for large datasets
|
||||
* Uses product quantization for dimensionality reduction and disk-based storage when needed
|
||||
*/
|
||||
import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
|
||||
import { HNSWIndex } from './hnswIndex.js';
|
||||
import { StorageAdapter } from '../coreTypes.js';
|
||||
export interface HNSWOptimizedConfig extends HNSWConfig {
|
||||
memoryThreshold?: number;
|
||||
productQuantization?: {
|
||||
enabled: boolean;
|
||||
numSubvectors?: number;
|
||||
numCentroids?: number;
|
||||
};
|
||||
useDiskBasedIndex?: boolean;
|
||||
}
|
||||
/**
|
||||
* Product Quantization implementation
|
||||
* Reduces vector dimensionality by splitting vectors into subvectors
|
||||
* and quantizing each subvector to the nearest centroid
|
||||
*/
|
||||
declare class ProductQuantizer {
|
||||
private numSubvectors;
|
||||
private numCentroids;
|
||||
private centroids;
|
||||
private subvectorSize;
|
||||
private initialized;
|
||||
private dimension;
|
||||
constructor(numSubvectors?: number, numCentroids?: number);
|
||||
/**
|
||||
* Initialize the product quantizer with training data
|
||||
* @param vectors Training vectors to use for learning centroids
|
||||
*/
|
||||
train(vectors: Vector[]): void;
|
||||
/**
|
||||
* Quantize a vector using product quantization
|
||||
* @param vector Vector to quantize
|
||||
* @returns Array of centroid indices, one for each subvector
|
||||
*/
|
||||
quantize(vector: Vector): number[];
|
||||
/**
|
||||
* Reconstruct a vector from its quantized representation
|
||||
* @param codes Array of centroid indices
|
||||
* @returns Reconstructed vector
|
||||
*/
|
||||
reconstruct(codes: number[]): Vector;
|
||||
/**
|
||||
* Compute squared Euclidean distance between two vectors
|
||||
* @param a First vector
|
||||
* @param b Second vector
|
||||
* @returns Squared Euclidean distance
|
||||
*/
|
||||
private euclideanDistanceSquared;
|
||||
/**
|
||||
* Implement k-means++ algorithm to initialize centroids
|
||||
* @param vectors Vectors to cluster
|
||||
* @param k Number of clusters
|
||||
* @returns Array of centroids
|
||||
*/
|
||||
private kMeansPlusPlus;
|
||||
/**
|
||||
* Get the centroids for each subvector
|
||||
* @returns Array of centroid arrays
|
||||
*/
|
||||
getCentroids(): Vector[][];
|
||||
/**
|
||||
* Set the centroids for each subvector
|
||||
* @param centroids Array of centroid arrays
|
||||
*/
|
||||
setCentroids(centroids: Vector[][]): void;
|
||||
/**
|
||||
* Get the dimension of the vectors
|
||||
* @returns Dimension
|
||||
*/
|
||||
getDimension(): number;
|
||||
/**
|
||||
* Set the dimension of the vectors
|
||||
* @param dimension Dimension
|
||||
*/
|
||||
setDimension(dimension: number): void;
|
||||
}
|
||||
/**
|
||||
* Optimized HNSW Index implementation
|
||||
* Extends the base HNSW implementation with support for large datasets
|
||||
* Uses product quantization for dimensionality reduction and disk-based storage when needed
|
||||
*/
|
||||
export declare class HNSWIndexOptimized extends HNSWIndex {
|
||||
private optimizedConfig;
|
||||
private productQuantizer;
|
||||
private storage;
|
||||
private useDiskBasedIndex;
|
||||
private useProductQuantization;
|
||||
private quantizedVectors;
|
||||
private memoryUsage;
|
||||
private vectorCount;
|
||||
private memoryUpdateLock;
|
||||
private unifiedCache;
|
||||
constructor(config: Partial<HNSWOptimizedConfig> | undefined, distanceFunction: DistanceFunction, storage?: StorageAdapter | null);
|
||||
/**
|
||||
* Thread-safe method to update memory usage
|
||||
* @param memoryDelta Change in memory usage (can be negative)
|
||||
* @param vectorCountDelta Change in vector count (can be negative)
|
||||
*/
|
||||
private updateMemoryUsage;
|
||||
/**
|
||||
* Thread-safe method to get current memory usage
|
||||
* @returns Current memory usage and vector count
|
||||
*/
|
||||
private getMemoryUsageAsync;
|
||||
/**
|
||||
* Add a vector to the index
|
||||
* Uses product quantization if enabled and memory threshold is exceeded
|
||||
*/
|
||||
addItem(item: VectorDocument): Promise<string>;
|
||||
/**
|
||||
* Search for nearest neighbors
|
||||
* Uses product quantization if enabled
|
||||
*/
|
||||
search(queryVector: Vector, k?: number): Promise<Array<[string, number]>>;
|
||||
/**
|
||||
* Remove an item from the index
|
||||
*/
|
||||
removeItem(id: string): boolean;
|
||||
/**
|
||||
* Clear the index
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
/**
|
||||
* Initialize product quantizer with existing vectors
|
||||
*/
|
||||
private initializeProductQuantizer;
|
||||
/**
|
||||
* Get the product quantizer
|
||||
* @returns Product quantizer or null if not enabled
|
||||
*/
|
||||
getProductQuantizer(): ProductQuantizer | null;
|
||||
/**
|
||||
* Get the optimized configuration
|
||||
* @returns Optimized configuration
|
||||
*/
|
||||
getOptimizedConfig(): HNSWOptimizedConfig;
|
||||
/**
|
||||
* Get the estimated memory usage
|
||||
* @returns Estimated memory usage in bytes
|
||||
*/
|
||||
getMemoryUsage(): number;
|
||||
/**
|
||||
* Set the storage adapter
|
||||
* @param storage Storage adapter
|
||||
*/
|
||||
setStorage(storage: StorageAdapter): void;
|
||||
/**
|
||||
* Get the storage adapter
|
||||
* @returns Storage adapter or null if not set
|
||||
*/
|
||||
getStorage(): StorageAdapter | null;
|
||||
/**
|
||||
* Set whether to use disk-based index
|
||||
* @param useDiskBasedIndex Whether to use disk-based index
|
||||
*/
|
||||
setUseDiskBasedIndex(useDiskBasedIndex: boolean): void;
|
||||
/**
|
||||
* Get whether disk-based index is used
|
||||
* @returns Whether disk-based index is used
|
||||
*/
|
||||
getUseDiskBasedIndex(): boolean;
|
||||
/**
|
||||
* Set whether to use product quantization
|
||||
* @param useProductQuantization Whether to use product quantization
|
||||
*/
|
||||
setUseProductQuantization(useProductQuantization: boolean): void;
|
||||
/**
|
||||
* Get whether product quantization is used
|
||||
* @returns Whether product quantization is used
|
||||
*/
|
||||
getUseProductQuantization(): boolean;
|
||||
}
|
||||
export {};
|
||||
|
|
@ -0,0 +1,474 @@
|
|||
/**
|
||||
* Optimized HNSW (Hierarchical Navigable Small World) Index implementation
|
||||
* Extends the base HNSW implementation with support for large datasets
|
||||
* Uses product quantization for dimensionality reduction and disk-based storage when needed
|
||||
*/
|
||||
import { HNSWIndex } from './hnswIndex.js';
|
||||
import { getGlobalCache } from '../utils/unifiedCache.js';
|
||||
// Default configuration for the optimized HNSW index
|
||||
const DEFAULT_OPTIMIZED_CONFIG = {
|
||||
M: 16,
|
||||
efConstruction: 200,
|
||||
efSearch: 50,
|
||||
ml: 16,
|
||||
memoryThreshold: 1024 * 1024 * 1024, // 1GB default threshold
|
||||
productQuantization: {
|
||||
enabled: false,
|
||||
numSubvectors: 16,
|
||||
numCentroids: 256
|
||||
},
|
||||
useDiskBasedIndex: false
|
||||
};
|
||||
/**
|
||||
* Product Quantization implementation
|
||||
* Reduces vector dimensionality by splitting vectors into subvectors
|
||||
* and quantizing each subvector to the nearest centroid
|
||||
*/
|
||||
class ProductQuantizer {
|
||||
constructor(numSubvectors = 16, numCentroids = 256) {
|
||||
this.centroids = [];
|
||||
this.subvectorSize = 0;
|
||||
this.initialized = false;
|
||||
this.dimension = 0;
|
||||
this.numSubvectors = numSubvectors;
|
||||
this.numCentroids = numCentroids;
|
||||
}
|
||||
/**
|
||||
* Initialize the product quantizer with training data
|
||||
* @param vectors Training vectors to use for learning centroids
|
||||
*/
|
||||
train(vectors) {
|
||||
if (vectors.length === 0) {
|
||||
throw new Error('Cannot train product quantizer with empty vector set');
|
||||
}
|
||||
this.dimension = vectors[0].length;
|
||||
this.subvectorSize = Math.ceil(this.dimension / this.numSubvectors);
|
||||
// Initialize centroids for each subvector
|
||||
for (let i = 0; i < this.numSubvectors; i++) {
|
||||
// Extract subvectors from training data
|
||||
const subvectors = vectors.map((vector) => {
|
||||
const start = i * this.subvectorSize;
|
||||
const end = Math.min(start + this.subvectorSize, this.dimension);
|
||||
return vector.slice(start, end);
|
||||
});
|
||||
// Initialize centroids for this subvector using k-means++
|
||||
this.centroids[i] = this.kMeansPlusPlus(subvectors, this.numCentroids);
|
||||
}
|
||||
this.initialized = true;
|
||||
}
|
||||
/**
|
||||
* Quantize a vector using product quantization
|
||||
* @param vector Vector to quantize
|
||||
* @returns Array of centroid indices, one for each subvector
|
||||
*/
|
||||
quantize(vector) {
|
||||
if (!this.initialized) {
|
||||
throw new Error('Product quantizer not initialized. Call train() first.');
|
||||
}
|
||||
if (vector.length !== this.dimension) {
|
||||
throw new Error(`Vector dimension mismatch: expected ${this.dimension}, got ${vector.length}`);
|
||||
}
|
||||
const codes = [];
|
||||
// Quantize each subvector
|
||||
for (let i = 0; i < this.numSubvectors; i++) {
|
||||
const start = i * this.subvectorSize;
|
||||
const end = Math.min(start + this.subvectorSize, this.dimension);
|
||||
const subvector = vector.slice(start, end);
|
||||
// Find nearest centroid
|
||||
let minDist = Number.MAX_VALUE;
|
||||
let nearestCentroidIndex = 0;
|
||||
for (let j = 0; j < this.centroids[i].length; j++) {
|
||||
const centroid = this.centroids[i][j];
|
||||
const dist = this.euclideanDistanceSquared(subvector, centroid);
|
||||
if (dist < minDist) {
|
||||
minDist = dist;
|
||||
nearestCentroidIndex = j;
|
||||
}
|
||||
}
|
||||
codes.push(nearestCentroidIndex);
|
||||
}
|
||||
return codes;
|
||||
}
|
||||
/**
|
||||
* Reconstruct a vector from its quantized representation
|
||||
* @param codes Array of centroid indices
|
||||
* @returns Reconstructed vector
|
||||
*/
|
||||
reconstruct(codes) {
|
||||
if (!this.initialized) {
|
||||
throw new Error('Product quantizer not initialized. Call train() first.');
|
||||
}
|
||||
if (codes.length !== this.numSubvectors) {
|
||||
throw new Error(`Code length mismatch: expected ${this.numSubvectors}, got ${codes.length}`);
|
||||
}
|
||||
const reconstructed = [];
|
||||
// Reconstruct each subvector
|
||||
for (let i = 0; i < this.numSubvectors; i++) {
|
||||
const centroidIndex = codes[i];
|
||||
const centroid = this.centroids[i][centroidIndex];
|
||||
// Add centroid components to reconstructed vector
|
||||
for (const component of centroid) {
|
||||
reconstructed.push(component);
|
||||
}
|
||||
}
|
||||
// Trim to original dimension if needed
|
||||
return reconstructed.slice(0, this.dimension);
|
||||
}
|
||||
/**
|
||||
* Compute squared Euclidean distance between two vectors
|
||||
* @param a First vector
|
||||
* @param b Second vector
|
||||
* @returns Squared Euclidean distance
|
||||
*/
|
||||
euclideanDistanceSquared(a, b) {
|
||||
let sum = 0;
|
||||
const length = Math.min(a.length, b.length);
|
||||
for (let i = 0; i < length; i++) {
|
||||
const diff = a[i] - b[i];
|
||||
sum += diff * diff;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
/**
|
||||
* Implement k-means++ algorithm to initialize centroids
|
||||
* @param vectors Vectors to cluster
|
||||
* @param k Number of clusters
|
||||
* @returns Array of centroids
|
||||
*/
|
||||
kMeansPlusPlus(vectors, k) {
|
||||
if (vectors.length < k) {
|
||||
// If we have fewer vectors than centroids, use the vectors as centroids
|
||||
return [...vectors];
|
||||
}
|
||||
const centroids = [];
|
||||
// Choose first centroid randomly
|
||||
const firstIndex = Math.floor(Math.random() * vectors.length);
|
||||
centroids.push([...vectors[firstIndex]]);
|
||||
// Choose remaining centroids
|
||||
for (let i = 1; i < k; i++) {
|
||||
// Compute distances to nearest centroid for each vector
|
||||
const distances = vectors.map((vector) => {
|
||||
let minDist = Number.MAX_VALUE;
|
||||
for (const centroid of centroids) {
|
||||
const dist = this.euclideanDistanceSquared(vector, centroid);
|
||||
minDist = Math.min(minDist, dist);
|
||||
}
|
||||
return minDist;
|
||||
});
|
||||
// Compute sum of distances
|
||||
const distSum = distances.reduce((sum, dist) => sum + dist, 0);
|
||||
// Choose next centroid with probability proportional to distance
|
||||
let r = Math.random() * distSum;
|
||||
let nextIndex = 0;
|
||||
for (let j = 0; j < distances.length; j++) {
|
||||
r -= distances[j];
|
||||
if (r <= 0) {
|
||||
nextIndex = j;
|
||||
break;
|
||||
}
|
||||
}
|
||||
centroids.push([...vectors[nextIndex]]);
|
||||
}
|
||||
return centroids;
|
||||
}
|
||||
/**
|
||||
* Get the centroids for each subvector
|
||||
* @returns Array of centroid arrays
|
||||
*/
|
||||
getCentroids() {
|
||||
return this.centroids;
|
||||
}
|
||||
/**
|
||||
* Set the centroids for each subvector
|
||||
* @param centroids Array of centroid arrays
|
||||
*/
|
||||
setCentroids(centroids) {
|
||||
this.centroids = centroids;
|
||||
this.numSubvectors = centroids.length;
|
||||
this.numCentroids = centroids[0].length;
|
||||
this.initialized = true;
|
||||
}
|
||||
/**
|
||||
* Get the dimension of the vectors
|
||||
* @returns Dimension
|
||||
*/
|
||||
getDimension() {
|
||||
return this.dimension;
|
||||
}
|
||||
/**
|
||||
* Set the dimension of the vectors
|
||||
* @param dimension Dimension
|
||||
*/
|
||||
setDimension(dimension) {
|
||||
this.dimension = dimension;
|
||||
this.subvectorSize = Math.ceil(dimension / this.numSubvectors);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Optimized HNSW Index implementation
|
||||
* Extends the base HNSW implementation with support for large datasets
|
||||
* Uses product quantization for dimensionality reduction and disk-based storage when needed
|
||||
*/
|
||||
export class HNSWIndexOptimized extends HNSWIndex {
|
||||
constructor(config = {}, distanceFunction, storage = null) {
|
||||
// Initialize base HNSW index with standard config
|
||||
super(config, distanceFunction);
|
||||
this.productQuantizer = null;
|
||||
this.storage = null;
|
||||
this.useDiskBasedIndex = false;
|
||||
this.useProductQuantization = false;
|
||||
this.quantizedVectors = new Map();
|
||||
this.memoryUsage = 0;
|
||||
this.vectorCount = 0;
|
||||
// Thread safety for memory usage tracking
|
||||
this.memoryUpdateLock = Promise.resolve();
|
||||
// Set optimized config
|
||||
this.optimizedConfig = { ...DEFAULT_OPTIMIZED_CONFIG, ...config };
|
||||
// Set storage adapter
|
||||
this.storage = storage;
|
||||
// Initialize product quantizer if enabled
|
||||
if (this.optimizedConfig.productQuantization?.enabled) {
|
||||
this.useProductQuantization = true;
|
||||
this.productQuantizer = new ProductQuantizer(this.optimizedConfig.productQuantization.numSubvectors, this.optimizedConfig.productQuantization.numCentroids);
|
||||
}
|
||||
// Set disk-based index flag
|
||||
this.useDiskBasedIndex = this.optimizedConfig.useDiskBasedIndex || false;
|
||||
// Get global unified cache for coordinated memory management
|
||||
this.unifiedCache = getGlobalCache();
|
||||
}
|
||||
/**
|
||||
* Thread-safe method to update memory usage
|
||||
* @param memoryDelta Change in memory usage (can be negative)
|
||||
* @param vectorCountDelta Change in vector count (can be negative)
|
||||
*/
|
||||
async updateMemoryUsage(memoryDelta, vectorCountDelta) {
|
||||
this.memoryUpdateLock = this.memoryUpdateLock.then(async () => {
|
||||
this.memoryUsage = Math.max(0, this.memoryUsage + memoryDelta);
|
||||
this.vectorCount = Math.max(0, this.vectorCount + vectorCountDelta);
|
||||
});
|
||||
await this.memoryUpdateLock;
|
||||
}
|
||||
/**
|
||||
* Thread-safe method to get current memory usage
|
||||
* @returns Current memory usage and vector count
|
||||
*/
|
||||
async getMemoryUsageAsync() {
|
||||
await this.memoryUpdateLock;
|
||||
return {
|
||||
memoryUsage: this.memoryUsage,
|
||||
vectorCount: this.vectorCount
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Add a vector to the index
|
||||
* Uses product quantization if enabled and memory threshold is exceeded
|
||||
*/
|
||||
async addItem(item) {
|
||||
// Check if item is defined
|
||||
if (!item) {
|
||||
throw new Error('Item is undefined or null');
|
||||
}
|
||||
const { id, vector } = item;
|
||||
// Check if vector is defined
|
||||
if (!vector) {
|
||||
throw new Error('Vector is undefined or null');
|
||||
}
|
||||
// Estimate memory usage for this vector
|
||||
const vectorMemory = vector.length * 8; // 8 bytes per number (Float64)
|
||||
const connectionsMemory = this.optimizedConfig.M * this.optimizedConfig.ml * 16; // Estimate for connections
|
||||
const totalMemory = vectorMemory + connectionsMemory;
|
||||
// Update memory usage estimate (thread-safe)
|
||||
await this.updateMemoryUsage(totalMemory, 1);
|
||||
// Check if we should switch to product quantization
|
||||
const currentMemoryUsage = await this.getMemoryUsageAsync();
|
||||
if (this.useProductQuantization &&
|
||||
currentMemoryUsage.memoryUsage > this.optimizedConfig.memoryThreshold &&
|
||||
this.productQuantizer &&
|
||||
!this.productQuantizer.getDimension()) {
|
||||
// Initialize product quantizer with existing vectors
|
||||
this.initializeProductQuantizer();
|
||||
}
|
||||
// If product quantization is active, quantize the vector
|
||||
if (this.useProductQuantization &&
|
||||
this.productQuantizer &&
|
||||
this.productQuantizer.getDimension() > 0) {
|
||||
// Quantize the vector
|
||||
const codes = this.productQuantizer.quantize(vector);
|
||||
// Store the quantized vector
|
||||
this.quantizedVectors.set(id, codes);
|
||||
// Reconstruct the vector for indexing
|
||||
const reconstructedVector = this.productQuantizer.reconstruct(codes);
|
||||
// Add the reconstructed vector to the index
|
||||
return await super.addItem({ id, vector: reconstructedVector });
|
||||
}
|
||||
// If disk-based index is active and storage is available, store the vector
|
||||
if (this.useDiskBasedIndex && this.storage) {
|
||||
// Create a noun object
|
||||
const noun = {
|
||||
id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
};
|
||||
// Store the noun
|
||||
this.storage.saveNoun(noun).catch((error) => {
|
||||
console.error(`Failed to save noun ${id} to storage:`, error);
|
||||
});
|
||||
}
|
||||
// Add the vector to the in-memory index
|
||||
return await super.addItem(item);
|
||||
}
|
||||
/**
|
||||
* Search for nearest neighbors
|
||||
* Uses product quantization if enabled
|
||||
*/
|
||||
async search(queryVector, k = 10) {
|
||||
// Check if query vector is defined
|
||||
if (!queryVector) {
|
||||
throw new Error('Query vector is undefined or null');
|
||||
}
|
||||
// If product quantization is active, quantize the query vector
|
||||
if (this.useProductQuantization &&
|
||||
this.productQuantizer &&
|
||||
this.productQuantizer.getDimension() > 0) {
|
||||
// Quantize the query vector
|
||||
const codes = this.productQuantizer.quantize(queryVector);
|
||||
// Reconstruct the query vector
|
||||
const reconstructedVector = this.productQuantizer.reconstruct(codes);
|
||||
// Search with the reconstructed vector
|
||||
return await super.search(reconstructedVector, k);
|
||||
}
|
||||
// Otherwise, use the standard search
|
||||
return await super.search(queryVector, k);
|
||||
}
|
||||
/**
|
||||
* Remove an item from the index
|
||||
*/
|
||||
removeItem(id) {
|
||||
// If product quantization is active, remove the quantized vector
|
||||
if (this.useProductQuantization) {
|
||||
this.quantizedVectors.delete(id);
|
||||
}
|
||||
// If disk-based index is active and storage is available, remove the vector from storage
|
||||
if (this.useDiskBasedIndex && this.storage) {
|
||||
this.storage.deleteNoun(id).catch((error) => {
|
||||
console.error(`Failed to delete noun ${id} from storage:`, error);
|
||||
});
|
||||
}
|
||||
// Update memory usage estimate (async operation, but don't block removal)
|
||||
this.getMemoryUsageAsync().then((currentMemoryUsage) => {
|
||||
if (currentMemoryUsage.vectorCount > 0) {
|
||||
const memoryPerVector = currentMemoryUsage.memoryUsage / currentMemoryUsage.vectorCount;
|
||||
this.updateMemoryUsage(-memoryPerVector, -1);
|
||||
}
|
||||
}).catch((error) => {
|
||||
console.error('Failed to update memory usage after removal:', error);
|
||||
});
|
||||
// Remove the item from the in-memory index
|
||||
return super.removeItem(id);
|
||||
}
|
||||
/**
|
||||
* Clear the index
|
||||
*/
|
||||
async clear() {
|
||||
// Clear product quantization data
|
||||
if (this.useProductQuantization) {
|
||||
this.quantizedVectors.clear();
|
||||
this.productQuantizer = new ProductQuantizer(this.optimizedConfig.productQuantization.numSubvectors, this.optimizedConfig.productQuantization.numCentroids);
|
||||
}
|
||||
// Reset memory usage (thread-safe)
|
||||
const currentMemoryUsage = await this.getMemoryUsageAsync();
|
||||
await this.updateMemoryUsage(-currentMemoryUsage.memoryUsage, -currentMemoryUsage.vectorCount);
|
||||
// Clear the in-memory index
|
||||
super.clear();
|
||||
}
|
||||
/**
|
||||
* Initialize product quantizer with existing vectors
|
||||
*/
|
||||
initializeProductQuantizer() {
|
||||
if (!this.productQuantizer) {
|
||||
return;
|
||||
}
|
||||
// Get all vectors from the index
|
||||
const nouns = super.getNouns();
|
||||
const vectors = [];
|
||||
// Extract vectors
|
||||
for (const [_, noun] of nouns) {
|
||||
vectors.push(noun.vector);
|
||||
}
|
||||
// Train the product quantizer
|
||||
if (vectors.length > 0) {
|
||||
this.productQuantizer.train(vectors);
|
||||
// Quantize all existing vectors
|
||||
for (const [id, noun] of nouns) {
|
||||
const codes = this.productQuantizer.quantize(noun.vector);
|
||||
this.quantizedVectors.set(id, codes);
|
||||
}
|
||||
console.log(`Initialized product quantizer with ${vectors.length} vectors`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get the product quantizer
|
||||
* @returns Product quantizer or null if not enabled
|
||||
*/
|
||||
getProductQuantizer() {
|
||||
return this.productQuantizer;
|
||||
}
|
||||
/**
|
||||
* Get the optimized configuration
|
||||
* @returns Optimized configuration
|
||||
*/
|
||||
getOptimizedConfig() {
|
||||
return { ...this.optimizedConfig };
|
||||
}
|
||||
/**
|
||||
* Get the estimated memory usage
|
||||
* @returns Estimated memory usage in bytes
|
||||
*/
|
||||
getMemoryUsage() {
|
||||
return this.memoryUsage;
|
||||
}
|
||||
/**
|
||||
* Set the storage adapter
|
||||
* @param storage Storage adapter
|
||||
*/
|
||||
setStorage(storage) {
|
||||
this.storage = storage;
|
||||
}
|
||||
/**
|
||||
* Get the storage adapter
|
||||
* @returns Storage adapter or null if not set
|
||||
*/
|
||||
getStorage() {
|
||||
return this.storage;
|
||||
}
|
||||
/**
|
||||
* Set whether to use disk-based index
|
||||
* @param useDiskBasedIndex Whether to use disk-based index
|
||||
*/
|
||||
setUseDiskBasedIndex(useDiskBasedIndex) {
|
||||
this.useDiskBasedIndex = useDiskBasedIndex;
|
||||
}
|
||||
/**
|
||||
* Get whether disk-based index is used
|
||||
* @returns Whether disk-based index is used
|
||||
*/
|
||||
getUseDiskBasedIndex() {
|
||||
return this.useDiskBasedIndex;
|
||||
}
|
||||
/**
|
||||
* Set whether to use product quantization
|
||||
* @param useProductQuantization Whether to use product quantization
|
||||
*/
|
||||
setUseProductQuantization(useProductQuantization) {
|
||||
this.useProductQuantization = useProductQuantization;
|
||||
}
|
||||
/**
|
||||
* Get whether product quantization is used
|
||||
* @returns Whether product quantization is used
|
||||
*/
|
||||
getUseProductQuantization() {
|
||||
return this.useProductQuantization;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=hnswIndexOptimized.js.map
|
||||
File diff suppressed because one or more lines are too long
97
.recovery-workspace/dist-backup-20250910-141917/hnsw/optimizedHNSWIndex.d.ts
vendored
Normal file
97
.recovery-workspace/dist-backup-20250910-141917/hnsw/optimizedHNSWIndex.d.ts
vendored
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
/**
|
||||
* Optimized HNSW Index for Large-Scale Vector Search
|
||||
* Implements dynamic parameter tuning and performance optimizations
|
||||
*/
|
||||
import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
|
||||
import { HNSWIndex } from './hnswIndex.js';
|
||||
export interface OptimizedHNSWConfig extends HNSWConfig {
|
||||
dynamicParameterTuning?: boolean;
|
||||
targetSearchLatency?: number;
|
||||
targetRecall?: number;
|
||||
maxNodes?: number;
|
||||
memoryBudget?: number;
|
||||
diskCacheEnabled?: boolean;
|
||||
compressionEnabled?: boolean;
|
||||
performanceTracking?: boolean;
|
||||
adaptiveEfSearch?: boolean;
|
||||
levelMultiplier?: number;
|
||||
seedConnections?: number;
|
||||
pruningStrategy?: 'simple' | 'diverse' | 'hybrid';
|
||||
}
|
||||
interface PerformanceMetrics {
|
||||
averageSearchTime: number;
|
||||
averageRecall: number;
|
||||
memoryUsage: number;
|
||||
indexSize: number;
|
||||
apiCalls: number;
|
||||
cacheHitRate: number;
|
||||
}
|
||||
interface DynamicParameters {
|
||||
efSearch: number;
|
||||
efConstruction: number;
|
||||
M: number;
|
||||
ml: number;
|
||||
}
|
||||
/**
|
||||
* Optimized HNSW Index with dynamic parameter tuning for large datasets
|
||||
*/
|
||||
export declare class OptimizedHNSWIndex extends HNSWIndex {
|
||||
private optimizedConfig;
|
||||
private performanceMetrics;
|
||||
private dynamicParams;
|
||||
private searchHistory;
|
||||
private parameterTuningInterval?;
|
||||
constructor(config?: Partial<OptimizedHNSWConfig>, distanceFunction?: DistanceFunction);
|
||||
/**
|
||||
* Optimized search with dynamic parameter adjustment
|
||||
*/
|
||||
search(queryVector: Vector, k?: number, filter?: (id: string) => Promise<boolean>): Promise<Array<[string, number]>>;
|
||||
/**
|
||||
* Dynamically adjust efSearch based on performance requirements
|
||||
*/
|
||||
private adjustEfSearch;
|
||||
/**
|
||||
* Record search performance metrics
|
||||
*/
|
||||
private recordSearchMetrics;
|
||||
/**
|
||||
* Check memory usage and trigger optimizations
|
||||
*/
|
||||
private checkMemoryUsage;
|
||||
/**
|
||||
* Compress index to reduce memory usage (placeholder)
|
||||
*/
|
||||
private compressIndex;
|
||||
/**
|
||||
* Start automatic parameter tuning
|
||||
*/
|
||||
private startParameterTuning;
|
||||
/**
|
||||
* Automatic parameter tuning based on performance metrics
|
||||
*/
|
||||
private tuneParameters;
|
||||
/**
|
||||
* Get optimized configuration recommendations for current dataset size
|
||||
*/
|
||||
getOptimizedConfig(): OptimizedHNSWConfig;
|
||||
/**
|
||||
* Get current performance metrics
|
||||
*/
|
||||
getPerformanceMetrics(): PerformanceMetrics & {
|
||||
currentParams: DynamicParameters;
|
||||
searchHistorySize: number;
|
||||
};
|
||||
/**
|
||||
* Apply optimized bulk insertion strategy
|
||||
*/
|
||||
bulkInsert(items: VectorDocument[]): Promise<string[]>;
|
||||
/**
|
||||
* Optimize insertion order to improve index quality
|
||||
*/
|
||||
private optimizeInsertionOrder;
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
destroy(): void;
|
||||
}
|
||||
export {};
|
||||
|
|
@ -0,0 +1,313 @@
|
|||
/**
|
||||
* Optimized HNSW Index for Large-Scale Vector Search
|
||||
* Implements dynamic parameter tuning and performance optimizations
|
||||
*/
|
||||
import { HNSWIndex } from './hnswIndex.js';
|
||||
import { euclideanDistance } from '../utils/index.js';
|
||||
/**
|
||||
* Optimized HNSW Index with dynamic parameter tuning for large datasets
|
||||
*/
|
||||
export class OptimizedHNSWIndex extends HNSWIndex {
|
||||
constructor(config = {}, distanceFunction = euclideanDistance) {
|
||||
// Set optimized defaults for large scale
|
||||
const defaultConfig = {
|
||||
M: 32, // Higher connectivity for better recall
|
||||
efConstruction: 400, // Better build quality
|
||||
efSearch: 100, // Dynamic - will be tuned
|
||||
ml: 24, // Deeper hierarchy
|
||||
useDiskBasedIndex: false, // Added missing property
|
||||
dynamicParameterTuning: true,
|
||||
targetSearchLatency: 100, // 100ms target
|
||||
targetRecall: 0.95, // 95% recall target
|
||||
maxNodes: 1000000, // 1M node limit
|
||||
memoryBudget: 8 * 1024 * 1024 * 1024, // 8GB
|
||||
diskCacheEnabled: true,
|
||||
compressionEnabled: false, // Disabled by default for compatibility
|
||||
performanceTracking: true,
|
||||
adaptiveEfSearch: true,
|
||||
levelMultiplier: 16,
|
||||
seedConnections: 8,
|
||||
pruningStrategy: 'hybrid'
|
||||
};
|
||||
const mergedConfig = { ...defaultConfig, ...config };
|
||||
// Initialize parent with base config
|
||||
super({
|
||||
M: mergedConfig.M,
|
||||
efConstruction: mergedConfig.efConstruction,
|
||||
efSearch: mergedConfig.efSearch,
|
||||
ml: mergedConfig.ml
|
||||
}, distanceFunction, { useParallelization: true });
|
||||
this.searchHistory = [];
|
||||
this.optimizedConfig = mergedConfig;
|
||||
// Initialize dynamic parameters
|
||||
this.dynamicParams = {
|
||||
efSearch: mergedConfig.efSearch,
|
||||
efConstruction: mergedConfig.efConstruction,
|
||||
M: mergedConfig.M,
|
||||
ml: mergedConfig.ml
|
||||
};
|
||||
// Initialize performance metrics
|
||||
this.performanceMetrics = {
|
||||
averageSearchTime: 0,
|
||||
averageRecall: 0,
|
||||
memoryUsage: 0,
|
||||
indexSize: 0,
|
||||
apiCalls: 0,
|
||||
cacheHitRate: 0
|
||||
};
|
||||
// Start parameter tuning if enabled
|
||||
if (this.optimizedConfig.dynamicParameterTuning) {
|
||||
this.startParameterTuning();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Optimized search with dynamic parameter adjustment
|
||||
*/
|
||||
async search(queryVector, k = 10, filter) {
|
||||
const startTime = Date.now();
|
||||
// Adjust efSearch dynamically based on k and performance history
|
||||
if (this.optimizedConfig.adaptiveEfSearch) {
|
||||
this.adjustEfSearch(k);
|
||||
}
|
||||
// Check memory usage and trigger optimizations if needed
|
||||
if (this.optimizedConfig.performanceTracking) {
|
||||
this.checkMemoryUsage();
|
||||
}
|
||||
// Perform the search with current parameters
|
||||
const originalConfig = this.getConfig();
|
||||
// Temporarily update search parameters
|
||||
const tempConfig = {
|
||||
...originalConfig,
|
||||
efSearch: this.dynamicParams.efSearch
|
||||
};
|
||||
// Use the parent's search method with optimized parameters
|
||||
let results;
|
||||
try {
|
||||
// This is a simplified approach - in practice, we'd need to modify
|
||||
// the parent class to accept runtime parameter changes
|
||||
results = await super.search(queryVector, k, filter);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Optimized search failed, falling back to default:', error);
|
||||
results = await super.search(queryVector, k, filter);
|
||||
}
|
||||
// Record performance metrics
|
||||
const searchTime = Date.now() - startTime;
|
||||
this.recordSearchMetrics(searchTime, k, results.length);
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Dynamically adjust efSearch based on performance requirements
|
||||
*/
|
||||
adjustEfSearch(k) {
|
||||
const recentSearches = this.searchHistory.slice(-10);
|
||||
if (recentSearches.length < 3) {
|
||||
// Not enough data, use heuristic
|
||||
this.dynamicParams.efSearch = Math.max(k * 2, 50);
|
||||
return;
|
||||
}
|
||||
const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
|
||||
const targetLatency = this.optimizedConfig.targetSearchLatency;
|
||||
// Adjust efSearch based on latency performance
|
||||
if (averageLatency > targetLatency * 1.2) {
|
||||
// Too slow, reduce efSearch
|
||||
this.dynamicParams.efSearch = Math.max(Math.floor(this.dynamicParams.efSearch * 0.9), k);
|
||||
}
|
||||
else if (averageLatency < targetLatency * 0.8) {
|
||||
// Fast enough, can increase efSearch for better recall
|
||||
this.dynamicParams.efSearch = Math.min(Math.floor(this.dynamicParams.efSearch * 1.1), 500 // Maximum efSearch
|
||||
);
|
||||
}
|
||||
// Ensure efSearch is at least k
|
||||
this.dynamicParams.efSearch = Math.max(this.dynamicParams.efSearch, k);
|
||||
}
|
||||
/**
|
||||
* Record search performance metrics
|
||||
*/
|
||||
recordSearchMetrics(latency, k, resultCount) {
|
||||
if (!this.optimizedConfig.performanceTracking) {
|
||||
return;
|
||||
}
|
||||
// Add to search history
|
||||
this.searchHistory.push({
|
||||
latency,
|
||||
k,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
// Keep only recent history (last 100 searches)
|
||||
if (this.searchHistory.length > 100) {
|
||||
this.searchHistory.shift();
|
||||
}
|
||||
// Update performance metrics
|
||||
const recentSearches = this.searchHistory.slice(-20);
|
||||
this.performanceMetrics.averageSearchTime =
|
||||
recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
|
||||
// Estimate recall (simplified - would need ground truth for accurate measurement)
|
||||
this.performanceMetrics.averageRecall = Math.min(resultCount / k, 1.0);
|
||||
}
|
||||
/**
|
||||
* Check memory usage and trigger optimizations
|
||||
*/
|
||||
checkMemoryUsage() {
|
||||
// Estimate memory usage (simplified)
|
||||
const estimatedMemory = this.size() * 1000; // Rough estimate per node
|
||||
this.performanceMetrics.memoryUsage = estimatedMemory;
|
||||
if (estimatedMemory > this.optimizedConfig.memoryBudget * 0.9) {
|
||||
console.warn('Memory usage approaching limit, consider index partitioning');
|
||||
// Could trigger automatic partitioning or compression here
|
||||
if (this.optimizedConfig.compressionEnabled) {
|
||||
this.compressIndex();
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Compress index to reduce memory usage (placeholder)
|
||||
*/
|
||||
compressIndex() {
|
||||
console.log('Index compression not implemented yet');
|
||||
// This would implement vector quantization or other compression techniques
|
||||
}
|
||||
/**
|
||||
* Start automatic parameter tuning
|
||||
*/
|
||||
startParameterTuning() {
|
||||
this.parameterTuningInterval = setInterval(() => {
|
||||
this.tuneParameters();
|
||||
}, 30000); // Tune every 30 seconds
|
||||
}
|
||||
/**
|
||||
* Automatic parameter tuning based on performance metrics
|
||||
*/
|
||||
tuneParameters() {
|
||||
if (this.searchHistory.length < 10) {
|
||||
return; // Not enough data
|
||||
}
|
||||
const recentSearches = this.searchHistory.slice(-20);
|
||||
const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
|
||||
// Tune based on performance vs targets
|
||||
const latencyRatio = averageLatency / this.optimizedConfig.targetSearchLatency;
|
||||
const recallRatio = this.performanceMetrics.averageRecall / this.optimizedConfig.targetRecall;
|
||||
// Adjust M (connectivity) for long-term performance
|
||||
if (this.size() > 10000) { // Only tune for larger indices
|
||||
if (recallRatio < 0.95 && latencyRatio < 1.5) {
|
||||
// Recall is low but we have latency budget, increase M
|
||||
this.dynamicParams.M = Math.min(this.dynamicParams.M + 2, 64);
|
||||
}
|
||||
else if (latencyRatio > 1.2 && recallRatio > 1.0) {
|
||||
// Latency is high but recall is good, can reduce M
|
||||
this.dynamicParams.M = Math.max(this.dynamicParams.M - 2, 16);
|
||||
}
|
||||
}
|
||||
console.log(`Parameter tuning: efSearch=${this.dynamicParams.efSearch}, M=${this.dynamicParams.M}, latency=${averageLatency.toFixed(1)}ms`);
|
||||
}
|
||||
/**
|
||||
* Get optimized configuration recommendations for current dataset size
|
||||
*/
|
||||
getOptimizedConfig() {
|
||||
const currentSize = this.size();
|
||||
let recommendedConfig = {};
|
||||
if (currentSize < 10000) {
|
||||
// Small dataset - optimize for speed
|
||||
recommendedConfig = {
|
||||
M: 16,
|
||||
efConstruction: 200,
|
||||
efSearch: 50,
|
||||
ml: 16
|
||||
};
|
||||
}
|
||||
else if (currentSize < 100000) {
|
||||
// Medium dataset - balance speed and recall
|
||||
recommendedConfig = {
|
||||
M: 24,
|
||||
efConstruction: 300,
|
||||
efSearch: 75,
|
||||
ml: 20
|
||||
};
|
||||
}
|
||||
else if (currentSize < 1000000) {
|
||||
// Large dataset - optimize for recall
|
||||
recommendedConfig = {
|
||||
M: 32,
|
||||
efConstruction: 400,
|
||||
efSearch: 100,
|
||||
ml: 24
|
||||
};
|
||||
}
|
||||
else {
|
||||
// Very large dataset - maximum quality
|
||||
recommendedConfig = {
|
||||
M: 48,
|
||||
efConstruction: 500,
|
||||
efSearch: 150,
|
||||
ml: 28
|
||||
};
|
||||
}
|
||||
return {
|
||||
...this.optimizedConfig,
|
||||
...recommendedConfig
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get current performance metrics
|
||||
*/
|
||||
getPerformanceMetrics() {
|
||||
return {
|
||||
...this.performanceMetrics,
|
||||
currentParams: { ...this.dynamicParams },
|
||||
searchHistorySize: this.searchHistory.length
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Apply optimized bulk insertion strategy
|
||||
*/
|
||||
async bulkInsert(items) {
|
||||
console.log(`Starting optimized bulk insert of ${items.length} items`);
|
||||
// Sort items to optimize insertion order (by vector similarity)
|
||||
const sortedItems = this.optimizeInsertionOrder(items);
|
||||
// Temporarily adjust construction parameters for bulk operations
|
||||
const originalEfConstruction = this.dynamicParams.efConstruction;
|
||||
this.dynamicParams.efConstruction = Math.min(this.dynamicParams.efConstruction * 1.5, 800);
|
||||
const results = [];
|
||||
const batchSize = 100;
|
||||
try {
|
||||
// Process in batches to manage memory
|
||||
for (let i = 0; i < sortedItems.length; i += batchSize) {
|
||||
const batch = sortedItems.slice(i, i + batchSize);
|
||||
for (const item of batch) {
|
||||
const id = await this.addItem(item);
|
||||
results.push(id);
|
||||
}
|
||||
// Periodic memory check
|
||||
if (i % (batchSize * 10) === 0) {
|
||||
this.checkMemoryUsage();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
// Restore original construction parameters
|
||||
this.dynamicParams.efConstruction = originalEfConstruction;
|
||||
}
|
||||
console.log(`Completed bulk insert of ${results.length} items`);
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Optimize insertion order to improve index quality
|
||||
*/
|
||||
optimizeInsertionOrder(items) {
|
||||
if (items.length < 100) {
|
||||
return items; // Not worth optimizing small batches
|
||||
}
|
||||
// Simple clustering-based ordering
|
||||
// In practice, you might use more sophisticated methods
|
||||
return items.sort(() => Math.random() - 0.5); // Shuffle for now
|
||||
}
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
destroy() {
|
||||
if (this.parameterTuningInterval) {
|
||||
clearInterval(this.parameterTuningInterval);
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=optimizedHNSWIndex.js.map
|
||||
File diff suppressed because one or more lines are too long
101
.recovery-workspace/dist-backup-20250910-141917/hnsw/partitionedHNSWIndex.d.ts
vendored
Normal file
101
.recovery-workspace/dist-backup-20250910-141917/hnsw/partitionedHNSWIndex.d.ts
vendored
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
/**
|
||||
* Partitioned HNSW Index for Large-Scale Vector Search
|
||||
* Implements sharding strategies to handle millions of vectors efficiently
|
||||
*/
|
||||
import { DistanceFunction, HNSWConfig, Vector, VectorDocument } from '../coreTypes.js';
|
||||
export interface PartitionConfig {
|
||||
maxNodesPerPartition: number;
|
||||
partitionStrategy: 'semantic' | 'hash';
|
||||
semanticClusters?: number;
|
||||
autoTuneSemanticClusters?: boolean;
|
||||
}
|
||||
export interface PartitionMetadata {
|
||||
id: string;
|
||||
nodeCount: number;
|
||||
bounds?: {
|
||||
centroid: Vector;
|
||||
radius: number;
|
||||
};
|
||||
strategy: string;
|
||||
created: Date;
|
||||
}
|
||||
/**
|
||||
* Partitioned HNSW Index that splits large datasets across multiple smaller indices
|
||||
* This enables efficient search across millions of vectors by reducing memory usage
|
||||
* and parallelizing search operations
|
||||
*/
|
||||
export declare class PartitionedHNSWIndex {
|
||||
private partitions;
|
||||
private partitionMetadata;
|
||||
private config;
|
||||
private hnswConfig;
|
||||
private distanceFunction;
|
||||
private dimension;
|
||||
private nextPartitionId;
|
||||
constructor(partitionConfig?: Partial<PartitionConfig>, hnswConfig?: Partial<HNSWConfig>, distanceFunction?: DistanceFunction);
|
||||
/**
|
||||
* Add a vector to the partitioned index
|
||||
*/
|
||||
addItem(item: VectorDocument): Promise<string>;
|
||||
/**
|
||||
* Search across all partitions for nearest neighbors
|
||||
*/
|
||||
search(queryVector: Vector, k?: number, searchScope?: {
|
||||
partitionIds?: string[];
|
||||
maxPartitions?: number;
|
||||
}): Promise<Array<[string, number]>>;
|
||||
/**
|
||||
* Select the appropriate partition for a new item
|
||||
* Automatically chooses semantic partitioning when beneficial, falls back to hash
|
||||
*/
|
||||
private selectPartition;
|
||||
/**
|
||||
* Hash-based partitioning for even distribution
|
||||
*/
|
||||
private hashPartition;
|
||||
/**
|
||||
* Semantic clustering partitioning
|
||||
*/
|
||||
private semanticPartition;
|
||||
/**
|
||||
* Auto-tune semantic clusters based on dataset size and performance
|
||||
*/
|
||||
private autoTuneSemanticClusters;
|
||||
/**
|
||||
* Select which partitions to search based on query
|
||||
*/
|
||||
private selectSearchPartitions;
|
||||
/**
|
||||
* Update partition bounds for semantic clustering
|
||||
*/
|
||||
private updatePartitionBounds;
|
||||
/**
|
||||
* Split an overgrown partition into smaller partitions
|
||||
*/
|
||||
private splitPartition;
|
||||
/**
|
||||
* Simple hash function for consistent partitioning
|
||||
*/
|
||||
private simpleHash;
|
||||
/**
|
||||
* Get partition statistics
|
||||
*/
|
||||
getPartitionStats(): {
|
||||
totalPartitions: number;
|
||||
totalNodes: number;
|
||||
averageNodesPerPartition: number;
|
||||
partitionDetails: PartitionMetadata[];
|
||||
};
|
||||
/**
|
||||
* Remove an item from the index
|
||||
*/
|
||||
removeItem(id: string): Promise<boolean>;
|
||||
/**
|
||||
* Clear all partitions
|
||||
*/
|
||||
clear(): void;
|
||||
/**
|
||||
* Get total size across all partitions
|
||||
*/
|
||||
size(): number;
|
||||
}
|
||||
|
|
@ -0,0 +1,304 @@
|
|||
/**
|
||||
* Partitioned HNSW Index for Large-Scale Vector Search
|
||||
* Implements sharding strategies to handle millions of vectors efficiently
|
||||
*/
|
||||
import { HNSWIndex } from './hnswIndex.js';
|
||||
import { euclideanDistance } from '../utils/index.js';
|
||||
/**
|
||||
* Partitioned HNSW Index that splits large datasets across multiple smaller indices
|
||||
* This enables efficient search across millions of vectors by reducing memory usage
|
||||
* and parallelizing search operations
|
||||
*/
|
||||
export class PartitionedHNSWIndex {
|
||||
constructor(partitionConfig = {}, hnswConfig = {}, distanceFunction = euclideanDistance) {
|
||||
this.partitions = new Map();
|
||||
this.partitionMetadata = new Map();
|
||||
this.dimension = null;
|
||||
this.nextPartitionId = 0;
|
||||
this.config = {
|
||||
maxNodesPerPartition: 50000, // Optimal size for memory efficiency
|
||||
partitionStrategy: 'semantic', // Default to semantic for better performance
|
||||
semanticClusters: 8, // Auto-tuned based on dataset
|
||||
autoTuneSemanticClusters: true,
|
||||
...partitionConfig
|
||||
};
|
||||
// Optimized HNSW parameters for large scale
|
||||
this.hnswConfig = {
|
||||
M: 32, // Higher connectivity for better recall
|
||||
efConstruction: 400, // Better build quality
|
||||
efSearch: 100, // Balance speed vs accuracy
|
||||
ml: 24, // Deeper hierarchy
|
||||
...hnswConfig
|
||||
};
|
||||
this.distanceFunction = distanceFunction;
|
||||
}
|
||||
/**
|
||||
* Add a vector to the partitioned index
|
||||
*/
|
||||
async addItem(item) {
|
||||
if (this.dimension === null) {
|
||||
this.dimension = item.vector.length;
|
||||
}
|
||||
// Determine which partition this item belongs to
|
||||
const partitionId = await this.selectPartition(item);
|
||||
// Get or create the partition
|
||||
let partition = this.partitions.get(partitionId);
|
||||
if (!partition) {
|
||||
partition = new HNSWIndex(this.hnswConfig, this.distanceFunction, { useParallelization: true });
|
||||
this.partitions.set(partitionId, partition);
|
||||
// Initialize partition metadata
|
||||
this.partitionMetadata.set(partitionId, {
|
||||
id: partitionId,
|
||||
nodeCount: 0,
|
||||
strategy: this.config.partitionStrategy,
|
||||
created: new Date()
|
||||
});
|
||||
}
|
||||
// Add item to the selected partition
|
||||
await partition.addItem(item);
|
||||
// Update partition metadata
|
||||
const metadata = this.partitionMetadata.get(partitionId);
|
||||
metadata.nodeCount = partition.size();
|
||||
// Update bounds for semantic strategy
|
||||
if (this.config.partitionStrategy === 'semantic') {
|
||||
this.updatePartitionBounds(partitionId, item.vector);
|
||||
}
|
||||
// Check if partition is getting too large and needs splitting
|
||||
if (metadata.nodeCount > this.config.maxNodesPerPartition * 1.2) {
|
||||
await this.splitPartition(partitionId);
|
||||
}
|
||||
return item.id;
|
||||
}
|
||||
/**
|
||||
* Search across all partitions for nearest neighbors
|
||||
*/
|
||||
async search(queryVector, k = 10, searchScope) {
|
||||
if (this.partitions.size === 0) {
|
||||
return [];
|
||||
}
|
||||
// Determine which partitions to search
|
||||
const partitionsToSearch = await this.selectSearchPartitions(queryVector, searchScope);
|
||||
// Search partitions in parallel
|
||||
const searchPromises = partitionsToSearch.map(async (partitionId) => {
|
||||
const partition = this.partitions.get(partitionId);
|
||||
if (!partition)
|
||||
return [];
|
||||
// Search with higher k to get better global results
|
||||
const partitionK = Math.min(k * 2, partition.size());
|
||||
return partition.search(queryVector, partitionK);
|
||||
});
|
||||
const partitionResults = await Promise.all(searchPromises);
|
||||
// Merge and sort results from all partitions
|
||||
const allResults = [];
|
||||
for (const results of partitionResults) {
|
||||
allResults.push(...results);
|
||||
}
|
||||
// Sort by distance and return top k
|
||||
allResults.sort((a, b) => a[1] - b[1]);
|
||||
return allResults.slice(0, k);
|
||||
}
|
||||
/**
|
||||
* Select the appropriate partition for a new item
|
||||
* Automatically chooses semantic partitioning when beneficial, falls back to hash
|
||||
*/
|
||||
async selectPartition(item) {
|
||||
// Auto-tune semantic clusters based on current dataset size
|
||||
if (this.config.autoTuneSemanticClusters && this.config.partitionStrategy === 'semantic') {
|
||||
this.autoTuneSemanticClusters();
|
||||
}
|
||||
switch (this.config.partitionStrategy) {
|
||||
case 'semantic':
|
||||
return await this.semanticPartition(item.vector);
|
||||
case 'hash':
|
||||
default:
|
||||
return this.hashPartition(item.id);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Hash-based partitioning for even distribution
|
||||
*/
|
||||
hashPartition(id) {
|
||||
const hash = this.simpleHash(id);
|
||||
const existingPartitions = Array.from(this.partitions.keys());
|
||||
// Find partition with space, or create new one
|
||||
for (const partitionId of existingPartitions) {
|
||||
const metadata = this.partitionMetadata.get(partitionId);
|
||||
if (metadata && metadata.nodeCount < this.config.maxNodesPerPartition) {
|
||||
return partitionId;
|
||||
}
|
||||
}
|
||||
// Create new partition
|
||||
return `partition_${this.nextPartitionId++}`;
|
||||
}
|
||||
/**
|
||||
* Semantic clustering partitioning
|
||||
*/
|
||||
async semanticPartition(vector) {
|
||||
// Find closest partition centroid
|
||||
let closestPartition = '';
|
||||
let minDistance = Infinity;
|
||||
for (const [partitionId, metadata] of this.partitionMetadata.entries()) {
|
||||
if (metadata.bounds?.centroid) {
|
||||
const distance = this.distanceFunction(vector, metadata.bounds.centroid);
|
||||
if (distance < minDistance) {
|
||||
minDistance = distance;
|
||||
closestPartition = partitionId;
|
||||
}
|
||||
}
|
||||
}
|
||||
// If no suitable partition found or it's full, create new one
|
||||
if (!closestPartition ||
|
||||
this.partitionMetadata.get(closestPartition).nodeCount >= this.config.maxNodesPerPartition) {
|
||||
closestPartition = `semantic_${this.nextPartitionId++}`;
|
||||
}
|
||||
return closestPartition;
|
||||
}
|
||||
/**
|
||||
* Auto-tune semantic clusters based on dataset size and performance
|
||||
*/
|
||||
autoTuneSemanticClusters() {
|
||||
const totalNodes = this.size();
|
||||
const currentPartitions = this.partitions.size;
|
||||
// Optimal clusters based on dataset size
|
||||
let optimalClusters = Math.max(4, Math.min(32, Math.floor(totalNodes / 10000)));
|
||||
// Adjust based on current partition performance
|
||||
if (currentPartitions > 0) {
|
||||
const avgNodesPerPartition = totalNodes / currentPartitions;
|
||||
if (avgNodesPerPartition > this.config.maxNodesPerPartition * 0.8) {
|
||||
// Partitions are getting full, increase clusters
|
||||
optimalClusters = Math.min(32, this.config.semanticClusters + 2);
|
||||
}
|
||||
else if (avgNodesPerPartition < this.config.maxNodesPerPartition * 0.3 && currentPartitions > 4) {
|
||||
// Partitions are underutilized, decrease clusters
|
||||
optimalClusters = Math.max(4, this.config.semanticClusters - 1);
|
||||
}
|
||||
}
|
||||
if (optimalClusters !== this.config.semanticClusters) {
|
||||
console.log(`Auto-tuning semantic clusters: ${this.config.semanticClusters} → ${optimalClusters}`);
|
||||
this.config.semanticClusters = optimalClusters;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Select which partitions to search based on query
|
||||
*/
|
||||
async selectSearchPartitions(queryVector, searchScope) {
|
||||
if (searchScope?.partitionIds) {
|
||||
return searchScope.partitionIds.filter(id => this.partitions.has(id));
|
||||
}
|
||||
const maxPartitions = searchScope?.maxPartitions || Math.min(5, this.partitions.size);
|
||||
if (this.config.partitionStrategy === 'semantic') {
|
||||
// Search partitions with closest centroids
|
||||
const distances = [];
|
||||
for (const [partitionId, metadata] of this.partitionMetadata.entries()) {
|
||||
if (metadata.bounds?.centroid) {
|
||||
const distance = this.distanceFunction(queryVector, metadata.bounds.centroid);
|
||||
distances.push([partitionId, distance]);
|
||||
}
|
||||
}
|
||||
distances.sort((a, b) => a[1] - b[1]);
|
||||
return distances.slice(0, maxPartitions).map(([id]) => id);
|
||||
}
|
||||
// For other strategies, search all partitions or random subset
|
||||
const allPartitionIds = Array.from(this.partitions.keys());
|
||||
if (allPartitionIds.length <= maxPartitions) {
|
||||
return allPartitionIds;
|
||||
}
|
||||
// Return random subset
|
||||
const shuffled = [...allPartitionIds].sort(() => Math.random() - 0.5);
|
||||
return shuffled.slice(0, maxPartitions);
|
||||
}
|
||||
/**
|
||||
* Update partition bounds for semantic clustering
|
||||
*/
|
||||
updatePartitionBounds(partitionId, vector) {
|
||||
const metadata = this.partitionMetadata.get(partitionId);
|
||||
if (!metadata.bounds) {
|
||||
metadata.bounds = {
|
||||
centroid: [...vector],
|
||||
radius: 0
|
||||
};
|
||||
return;
|
||||
}
|
||||
// Update centroid using incremental mean
|
||||
const { centroid } = metadata.bounds;
|
||||
const nodeCount = metadata.nodeCount;
|
||||
for (let i = 0; i < centroid.length; i++) {
|
||||
centroid[i] = (centroid[i] * (nodeCount - 1) + vector[i]) / nodeCount;
|
||||
}
|
||||
// Update radius
|
||||
const distance = this.distanceFunction(vector, centroid);
|
||||
metadata.bounds.radius = Math.max(metadata.bounds.radius, distance);
|
||||
}
|
||||
/**
|
||||
* Split an overgrown partition into smaller partitions
|
||||
*/
|
||||
async splitPartition(partitionId) {
|
||||
const partition = this.partitions.get(partitionId);
|
||||
if (!partition)
|
||||
return;
|
||||
console.log(`Splitting partition ${partitionId} with ${partition.size()} nodes`);
|
||||
// For now, we'll implement a simple strategy
|
||||
// In a full implementation, you'd want to analyze the data distribution
|
||||
// and create more intelligent splits
|
||||
// This is a placeholder - actual implementation would require
|
||||
// accessing the internal nodes of the HNSW index
|
||||
}
|
||||
/**
|
||||
* Simple hash function for consistent partitioning
|
||||
*/
|
||||
simpleHash(str) {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
const char = str.charCodeAt(i);
|
||||
hash = ((hash << 5) - hash) + char;
|
||||
hash = hash & hash; // Convert to 32-bit integer
|
||||
}
|
||||
return Math.abs(hash);
|
||||
}
|
||||
/**
|
||||
* Get partition statistics
|
||||
*/
|
||||
getPartitionStats() {
|
||||
const partitionDetails = Array.from(this.partitionMetadata.values());
|
||||
const totalNodes = partitionDetails.reduce((sum, p) => sum + p.nodeCount, 0);
|
||||
return {
|
||||
totalPartitions: partitionDetails.length,
|
||||
totalNodes,
|
||||
averageNodesPerPartition: totalNodes / partitionDetails.length || 0,
|
||||
partitionDetails
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Remove an item from the index
|
||||
*/
|
||||
async removeItem(id) {
|
||||
// Find which partition contains this item
|
||||
for (const [partitionId, partition] of this.partitions.entries()) {
|
||||
if (partition.removeItem(id)) {
|
||||
// Update metadata
|
||||
const metadata = this.partitionMetadata.get(partitionId);
|
||||
metadata.nodeCount = partition.size();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Clear all partitions
|
||||
*/
|
||||
clear() {
|
||||
for (const partition of this.partitions.values()) {
|
||||
partition.clear();
|
||||
}
|
||||
this.partitions.clear();
|
||||
this.partitionMetadata.clear();
|
||||
this.nextPartitionId = 0;
|
||||
}
|
||||
/**
|
||||
* Get total size across all partitions
|
||||
*/
|
||||
size() {
|
||||
return Array.from(this.partitions.values()).reduce((sum, partition) => sum + partition.size(), 0);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=partitionedHNSWIndex.js.map
|
||||
File diff suppressed because one or more lines are too long
142
.recovery-workspace/dist-backup-20250910-141917/hnsw/scaledHNSWSystem.d.ts
vendored
Normal file
142
.recovery-workspace/dist-backup-20250910-141917/hnsw/scaledHNSWSystem.d.ts
vendored
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
/**
|
||||
* Scaled HNSW System - Integration of All Optimization Strategies
|
||||
* Production-ready system for handling millions of vectors with sub-second search
|
||||
*/
|
||||
import { Vector, VectorDocument } from '../coreTypes.js';
|
||||
import { PartitionConfig } from './partitionedHNSWIndex.js';
|
||||
import { OptimizedHNSWConfig } from './optimizedHNSWIndex.js';
|
||||
import { SearchStrategy } from './distributedSearch.js';
|
||||
export interface ScaledHNSWConfig {
|
||||
expectedDatasetSize?: number;
|
||||
maxMemoryUsage?: number;
|
||||
targetSearchLatency?: number;
|
||||
s3Config?: {
|
||||
bucketName: string;
|
||||
region: string;
|
||||
endpoint?: string;
|
||||
accessKeyId?: string;
|
||||
secretAccessKey?: string;
|
||||
};
|
||||
autoConfigureEnvironment?: boolean;
|
||||
learningEnabled?: boolean;
|
||||
enablePartitioning?: boolean;
|
||||
enableCompression?: boolean;
|
||||
enableDistributedSearch?: boolean;
|
||||
enablePredictiveCaching?: boolean;
|
||||
partitionConfig?: Partial<PartitionConfig>;
|
||||
hnswConfig?: Partial<OptimizedHNSWConfig>;
|
||||
readOnlyMode?: boolean;
|
||||
}
|
||||
/**
|
||||
* High-performance HNSW system with all optimizations integrated
|
||||
* Handles datasets from thousands to millions of vectors
|
||||
*/
|
||||
export declare class ScaledHNSWSystem {
|
||||
private config;
|
||||
private autoConfig;
|
||||
private partitionedIndex?;
|
||||
private distributedSearch?;
|
||||
private cacheManager?;
|
||||
private batchOperations?;
|
||||
private readOnlyOptimizations?;
|
||||
private performanceMetrics;
|
||||
constructor(config?: ScaledHNSWConfig);
|
||||
/**
|
||||
* Initialize the optimized system based on configuration
|
||||
*/
|
||||
private initializeOptimizedSystem;
|
||||
/**
|
||||
* Calculate optimal configuration based on dataset size and constraints
|
||||
*/
|
||||
private calculateOptimalConfiguration;
|
||||
/**
|
||||
* Add vector to the scaled system
|
||||
*/
|
||||
addVector(item: VectorDocument): Promise<string>;
|
||||
/**
|
||||
* Bulk insert vectors with optimizations
|
||||
*/
|
||||
bulkInsert(items: VectorDocument[]): Promise<string[]>;
|
||||
/**
|
||||
* High-performance vector search with all optimizations
|
||||
*/
|
||||
search(queryVector: Vector, k?: number, options?: {
|
||||
strategy?: SearchStrategy;
|
||||
useCache?: boolean;
|
||||
maxPartitions?: number;
|
||||
}): Promise<Array<[string, number]>>;
|
||||
/**
|
||||
* Get system performance metrics
|
||||
*/
|
||||
getPerformanceMetrics(): typeof this.performanceMetrics & {
|
||||
partitionStats?: any;
|
||||
cacheStats?: any;
|
||||
compressionStats?: any;
|
||||
distributedSearchStats?: any;
|
||||
};
|
||||
/**
|
||||
* Optimize insertion order for better index quality
|
||||
*/
|
||||
private optimizeInsertionOrder;
|
||||
/**
|
||||
* Calculate optimal batch size based on system resources
|
||||
*/
|
||||
private calculateOptimalBatchSize;
|
||||
/**
|
||||
* Update search performance metrics
|
||||
*/
|
||||
private updateSearchMetrics;
|
||||
/**
|
||||
* Estimate current memory usage
|
||||
*/
|
||||
private estimateMemoryUsage;
|
||||
/**
|
||||
* Generate performance report
|
||||
*/
|
||||
generatePerformanceReport(): string;
|
||||
/**
|
||||
* Get overall system status
|
||||
*/
|
||||
private getSystemStatus;
|
||||
/**
|
||||
* Check if adaptive learning should be triggered
|
||||
*/
|
||||
private shouldTriggerLearning;
|
||||
/**
|
||||
* Adaptively learn from performance and adjust configuration
|
||||
*/
|
||||
private adaptivelyLearnFromPerformance;
|
||||
/**
|
||||
* Update dataset analysis for better auto-configuration
|
||||
*/
|
||||
updateDatasetAnalysis(vectorCount: number, vectorDimension?: number): Promise<void>;
|
||||
/**
|
||||
* Infer access patterns from current metrics
|
||||
*/
|
||||
private inferAccessPatterns;
|
||||
/**
|
||||
* Cleanup system resources
|
||||
*/
|
||||
cleanup(): void;
|
||||
}
|
||||
/**
|
||||
* Create a fully auto-configured Brainy system - minimal setup required!
|
||||
* Just provide S3 config if you want persistence beyond the current session
|
||||
*/
|
||||
export declare function createAutoBrainy(s3Config?: {
|
||||
bucketName: string;
|
||||
region?: string;
|
||||
accessKeyId?: string;
|
||||
secretAccessKey?: string;
|
||||
}): ScaledHNSWSystem;
|
||||
/**
|
||||
* Create a Brainy system optimized for specific scenarios
|
||||
*/
|
||||
export declare function createQuickBrainy(scenario: 'small' | 'medium' | 'large' | 'enterprise', s3Config?: {
|
||||
bucketName: string;
|
||||
region?: string;
|
||||
}): Promise<ScaledHNSWSystem>;
|
||||
/**
|
||||
* Legacy factory function - still works but consider using createAutoBrainy() instead
|
||||
*/
|
||||
export declare function createScaledHNSWSystem(config?: ScaledHNSWConfig): ScaledHNSWSystem;
|
||||
|
|
@ -0,0 +1,559 @@
|
|||
/**
|
||||
* Scaled HNSW System - Integration of All Optimization Strategies
|
||||
* Production-ready system for handling millions of vectors with sub-second search
|
||||
*/
|
||||
import { PartitionedHNSWIndex } from './partitionedHNSWIndex.js';
|
||||
import { DistributedSearchSystem, SearchStrategy } from './distributedSearch.js';
|
||||
import { EnhancedCacheManager } from '../storage/enhancedCacheManager.js';
|
||||
import { BatchS3Operations } from '../storage/adapters/batchS3Operations.js';
|
||||
import { ReadOnlyOptimizations } from '../storage/readOnlyOptimizations.js';
|
||||
import { euclideanDistance } from '../utils/index.js';
|
||||
import { AutoConfiguration } from '../utils/autoConfiguration.js';
|
||||
/**
|
||||
* High-performance HNSW system with all optimizations integrated
|
||||
* Handles datasets from thousands to millions of vectors
|
||||
*/
|
||||
export class ScaledHNSWSystem {
|
||||
constructor(config = {}) {
|
||||
// Performance monitoring and learning
|
||||
this.performanceMetrics = {
|
||||
totalSearches: 0,
|
||||
averageSearchTime: 0,
|
||||
cacheHitRate: 0,
|
||||
compressionRatio: 0,
|
||||
memoryUsage: 0,
|
||||
indexSize: 0,
|
||||
lastLearningUpdate: Date.now()
|
||||
};
|
||||
this.autoConfig = AutoConfiguration.getInstance();
|
||||
// Set basic defaults - these will be overridden by auto-configuration
|
||||
this.config = {
|
||||
expectedDatasetSize: 100000,
|
||||
maxMemoryUsage: 4 * 1024 * 1024 * 1024,
|
||||
targetSearchLatency: 150,
|
||||
autoConfigureEnvironment: true,
|
||||
learningEnabled: true,
|
||||
enablePartitioning: true,
|
||||
enableCompression: true,
|
||||
enableDistributedSearch: true,
|
||||
enablePredictiveCaching: true,
|
||||
readOnlyMode: false,
|
||||
...config
|
||||
};
|
||||
this.initializeOptimizedSystem();
|
||||
}
|
||||
/**
|
||||
* Initialize the optimized system based on configuration
|
||||
*/
|
||||
async initializeOptimizedSystem() {
|
||||
console.log('Initializing Scaled HNSW System with auto-configuration...');
|
||||
// Auto-configure if enabled
|
||||
if (this.config.autoConfigureEnvironment) {
|
||||
const autoConfigResult = await this.autoConfig.detectAndConfigure({
|
||||
expectedDataSize: this.config.expectedDatasetSize,
|
||||
s3Available: !!this.config.s3Config,
|
||||
memoryBudget: this.config.maxMemoryUsage
|
||||
});
|
||||
console.log(`Detected environment: ${autoConfigResult.environment}`);
|
||||
console.log(`Available memory: ${(autoConfigResult.availableMemory / 1024 / 1024 / 1024).toFixed(1)}GB`);
|
||||
console.log(`CPU cores: ${autoConfigResult.cpuCores}`);
|
||||
// Override config with auto-detected values
|
||||
this.config = {
|
||||
...this.config,
|
||||
expectedDatasetSize: autoConfigResult.recommendedConfig.expectedDatasetSize,
|
||||
maxMemoryUsage: autoConfigResult.recommendedConfig.maxMemoryUsage,
|
||||
targetSearchLatency: autoConfigResult.recommendedConfig.targetSearchLatency,
|
||||
enablePartitioning: autoConfigResult.recommendedConfig.enablePartitioning,
|
||||
enableCompression: autoConfigResult.recommendedConfig.enableCompression,
|
||||
enableDistributedSearch: autoConfigResult.recommendedConfig.enableDistributedSearch,
|
||||
enablePredictiveCaching: autoConfigResult.recommendedConfig.enablePredictiveCaching
|
||||
};
|
||||
}
|
||||
// Determine optimal configuration
|
||||
const optimizedConfig = this.calculateOptimalConfiguration();
|
||||
// Initialize partitioned index with semantic partitioning as default
|
||||
if (this.config.enablePartitioning) {
|
||||
this.partitionedIndex = new PartitionedHNSWIndex({
|
||||
...optimizedConfig.partitionConfig,
|
||||
partitionStrategy: 'semantic', // Always use semantic for better performance
|
||||
autoTuneSemanticClusters: true // Enable auto-tuning
|
||||
}, optimizedConfig.hnswConfig, euclideanDistance);
|
||||
console.log('✓ Partitioned index initialized with semantic clustering');
|
||||
}
|
||||
// Initialize distributed search system
|
||||
if (this.config.enableDistributedSearch && this.partitionedIndex) {
|
||||
this.distributedSearch = new DistributedSearchSystem({
|
||||
maxConcurrentSearches: optimizedConfig.maxConcurrentSearches,
|
||||
searchTimeout: this.config.targetSearchLatency * 5,
|
||||
adaptivePartitionSelection: true,
|
||||
loadBalancing: true
|
||||
});
|
||||
console.log('✓ Distributed search system initialized');
|
||||
}
|
||||
// Initialize batch S3 operations
|
||||
if (this.config.s3Config) {
|
||||
this.batchOperations = new BatchS3Operations(null, // Would be initialized with actual S3 client
|
||||
this.config.s3Config.bucketName, {
|
||||
maxConcurrency: 50,
|
||||
useS3Select: this.config.expectedDatasetSize > 100000
|
||||
});
|
||||
console.log('✓ Batch S3 operations initialized');
|
||||
}
|
||||
// Initialize enhanced caching
|
||||
if (this.config.enablePredictiveCaching) {
|
||||
this.cacheManager = new EnhancedCacheManager({
|
||||
hotCacheMaxSize: optimizedConfig.hotCacheSize,
|
||||
warmCacheMaxSize: optimizedConfig.warmCacheSize,
|
||||
prefetchEnabled: true,
|
||||
prefetchStrategy: 'hybrid', // Type casting for enum compatibility
|
||||
prefetchBatchSize: 50
|
||||
});
|
||||
if (this.batchOperations) {
|
||||
this.cacheManager.setStorageAdapters(null, this.batchOperations);
|
||||
}
|
||||
console.log('✓ Enhanced cache manager initialized');
|
||||
}
|
||||
// Initialize read-only optimizations
|
||||
if (this.config.readOnlyMode && this.config.enableCompression) {
|
||||
this.readOnlyOptimizations = new ReadOnlyOptimizations({
|
||||
compression: {
|
||||
vectorCompression: 'quantization',
|
||||
metadataCompression: 'gzip',
|
||||
quantizationType: 'scalar',
|
||||
quantizationBits: 8
|
||||
},
|
||||
segmentSize: optimizedConfig.segmentSize,
|
||||
memoryMapped: true,
|
||||
cacheIndexInMemory: optimizedConfig.cacheIndexInMemory
|
||||
});
|
||||
console.log('✓ Read-only optimizations initialized');
|
||||
}
|
||||
console.log('Scaled HNSW System ready for', this.config.expectedDatasetSize, 'vectors');
|
||||
}
|
||||
/**
|
||||
* Calculate optimal configuration based on dataset size and constraints
|
||||
*/
|
||||
calculateOptimalConfiguration() {
|
||||
const size = this.config.expectedDatasetSize;
|
||||
const memoryBudget = this.config.maxMemoryUsage;
|
||||
let config = {};
|
||||
if (size <= 10000) {
|
||||
// Small dataset - optimize for speed
|
||||
config = {
|
||||
partitionConfig: {
|
||||
maxNodesPerPartition: 10000,
|
||||
partitionStrategy: 'hash'
|
||||
},
|
||||
hnswConfig: {
|
||||
M: 16,
|
||||
efConstruction: 200,
|
||||
efSearch: 50,
|
||||
targetSearchLatency: this.config.targetSearchLatency
|
||||
},
|
||||
hotCacheSize: 1000,
|
||||
warmCacheSize: 5000,
|
||||
maxConcurrentSearches: 4,
|
||||
segmentSize: 5000,
|
||||
cacheIndexInMemory: true
|
||||
};
|
||||
}
|
||||
else if (size <= 100000) {
|
||||
// Medium dataset - balance performance and memory
|
||||
config = {
|
||||
partitionConfig: {
|
||||
maxNodesPerPartition: 25000,
|
||||
partitionStrategy: 'semantic',
|
||||
semanticClusters: 8
|
||||
},
|
||||
hnswConfig: {
|
||||
M: 24,
|
||||
efConstruction: 300,
|
||||
efSearch: 75,
|
||||
targetSearchLatency: this.config.targetSearchLatency,
|
||||
dynamicParameterTuning: true
|
||||
},
|
||||
hotCacheSize: 2000,
|
||||
warmCacheSize: 15000,
|
||||
maxConcurrentSearches: 8,
|
||||
segmentSize: 10000,
|
||||
cacheIndexInMemory: memoryBudget > 2 * 1024 * 1024 * 1024 // 2GB
|
||||
};
|
||||
}
|
||||
else if (size <= 1000000) {
|
||||
// Large dataset - optimize for scale
|
||||
config = {
|
||||
partitionConfig: {
|
||||
maxNodesPerPartition: 50000,
|
||||
partitionStrategy: 'semantic',
|
||||
semanticClusters: 16
|
||||
},
|
||||
hnswConfig: {
|
||||
M: 32,
|
||||
efConstruction: 400,
|
||||
efSearch: 100,
|
||||
targetSearchLatency: this.config.targetSearchLatency,
|
||||
dynamicParameterTuning: true,
|
||||
memoryBudget: memoryBudget
|
||||
},
|
||||
hotCacheSize: 5000,
|
||||
warmCacheSize: 25000,
|
||||
maxConcurrentSearches: 12,
|
||||
segmentSize: 20000,
|
||||
cacheIndexInMemory: memoryBudget > 8 * 1024 * 1024 * 1024 // 8GB
|
||||
};
|
||||
}
|
||||
else {
|
||||
// Very large dataset - maximum optimization
|
||||
config = {
|
||||
partitionConfig: {
|
||||
maxNodesPerPartition: 100000,
|
||||
partitionStrategy: 'hybrid',
|
||||
semanticClusters: 32
|
||||
},
|
||||
hnswConfig: {
|
||||
M: 48,
|
||||
efConstruction: 500,
|
||||
efSearch: 150,
|
||||
targetSearchLatency: this.config.targetSearchLatency,
|
||||
dynamicParameterTuning: true,
|
||||
memoryBudget: memoryBudget,
|
||||
diskCacheEnabled: true
|
||||
},
|
||||
hotCacheSize: 10000,
|
||||
warmCacheSize: 50000,
|
||||
maxConcurrentSearches: 20,
|
||||
segmentSize: 50000,
|
||||
cacheIndexInMemory: false // Too large for memory
|
||||
};
|
||||
}
|
||||
return config;
|
||||
}
|
||||
/**
|
||||
* Add vector to the scaled system
|
||||
*/
|
||||
async addVector(item) {
|
||||
if (!this.partitionedIndex) {
|
||||
throw new Error('System not properly initialized');
|
||||
}
|
||||
const startTime = Date.now();
|
||||
const result = await this.partitionedIndex.addItem(item);
|
||||
// Update performance metrics
|
||||
this.performanceMetrics.indexSize = this.partitionedIndex.size();
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Bulk insert vectors with optimizations
|
||||
*/
|
||||
async bulkInsert(items) {
|
||||
if (!this.partitionedIndex) {
|
||||
throw new Error('System not properly initialized');
|
||||
}
|
||||
console.log(`Starting optimized bulk insert of ${items.length} vectors`);
|
||||
const startTime = Date.now();
|
||||
// Sort items for optimal insertion order
|
||||
const sortedItems = this.optimizeInsertionOrder(items);
|
||||
const results = [];
|
||||
const batchSize = this.calculateOptimalBatchSize(items.length);
|
||||
// Process in batches
|
||||
for (let i = 0; i < sortedItems.length; i += batchSize) {
|
||||
const batch = sortedItems.slice(i, i + batchSize);
|
||||
for (const item of batch) {
|
||||
const id = await this.partitionedIndex.addItem(item);
|
||||
results.push(id);
|
||||
}
|
||||
// Progress logging
|
||||
if (i % (batchSize * 10) === 0) {
|
||||
const progress = ((i / sortedItems.length) * 100).toFixed(1);
|
||||
console.log(`Bulk insert progress: ${progress}%`);
|
||||
}
|
||||
}
|
||||
const totalTime = Date.now() - startTime;
|
||||
console.log(`Bulk insert completed: ${results.length} vectors in ${totalTime}ms`);
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* High-performance vector search with all optimizations
|
||||
*/
|
||||
async search(queryVector, k = 10, options = {}) {
|
||||
const startTime = Date.now();
|
||||
try {
|
||||
let results;
|
||||
if (this.distributedSearch && this.partitionedIndex) {
|
||||
// Use distributed search for optimal performance
|
||||
results = await this.distributedSearch.distributedSearch(this.partitionedIndex, queryVector, k, options.strategy || SearchStrategy.ADAPTIVE);
|
||||
}
|
||||
else if (this.partitionedIndex) {
|
||||
// Fall back to partitioned search
|
||||
results = await this.partitionedIndex.search(queryVector, k, { maxPartitions: options.maxPartitions });
|
||||
}
|
||||
else {
|
||||
throw new Error('No search system available');
|
||||
}
|
||||
// Update performance metrics and learn from performance
|
||||
const searchTime = Date.now() - startTime;
|
||||
this.updateSearchMetrics(searchTime, results.length);
|
||||
// Adaptive learning - adjust configuration based on performance
|
||||
if (this.config.learningEnabled && this.shouldTriggerLearning()) {
|
||||
await this.adaptivelyLearnFromPerformance();
|
||||
}
|
||||
return results;
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Search failed:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get system performance metrics
|
||||
*/
|
||||
getPerformanceMetrics() {
|
||||
const metrics = { ...this.performanceMetrics };
|
||||
// Add subsystem metrics
|
||||
if (this.partitionedIndex) {
|
||||
metrics.partitionStats = this.partitionedIndex.getPartitionStats();
|
||||
}
|
||||
if (this.cacheManager) {
|
||||
metrics.cacheStats = this.cacheManager.getStats();
|
||||
}
|
||||
if (this.readOnlyOptimizations) {
|
||||
metrics.compressionStats = this.readOnlyOptimizations.getCompressionStats();
|
||||
}
|
||||
if (this.distributedSearch) {
|
||||
metrics.distributedSearchStats = this.distributedSearch.getSearchStats();
|
||||
}
|
||||
return metrics;
|
||||
}
|
||||
/**
|
||||
* Optimize insertion order for better index quality
|
||||
*/
|
||||
optimizeInsertionOrder(items) {
|
||||
if (items.length < 1000) {
|
||||
return items; // Not worth optimizing small batches
|
||||
}
|
||||
// Simple clustering-based approach for better HNSW construction
|
||||
// In production, you might use more sophisticated clustering
|
||||
return items.sort(() => Math.random() - 0.5);
|
||||
}
|
||||
/**
|
||||
* Calculate optimal batch size based on system resources
|
||||
*/
|
||||
calculateOptimalBatchSize(totalItems) {
|
||||
const memoryBudget = this.config.maxMemoryUsage;
|
||||
const estimatedItemSize = 1000; // Rough estimate per item in bytes
|
||||
const maxBatch = Math.floor(memoryBudget * 0.1 / estimatedItemSize);
|
||||
const targetBatch = Math.min(1000, Math.max(100, maxBatch));
|
||||
return Math.min(targetBatch, totalItems);
|
||||
}
|
||||
/**
|
||||
* Update search performance metrics
|
||||
*/
|
||||
updateSearchMetrics(searchTime, resultCount) {
|
||||
this.performanceMetrics.totalSearches++;
|
||||
this.performanceMetrics.averageSearchTime =
|
||||
(this.performanceMetrics.averageSearchTime + searchTime) / 2;
|
||||
// Update other metrics
|
||||
if (this.cacheManager) {
|
||||
const cacheStats = this.cacheManager.getStats();
|
||||
const totalOps = cacheStats.hotCacheHits + cacheStats.hotCacheMisses +
|
||||
cacheStats.warmCacheHits + cacheStats.warmCacheMisses;
|
||||
this.performanceMetrics.cacheHitRate = totalOps > 0 ?
|
||||
(cacheStats.hotCacheHits + cacheStats.warmCacheHits) / totalOps : 0;
|
||||
}
|
||||
if (this.readOnlyOptimizations) {
|
||||
const compressionStats = this.readOnlyOptimizations.getCompressionStats();
|
||||
this.performanceMetrics.compressionRatio = compressionStats.compressionRatio;
|
||||
}
|
||||
// Estimate memory usage
|
||||
this.performanceMetrics.memoryUsage = this.estimateMemoryUsage();
|
||||
}
|
||||
/**
|
||||
* Estimate current memory usage
|
||||
*/
|
||||
estimateMemoryUsage() {
|
||||
let totalMemory = 0;
|
||||
if (this.partitionedIndex) {
|
||||
// Rough estimate: 1KB per vector
|
||||
totalMemory += this.partitionedIndex.size() * 1024;
|
||||
}
|
||||
if (this.cacheManager) {
|
||||
const cacheStats = this.cacheManager.getStats();
|
||||
totalMemory += (cacheStats.hotCacheSize + cacheStats.warmCacheSize) * 1024;
|
||||
}
|
||||
return totalMemory;
|
||||
}
|
||||
/**
|
||||
* Generate performance report
|
||||
*/
|
||||
generatePerformanceReport() {
|
||||
const metrics = this.getPerformanceMetrics();
|
||||
return `
|
||||
=== Scaled HNSW System Performance Report ===
|
||||
|
||||
Dataset Configuration:
|
||||
- Expected Size: ${this.config.expectedDatasetSize.toLocaleString()} vectors
|
||||
- Current Size: ${metrics.indexSize.toLocaleString()} vectors
|
||||
- Memory Budget: ${(this.config.maxMemoryUsage / 1024 / 1024 / 1024).toFixed(1)}GB
|
||||
- Target Latency: ${this.config.targetSearchLatency}ms
|
||||
|
||||
Performance Metrics:
|
||||
- Total Searches: ${metrics.totalSearches.toLocaleString()}
|
||||
- Average Search Time: ${metrics.averageSearchTime.toFixed(1)}ms
|
||||
- Cache Hit Rate: ${(metrics.cacheHitRate * 100).toFixed(1)}%
|
||||
- Memory Usage: ${(metrics.memoryUsage / 1024 / 1024).toFixed(1)}MB
|
||||
- Compression Ratio: ${metrics.compressionRatio ? (metrics.compressionRatio * 100).toFixed(1) + '%' : 'N/A'}
|
||||
|
||||
System Status: ${this.getSystemStatus()}
|
||||
`.trim();
|
||||
}
|
||||
/**
|
||||
* Get overall system status
|
||||
*/
|
||||
getSystemStatus() {
|
||||
const metrics = this.getPerformanceMetrics();
|
||||
if (metrics.averageSearchTime <= this.config.targetSearchLatency) {
|
||||
return '✅ OPTIMAL';
|
||||
}
|
||||
else if (metrics.averageSearchTime <= this.config.targetSearchLatency * 2) {
|
||||
return '⚠️ ACCEPTABLE';
|
||||
}
|
||||
else {
|
||||
return '❌ NEEDS OPTIMIZATION';
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if adaptive learning should be triggered
|
||||
*/
|
||||
shouldTriggerLearning() {
|
||||
const timeSinceLastLearning = Date.now() - this.performanceMetrics.lastLearningUpdate;
|
||||
const minLearningInterval = 30000; // 30 seconds
|
||||
const minSearches = 20; // Minimum searches before learning
|
||||
return timeSinceLastLearning > minLearningInterval &&
|
||||
this.performanceMetrics.totalSearches > minSearches &&
|
||||
this.performanceMetrics.totalSearches % 50 === 0; // Learn every 50 searches
|
||||
}
|
||||
/**
|
||||
* Adaptively learn from performance and adjust configuration
|
||||
*/
|
||||
async adaptivelyLearnFromPerformance() {
|
||||
try {
|
||||
const currentMetrics = {
|
||||
averageSearchTime: this.performanceMetrics.averageSearchTime,
|
||||
memoryUsage: this.performanceMetrics.memoryUsage,
|
||||
cacheHitRate: this.performanceMetrics.cacheHitRate,
|
||||
errorRate: 0 // Could be tracked separately
|
||||
};
|
||||
const adjustments = await this.autoConfig.learnFromPerformance(currentMetrics);
|
||||
if (Object.keys(adjustments).length > 0) {
|
||||
console.log('🧠 Adaptive learning: Adjusting configuration based on performance');
|
||||
// Apply learned adjustments
|
||||
let configChanged = false;
|
||||
if (adjustments.enableDistributedSearch !== undefined &&
|
||||
adjustments.enableDistributedSearch !== this.config.enableDistributedSearch) {
|
||||
this.config.enableDistributedSearch = adjustments.enableDistributedSearch;
|
||||
configChanged = true;
|
||||
}
|
||||
if (adjustments.enableCompression !== undefined &&
|
||||
adjustments.enableCompression !== this.config.enableCompression) {
|
||||
this.config.enableCompression = adjustments.enableCompression;
|
||||
configChanged = true;
|
||||
}
|
||||
if (adjustments.enablePredictiveCaching !== undefined &&
|
||||
adjustments.enablePredictiveCaching !== this.config.enablePredictiveCaching) {
|
||||
this.config.enablePredictiveCaching = adjustments.enablePredictiveCaching;
|
||||
configChanged = true;
|
||||
}
|
||||
// Apply partition adjustments
|
||||
if (adjustments.maxNodesPerPartition &&
|
||||
this.partitionedIndex &&
|
||||
adjustments.maxNodesPerPartition !== this.partitionedIndex.getPartitionStats().averageNodesPerPartition) {
|
||||
// This would require rebuilding the index in a real implementation
|
||||
console.log(`Learning suggests partition size: ${adjustments.maxNodesPerPartition}`);
|
||||
}
|
||||
if (configChanged) {
|
||||
console.log('✅ Configuration updated based on performance learning');
|
||||
}
|
||||
}
|
||||
this.performanceMetrics.lastLearningUpdate = Date.now();
|
||||
}
|
||||
catch (error) {
|
||||
console.warn('Adaptive learning failed:', error);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Update dataset analysis for better auto-configuration
|
||||
*/
|
||||
async updateDatasetAnalysis(vectorCount, vectorDimension) {
|
||||
if (this.config.autoConfigureEnvironment) {
|
||||
const analysis = {
|
||||
estimatedSize: vectorCount,
|
||||
vectorDimension,
|
||||
accessPatterns: this.inferAccessPatterns()
|
||||
};
|
||||
await this.autoConfig.adaptToDataset(analysis);
|
||||
console.log(`📊 Dataset analysis updated: ${vectorCount} vectors${vectorDimension ? `, ${vectorDimension}D` : ''}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Infer access patterns from current metrics
|
||||
*/
|
||||
inferAccessPatterns() {
|
||||
// Simple heuristic - in practice, this would track read/write ratios
|
||||
if (this.performanceMetrics.totalSearches > 100) {
|
||||
return 'read-heavy';
|
||||
}
|
||||
return 'balanced';
|
||||
}
|
||||
/**
|
||||
* Cleanup system resources
|
||||
*/
|
||||
cleanup() {
|
||||
this.distributedSearch?.cleanup();
|
||||
this.cacheManager?.clear();
|
||||
this.readOnlyOptimizations?.cleanup();
|
||||
this.partitionedIndex?.clear();
|
||||
this.autoConfig.resetCache();
|
||||
console.log('Scaled HNSW System cleaned up');
|
||||
}
|
||||
}
|
||||
// Export convenience factory functions
|
||||
/**
|
||||
* Create a fully auto-configured Brainy system - minimal setup required!
|
||||
* Just provide S3 config if you want persistence beyond the current session
|
||||
*/
|
||||
export function createAutoBrainy(s3Config) {
|
||||
return new ScaledHNSWSystem({
|
||||
s3Config: s3Config ? {
|
||||
bucketName: s3Config.bucketName,
|
||||
region: s3Config.region || 'us-east-1',
|
||||
accessKeyId: s3Config.accessKeyId,
|
||||
secretAccessKey: s3Config.secretAccessKey
|
||||
} : undefined,
|
||||
autoConfigureEnvironment: true,
|
||||
learningEnabled: true
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Create a Brainy system optimized for specific scenarios
|
||||
*/
|
||||
export async function createQuickBrainy(scenario, s3Config) {
|
||||
const { getQuickSetup } = await import('../utils/autoConfiguration.js');
|
||||
const quickConfig = await getQuickSetup(scenario);
|
||||
return new ScaledHNSWSystem({
|
||||
...quickConfig,
|
||||
s3Config: s3Config && quickConfig.s3Required ? {
|
||||
bucketName: s3Config.bucketName,
|
||||
region: s3Config.region || 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
} : undefined,
|
||||
autoConfigureEnvironment: true,
|
||||
learningEnabled: true
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Legacy factory function - still works but consider using createAutoBrainy() instead
|
||||
*/
|
||||
export function createScaledHNSWSystem(config = {}) {
|
||||
return new ScaledHNSWSystem(config);
|
||||
}
|
||||
//# sourceMappingURL=scaledHNSWSystem.js.map
|
||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue