chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,313 @@
|
|||
/**
|
||||
* Optimized HNSW Index for Large-Scale Vector Search
|
||||
* Implements dynamic parameter tuning and performance optimizations
|
||||
*/
|
||||
import { HNSWIndex } from './hnswIndex.js';
|
||||
import { euclideanDistance } from '../utils/index.js';
|
||||
/**
|
||||
* Optimized HNSW Index with dynamic parameter tuning for large datasets
|
||||
*/
|
||||
export class OptimizedHNSWIndex extends HNSWIndex {
|
||||
constructor(config = {}, distanceFunction = euclideanDistance) {
|
||||
// Set optimized defaults for large scale
|
||||
const defaultConfig = {
|
||||
M: 32, // Higher connectivity for better recall
|
||||
efConstruction: 400, // Better build quality
|
||||
efSearch: 100, // Dynamic - will be tuned
|
||||
ml: 24, // Deeper hierarchy
|
||||
useDiskBasedIndex: false, // Added missing property
|
||||
dynamicParameterTuning: true,
|
||||
targetSearchLatency: 100, // 100ms target
|
||||
targetRecall: 0.95, // 95% recall target
|
||||
maxNodes: 1000000, // 1M node limit
|
||||
memoryBudget: 8 * 1024 * 1024 * 1024, // 8GB
|
||||
diskCacheEnabled: true,
|
||||
compressionEnabled: false, // Disabled by default for compatibility
|
||||
performanceTracking: true,
|
||||
adaptiveEfSearch: true,
|
||||
levelMultiplier: 16,
|
||||
seedConnections: 8,
|
||||
pruningStrategy: 'hybrid'
|
||||
};
|
||||
const mergedConfig = { ...defaultConfig, ...config };
|
||||
// Initialize parent with base config
|
||||
super({
|
||||
M: mergedConfig.M,
|
||||
efConstruction: mergedConfig.efConstruction,
|
||||
efSearch: mergedConfig.efSearch,
|
||||
ml: mergedConfig.ml
|
||||
}, distanceFunction, { useParallelization: true });
|
||||
this.searchHistory = [];
|
||||
this.optimizedConfig = mergedConfig;
|
||||
// Initialize dynamic parameters
|
||||
this.dynamicParams = {
|
||||
efSearch: mergedConfig.efSearch,
|
||||
efConstruction: mergedConfig.efConstruction,
|
||||
M: mergedConfig.M,
|
||||
ml: mergedConfig.ml
|
||||
};
|
||||
// Initialize performance metrics
|
||||
this.performanceMetrics = {
|
||||
averageSearchTime: 0,
|
||||
averageRecall: 0,
|
||||
memoryUsage: 0,
|
||||
indexSize: 0,
|
||||
apiCalls: 0,
|
||||
cacheHitRate: 0
|
||||
};
|
||||
// Start parameter tuning if enabled
|
||||
if (this.optimizedConfig.dynamicParameterTuning) {
|
||||
this.startParameterTuning();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Optimized search with dynamic parameter adjustment
|
||||
*/
|
||||
async search(queryVector, k = 10, filter) {
|
||||
const startTime = Date.now();
|
||||
// Adjust efSearch dynamically based on k and performance history
|
||||
if (this.optimizedConfig.adaptiveEfSearch) {
|
||||
this.adjustEfSearch(k);
|
||||
}
|
||||
// Check memory usage and trigger optimizations if needed
|
||||
if (this.optimizedConfig.performanceTracking) {
|
||||
this.checkMemoryUsage();
|
||||
}
|
||||
// Perform the search with current parameters
|
||||
const originalConfig = this.getConfig();
|
||||
// Temporarily update search parameters
|
||||
const tempConfig = {
|
||||
...originalConfig,
|
||||
efSearch: this.dynamicParams.efSearch
|
||||
};
|
||||
// Use the parent's search method with optimized parameters
|
||||
let results;
|
||||
try {
|
||||
// This is a simplified approach - in practice, we'd need to modify
|
||||
// the parent class to accept runtime parameter changes
|
||||
results = await super.search(queryVector, k, filter);
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Optimized search failed, falling back to default:', error);
|
||||
results = await super.search(queryVector, k, filter);
|
||||
}
|
||||
// Record performance metrics
|
||||
const searchTime = Date.now() - startTime;
|
||||
this.recordSearchMetrics(searchTime, k, results.length);
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Dynamically adjust efSearch based on performance requirements
|
||||
*/
|
||||
adjustEfSearch(k) {
|
||||
const recentSearches = this.searchHistory.slice(-10);
|
||||
if (recentSearches.length < 3) {
|
||||
// Not enough data, use heuristic
|
||||
this.dynamicParams.efSearch = Math.max(k * 2, 50);
|
||||
return;
|
||||
}
|
||||
const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
|
||||
const targetLatency = this.optimizedConfig.targetSearchLatency;
|
||||
// Adjust efSearch based on latency performance
|
||||
if (averageLatency > targetLatency * 1.2) {
|
||||
// Too slow, reduce efSearch
|
||||
this.dynamicParams.efSearch = Math.max(Math.floor(this.dynamicParams.efSearch * 0.9), k);
|
||||
}
|
||||
else if (averageLatency < targetLatency * 0.8) {
|
||||
// Fast enough, can increase efSearch for better recall
|
||||
this.dynamicParams.efSearch = Math.min(Math.floor(this.dynamicParams.efSearch * 1.1), 500 // Maximum efSearch
|
||||
);
|
||||
}
|
||||
// Ensure efSearch is at least k
|
||||
this.dynamicParams.efSearch = Math.max(this.dynamicParams.efSearch, k);
|
||||
}
|
||||
/**
|
||||
* Record search performance metrics
|
||||
*/
|
||||
recordSearchMetrics(latency, k, resultCount) {
|
||||
if (!this.optimizedConfig.performanceTracking) {
|
||||
return;
|
||||
}
|
||||
// Add to search history
|
||||
this.searchHistory.push({
|
||||
latency,
|
||||
k,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
// Keep only recent history (last 100 searches)
|
||||
if (this.searchHistory.length > 100) {
|
||||
this.searchHistory.shift();
|
||||
}
|
||||
// Update performance metrics
|
||||
const recentSearches = this.searchHistory.slice(-20);
|
||||
this.performanceMetrics.averageSearchTime =
|
||||
recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
|
||||
// Estimate recall (simplified - would need ground truth for accurate measurement)
|
||||
this.performanceMetrics.averageRecall = Math.min(resultCount / k, 1.0);
|
||||
}
|
||||
/**
|
||||
* Check memory usage and trigger optimizations
|
||||
*/
|
||||
checkMemoryUsage() {
|
||||
// Estimate memory usage (simplified)
|
||||
const estimatedMemory = this.size() * 1000; // Rough estimate per node
|
||||
this.performanceMetrics.memoryUsage = estimatedMemory;
|
||||
if (estimatedMemory > this.optimizedConfig.memoryBudget * 0.9) {
|
||||
console.warn('Memory usage approaching limit, consider index partitioning');
|
||||
// Could trigger automatic partitioning or compression here
|
||||
if (this.optimizedConfig.compressionEnabled) {
|
||||
this.compressIndex();
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Compress index to reduce memory usage (placeholder)
|
||||
*/
|
||||
compressIndex() {
|
||||
console.log('Index compression not implemented yet');
|
||||
// This would implement vector quantization or other compression techniques
|
||||
}
|
||||
/**
|
||||
* Start automatic parameter tuning
|
||||
*/
|
||||
startParameterTuning() {
|
||||
this.parameterTuningInterval = setInterval(() => {
|
||||
this.tuneParameters();
|
||||
}, 30000); // Tune every 30 seconds
|
||||
}
|
||||
/**
|
||||
* Automatic parameter tuning based on performance metrics
|
||||
*/
|
||||
tuneParameters() {
|
||||
if (this.searchHistory.length < 10) {
|
||||
return; // Not enough data
|
||||
}
|
||||
const recentSearches = this.searchHistory.slice(-20);
|
||||
const averageLatency = recentSearches.reduce((sum, s) => sum + s.latency, 0) / recentSearches.length;
|
||||
// Tune based on performance vs targets
|
||||
const latencyRatio = averageLatency / this.optimizedConfig.targetSearchLatency;
|
||||
const recallRatio = this.performanceMetrics.averageRecall / this.optimizedConfig.targetRecall;
|
||||
// Adjust M (connectivity) for long-term performance
|
||||
if (this.size() > 10000) { // Only tune for larger indices
|
||||
if (recallRatio < 0.95 && latencyRatio < 1.5) {
|
||||
// Recall is low but we have latency budget, increase M
|
||||
this.dynamicParams.M = Math.min(this.dynamicParams.M + 2, 64);
|
||||
}
|
||||
else if (latencyRatio > 1.2 && recallRatio > 1.0) {
|
||||
// Latency is high but recall is good, can reduce M
|
||||
this.dynamicParams.M = Math.max(this.dynamicParams.M - 2, 16);
|
||||
}
|
||||
}
|
||||
console.log(`Parameter tuning: efSearch=${this.dynamicParams.efSearch}, M=${this.dynamicParams.M}, latency=${averageLatency.toFixed(1)}ms`);
|
||||
}
|
||||
/**
|
||||
* Get optimized configuration recommendations for current dataset size
|
||||
*/
|
||||
getOptimizedConfig() {
|
||||
const currentSize = this.size();
|
||||
let recommendedConfig = {};
|
||||
if (currentSize < 10000) {
|
||||
// Small dataset - optimize for speed
|
||||
recommendedConfig = {
|
||||
M: 16,
|
||||
efConstruction: 200,
|
||||
efSearch: 50,
|
||||
ml: 16
|
||||
};
|
||||
}
|
||||
else if (currentSize < 100000) {
|
||||
// Medium dataset - balance speed and recall
|
||||
recommendedConfig = {
|
||||
M: 24,
|
||||
efConstruction: 300,
|
||||
efSearch: 75,
|
||||
ml: 20
|
||||
};
|
||||
}
|
||||
else if (currentSize < 1000000) {
|
||||
// Large dataset - optimize for recall
|
||||
recommendedConfig = {
|
||||
M: 32,
|
||||
efConstruction: 400,
|
||||
efSearch: 100,
|
||||
ml: 24
|
||||
};
|
||||
}
|
||||
else {
|
||||
// Very large dataset - maximum quality
|
||||
recommendedConfig = {
|
||||
M: 48,
|
||||
efConstruction: 500,
|
||||
efSearch: 150,
|
||||
ml: 28
|
||||
};
|
||||
}
|
||||
return {
|
||||
...this.optimizedConfig,
|
||||
...recommendedConfig
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get current performance metrics
|
||||
*/
|
||||
getPerformanceMetrics() {
|
||||
return {
|
||||
...this.performanceMetrics,
|
||||
currentParams: { ...this.dynamicParams },
|
||||
searchHistorySize: this.searchHistory.length
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Apply optimized bulk insertion strategy
|
||||
*/
|
||||
async bulkInsert(items) {
|
||||
console.log(`Starting optimized bulk insert of ${items.length} items`);
|
||||
// Sort items to optimize insertion order (by vector similarity)
|
||||
const sortedItems = this.optimizeInsertionOrder(items);
|
||||
// Temporarily adjust construction parameters for bulk operations
|
||||
const originalEfConstruction = this.dynamicParams.efConstruction;
|
||||
this.dynamicParams.efConstruction = Math.min(this.dynamicParams.efConstruction * 1.5, 800);
|
||||
const results = [];
|
||||
const batchSize = 100;
|
||||
try {
|
||||
// Process in batches to manage memory
|
||||
for (let i = 0; i < sortedItems.length; i += batchSize) {
|
||||
const batch = sortedItems.slice(i, i + batchSize);
|
||||
for (const item of batch) {
|
||||
const id = await this.addItem(item);
|
||||
results.push(id);
|
||||
}
|
||||
// Periodic memory check
|
||||
if (i % (batchSize * 10) === 0) {
|
||||
this.checkMemoryUsage();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
// Restore original construction parameters
|
||||
this.dynamicParams.efConstruction = originalEfConstruction;
|
||||
}
|
||||
console.log(`Completed bulk insert of ${results.length} items`);
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Optimize insertion order to improve index quality
|
||||
*/
|
||||
optimizeInsertionOrder(items) {
|
||||
if (items.length < 100) {
|
||||
return items; // Not worth optimizing small batches
|
||||
}
|
||||
// Simple clustering-based ordering
|
||||
// In practice, you might use more sophisticated methods
|
||||
return items.sort(() => Math.random() - 0.5); // Shuffle for now
|
||||
}
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
destroy() {
|
||||
if (this.parameterTuningInterval) {
|
||||
clearInterval(this.parameterTuningInterval);
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=optimizedHNSWIndex.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue