From fd3cfdda6b8f9fe16f6b7ca6496dd9ce7d4c30fe Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 3 Aug 2025 17:26:41 -0700 Subject: [PATCH] feat(partitioning): simplify partition strategies and enable auto-tuning of semantic clusters - Removed unused partition strategies: 'random' and 'geographic' - Defaulted to 'semantic' partitioning for improved performance - Introduced auto-tuning for semantic clusters based on dataset size - Enhanced configuration options for better adaptability --- src/hnsw/partitionedHNSWIndex.ts | 76 +++++---- src/hnsw/scaledHNSWSystem.ts | 256 +++++++++++++++++++++++++++---- 2 files changed, 266 insertions(+), 66 deletions(-) diff --git a/src/hnsw/partitionedHNSWIndex.ts b/src/hnsw/partitionedHNSWIndex.ts index e4e38270..2b9faad6 100644 --- a/src/hnsw/partitionedHNSWIndex.ts +++ b/src/hnsw/partitionedHNSWIndex.ts @@ -15,14 +15,9 @@ import { euclideanDistance } from '../utils/index.js' export interface PartitionConfig { maxNodesPerPartition: number - partitionStrategy: 'semantic' | 'random' | 'geographic' | 'hash' - semanticClusters?: number - geographicBounds?: { - minLat: number - maxLat: number - minLng: number - maxLng: number - } + partitionStrategy: 'semantic' | 'hash' // Simplified to focus on useful strategies + semanticClusters?: number // Auto-configured based on dataset size + autoTuneSemanticClusters?: boolean // Automatically adjust cluster count } export interface PartitionMetadata { @@ -57,8 +52,9 @@ export class PartitionedHNSWIndex { ) { this.config = { maxNodesPerPartition: 50000, // Optimal size for memory efficiency - partitionStrategy: 'hash', - semanticClusters: 10, + partitionStrategy: 'semantic', // Default to semantic for better performance + semanticClusters: 8, // Auto-tuned based on dataset + autoTuneSemanticClusters: true, ...partitionConfig } @@ -111,8 +107,8 @@ export class PartitionedHNSWIndex { const metadata = this.partitionMetadata.get(partitionId)! metadata.nodeCount = partition.size() - // Update bounds for semantic/geographic strategies - if (this.config.partitionStrategy === 'semantic' || this.config.partitionStrategy === 'geographic') { + // Update bounds for semantic strategy + if (this.config.partitionStrategy === 'semantic') { this.updatePartitionBounds(partitionId, item.vector) } @@ -167,21 +163,19 @@ export class PartitionedHNSWIndex { /** * Select the appropriate partition for a new item + * Automatically chooses semantic partitioning when beneficial, falls back to hash */ private async selectPartition(item: VectorDocument): Promise { + // Auto-tune semantic clusters based on current dataset size + if (this.config.autoTuneSemanticClusters && this.config.partitionStrategy === 'semantic') { + this.autoTuneSemanticClusters() + } + switch (this.config.partitionStrategy) { - case 'hash': - return this.hashPartition(item.id) - case 'semantic': return await this.semanticPartition(item.vector) - case 'geographic': - return this.geographicPartition(item) - - case 'random': - return this.randomPartition() - + case 'hash': default: return this.hashPartition(item.id) } @@ -234,30 +228,32 @@ export class PartitionedHNSWIndex { } /** - * Geographic partitioning (requires lat/lng in metadata) + * Auto-tune semantic clusters based on dataset size and performance */ - private geographicPartition(item: VectorDocument): string { - // This would require geographic metadata in the item - // For now, fall back to hash partitioning - return this.hashPartition(item.id) - } - - /** - * Random partitioning - */ - private randomPartition(): string { - const existingPartitions = Array.from(this.partitions.keys()) + private autoTuneSemanticClusters(): void { + const totalNodes = this.size() + const currentPartitions = this.partitions.size - // Find partition with space - for (const partitionId of existingPartitions) { - const metadata = this.partitionMetadata.get(partitionId) - if (metadata && metadata.nodeCount < this.config.maxNodesPerPartition) { - return partitionId + // Optimal clusters based on dataset size + let optimalClusters = Math.max(4, Math.min(32, Math.floor(totalNodes / 10000))) + + // Adjust based on current partition performance + if (currentPartitions > 0) { + const avgNodesPerPartition = totalNodes / currentPartitions + + if (avgNodesPerPartition > this.config.maxNodesPerPartition * 0.8) { + // Partitions are getting full, increase clusters + optimalClusters = Math.min(32, this.config.semanticClusters! + 2) + } else if (avgNodesPerPartition < this.config.maxNodesPerPartition * 0.3 && currentPartitions > 4) { + // Partitions are underutilized, decrease clusters + optimalClusters = Math.max(4, this.config.semanticClusters! - 1) } } - // Create new partition - return `random_${this.nextPartitionId++}` + if (optimalClusters !== this.config.semanticClusters) { + console.log(`Auto-tuning semantic clusters: ${this.config.semanticClusters} → ${optimalClusters}`) + this.config.semanticClusters = optimalClusters + } } /** diff --git a/src/hnsw/scaledHNSWSystem.ts b/src/hnsw/scaledHNSWSystem.ts index 4e69b7e5..5653a2eb 100644 --- a/src/hnsw/scaledHNSWSystem.ts +++ b/src/hnsw/scaledHNSWSystem.ts @@ -11,29 +11,34 @@ import { EnhancedCacheManager } from '../storage/enhancedCacheManager.js' import { BatchS3Operations } from '../storage/adapters/batchS3Operations.js' import { ReadOnlyOptimizations } from '../storage/readOnlyOptimizations.js' import { euclideanDistance } from '../utils/index.js' +import { autoConfigureBrainy, AutoConfiguration } from '../utils/autoConfiguration.js' export interface ScaledHNSWConfig { - // Dataset scale expectations - expectedDatasetSize: number - maxMemoryUsage: number // bytes - targetSearchLatency: number // ms + // Required: Basic dataset expectations (can be auto-detected if not provided) + expectedDatasetSize?: number // Auto-detected if not provided + maxMemoryUsage?: number // Auto-detected based on environment + targetSearchLatency?: number // Auto-configured based on environment - // Storage configuration + // Storage configuration (optional - auto-detects S3 availability) s3Config?: { bucketName: string region: string endpoint?: string - accessKeyId: string - secretAccessKey: string + accessKeyId?: string // Falls back to env vars + secretAccessKey?: string // Falls back to env vars } - // Optimization strategies + // Auto-configuration options + autoConfigureEnvironment?: boolean // Default: true + learningEnabled?: boolean // Default: true - adapts to performance + + // Manual overrides (optional - auto-configured if not provided) enablePartitioning?: boolean enableCompression?: boolean enableDistributedSearch?: boolean enablePredictiveCaching?: boolean - // Performance tuning + // Advanced manual tuning (optional) partitionConfig?: Partial hnswConfig?: Partial readOnlyMode?: boolean @@ -44,35 +49,52 @@ export interface ScaledHNSWConfig { * Handles datasets from thousands to millions of vectors */ export class ScaledHNSWSystem { - private config: ScaledHNSWConfig + private config: ScaledHNSWConfig & { + expectedDatasetSize: number + maxMemoryUsage: number + targetSearchLatency: number + autoConfigureEnvironment: boolean + learningEnabled: boolean + enablePartitioning: boolean + enableCompression: boolean + enableDistributedSearch: boolean + enablePredictiveCaching: boolean + readOnlyMode: boolean + } + private autoConfig: AutoConfiguration private partitionedIndex?: PartitionedHNSWIndex private distributedSearch?: DistributedSearchSystem private cacheManager?: EnhancedCacheManager private batchOperations?: BatchS3Operations private readOnlyOptimizations?: ReadOnlyOptimizations - // Performance monitoring + // Performance monitoring and learning private performanceMetrics = { totalSearches: 0, averageSearchTime: 0, cacheHitRate: 0, compressionRatio: 0, memoryUsage: 0, - indexSize: 0 + indexSize: 0, + lastLearningUpdate: Date.now() } - constructor(config: ScaledHNSWConfig) { + constructor(config: ScaledHNSWConfig = {}) { + this.autoConfig = AutoConfiguration.getInstance() + + // Set basic defaults - these will be overridden by auto-configuration this.config = { + expectedDatasetSize: 100000, + maxMemoryUsage: 4 * 1024 * 1024 * 1024, + targetSearchLatency: 150, + autoConfigureEnvironment: true, + learningEnabled: true, enablePartitioning: true, enableCompression: true, enableDistributedSearch: true, enablePredictiveCaching: true, readOnlyMode: false, - ...config, - // Set defaults only if not provided - expectedDatasetSize: config.expectedDatasetSize || 100000, - maxMemoryUsage: config.maxMemoryUsage || (4 * 1024 * 1024 * 1024), - targetSearchLatency: config.targetSearchLatency || 100 + ...config } this.initializeOptimizedSystem() @@ -82,19 +104,48 @@ export class ScaledHNSWSystem { * Initialize the optimized system based on configuration */ private async initializeOptimizedSystem(): Promise { - console.log('Initializing Scaled HNSW System...') + console.log('Initializing Scaled HNSW System with auto-configuration...') - // Determine optimal configuration based on dataset size + // Auto-configure if enabled + if (this.config.autoConfigureEnvironment) { + const autoConfigResult = await this.autoConfig.detectAndConfigure({ + expectedDataSize: this.config.expectedDatasetSize, + s3Available: !!this.config.s3Config, + memoryBudget: this.config.maxMemoryUsage + }) + + console.log(`Detected environment: ${autoConfigResult.environment}`) + console.log(`Available memory: ${(autoConfigResult.availableMemory / 1024 / 1024 / 1024).toFixed(1)}GB`) + console.log(`CPU cores: ${autoConfigResult.cpuCores}`) + + // Override config with auto-detected values + this.config = { + ...this.config, + expectedDatasetSize: autoConfigResult.recommendedConfig.expectedDatasetSize, + maxMemoryUsage: autoConfigResult.recommendedConfig.maxMemoryUsage, + targetSearchLatency: autoConfigResult.recommendedConfig.targetSearchLatency, + enablePartitioning: autoConfigResult.recommendedConfig.enablePartitioning, + enableCompression: autoConfigResult.recommendedConfig.enableCompression, + enableDistributedSearch: autoConfigResult.recommendedConfig.enableDistributedSearch, + enablePredictiveCaching: autoConfigResult.recommendedConfig.enablePredictiveCaching + } + } + + // Determine optimal configuration const optimizedConfig = this.calculateOptimalConfiguration() - // Initialize partitioned index + // Initialize partitioned index with semantic partitioning as default if (this.config.enablePartitioning) { this.partitionedIndex = new PartitionedHNSWIndex( - optimizedConfig.partitionConfig, + { + ...optimizedConfig.partitionConfig, + partitionStrategy: 'semantic', // Always use semantic for better performance + autoTuneSemanticClusters: true // Enable auto-tuning + }, optimizedConfig.hnswConfig, euclideanDistance ) - console.log('✓ Partitioned index initialized') + console.log('✓ Partitioned index initialized with semantic clustering') } // Initialize distributed search system @@ -355,9 +406,14 @@ export class ScaledHNSWSystem { throw new Error('No search system available') } - // Update performance metrics + // Update performance metrics and learn from performance const searchTime = Date.now() - startTime this.updateSearchMetrics(searchTime, results.length) + + // Adaptive learning - adjust configuration based on performance + if (this.config.learningEnabled && this.shouldTriggerLearning()) { + await this.adaptivelyLearnFromPerformance() + } return results @@ -511,6 +567,104 @@ System Status: ${this.getSystemStatus()} } } + /** + * Check if adaptive learning should be triggered + */ + private shouldTriggerLearning(): boolean { + const timeSinceLastLearning = Date.now() - this.performanceMetrics.lastLearningUpdate + const minLearningInterval = 30000 // 30 seconds + const minSearches = 20 // Minimum searches before learning + + return timeSinceLastLearning > minLearningInterval && + this.performanceMetrics.totalSearches > minSearches && + this.performanceMetrics.totalSearches % 50 === 0 // Learn every 50 searches + } + + /** + * Adaptively learn from performance and adjust configuration + */ + private async adaptivelyLearnFromPerformance(): Promise { + try { + const currentMetrics = { + averageSearchTime: this.performanceMetrics.averageSearchTime, + memoryUsage: this.performanceMetrics.memoryUsage, + cacheHitRate: this.performanceMetrics.cacheHitRate, + errorRate: 0 // Could be tracked separately + } + + const adjustments = await this.autoConfig.learnFromPerformance(currentMetrics) + + if (Object.keys(adjustments).length > 0) { + console.log('🧠 Adaptive learning: Adjusting configuration based on performance') + + // Apply learned adjustments + let configChanged = false + + if (adjustments.enableDistributedSearch !== undefined && + adjustments.enableDistributedSearch !== this.config.enableDistributedSearch) { + this.config.enableDistributedSearch = adjustments.enableDistributedSearch + configChanged = true + } + + if (adjustments.enableCompression !== undefined && + adjustments.enableCompression !== this.config.enableCompression) { + this.config.enableCompression = adjustments.enableCompression + configChanged = true + } + + if (adjustments.enablePredictiveCaching !== undefined && + adjustments.enablePredictiveCaching !== this.config.enablePredictiveCaching) { + this.config.enablePredictiveCaching = adjustments.enablePredictiveCaching + configChanged = true + } + + // Apply partition adjustments + if (adjustments.maxNodesPerPartition && + this.partitionedIndex && + adjustments.maxNodesPerPartition !== this.partitionedIndex.getPartitionStats().averageNodesPerPartition) { + // This would require rebuilding the index in a real implementation + console.log(`Learning suggests partition size: ${adjustments.maxNodesPerPartition}`) + } + + if (configChanged) { + console.log('✅ Configuration updated based on performance learning') + } + } + + this.performanceMetrics.lastLearningUpdate = Date.now() + + } catch (error) { + console.warn('Adaptive learning failed:', error) + } + } + + /** + * Update dataset analysis for better auto-configuration + */ + public async updateDatasetAnalysis(vectorCount: number, vectorDimension?: number): Promise { + if (this.config.autoConfigureEnvironment) { + const analysis = { + estimatedSize: vectorCount, + vectorDimension, + accessPatterns: this.inferAccessPatterns() + } + + await this.autoConfig.adaptToDataset(analysis) + console.log(`📊 Dataset analysis updated: ${vectorCount} vectors${vectorDimension ? `, ${vectorDimension}D` : ''}`) + } + } + + /** + * Infer access patterns from current metrics + */ + private inferAccessPatterns(): 'read-heavy' | 'write-heavy' | 'balanced' { + // Simple heuristic - in practice, this would track read/write ratios + if (this.performanceMetrics.totalSearches > 100) { + return 'read-heavy' + } + return 'balanced' + } + /** * Cleanup system resources */ @@ -519,12 +673,62 @@ System Status: ${this.getSystemStatus()} this.cacheManager?.clear() this.readOnlyOptimizations?.cleanup() this.partitionedIndex?.clear() + this.autoConfig.resetCache() console.log('Scaled HNSW System cleaned up') } } -// Export convenience factory function -export function createScaledHNSWSystem(config: ScaledHNSWConfig): ScaledHNSWSystem { +// Export convenience factory functions + +/** + * Create a fully auto-configured Brainy system - minimal setup required! + * Just provide S3 config if you want persistence beyond the current session + */ +export function createAutoBrainy(s3Config?: { + bucketName: string + region?: string + accessKeyId?: string + secretAccessKey?: string +}): ScaledHNSWSystem { + return new ScaledHNSWSystem({ + s3Config: s3Config ? { + bucketName: s3Config.bucketName, + region: s3Config.region || 'us-east-1', + accessKeyId: s3Config.accessKeyId, + secretAccessKey: s3Config.secretAccessKey + } : undefined, + autoConfigureEnvironment: true, + learningEnabled: true + }) +} + +/** + * Create a Brainy system optimized for specific scenarios + */ +export async function createQuickBrainy( + scenario: 'small' | 'medium' | 'large' | 'enterprise', + s3Config?: { bucketName: string; region?: string } +): Promise { + const { getQuickSetup } = await import('../utils/autoConfiguration.js') + const quickConfig = await getQuickSetup(scenario) + + return new ScaledHNSWSystem({ + ...quickConfig, + s3Config: s3Config && quickConfig.s3Required ? { + bucketName: s3Config.bucketName, + region: s3Config.region || 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY + } : undefined, + autoConfigureEnvironment: true, + learningEnabled: true + }) +} + +/** + * Legacy factory function - still works but consider using createAutoBrainy() instead + */ +export function createScaledHNSWSystem(config: ScaledHNSWConfig = {}): ScaledHNSWSystem { return new ScaledHNSWSystem(config) } \ No newline at end of file