diff --git a/src/utils/autoConfiguration.ts b/src/utils/autoConfiguration.ts new file mode 100644 index 00000000..a4367a58 --- /dev/null +++ b/src/utils/autoConfiguration.ts @@ -0,0 +1,474 @@ +/** + * Automatic Configuration System for Brainy Vector Database + * Detects environment, resources, and data patterns to provide optimal settings + */ + +import { isBrowser, isNode, isThreadingAvailable } from './environment.js' + +export interface AutoConfigResult { + // Environment details + environment: 'browser' | 'nodejs' | 'serverless' | 'unknown' + + // Resource detection + availableMemory: number // bytes + cpuCores: number + threadingAvailable: boolean + + // Storage capabilities + persistentStorageAvailable: boolean + s3StorageDetected: boolean + + // Recommended configuration + recommendedConfig: { + expectedDatasetSize: number + maxMemoryUsage: number + targetSearchLatency: number + enablePartitioning: boolean + enableCompression: boolean + enableDistributedSearch: boolean + enablePredictiveCaching: boolean + partitionStrategy: 'semantic' | 'hash' + maxNodesPerPartition: number + semanticClusters: number + } + + // Performance optimization flags + optimizationFlags: { + useMemoryMapping: boolean + aggressiveCaching: boolean + backgroundOptimization: boolean + compressionLevel: 'none' | 'light' | 'aggressive' + } +} + +export interface DatasetAnalysis { + estimatedSize: number + vectorDimension?: number + growthRate?: number // vectors per second + accessPatterns?: 'read-heavy' | 'write-heavy' | 'balanced' +} + +/** + * Automatic configuration system that detects environment and optimizes settings + */ +export class AutoConfiguration { + private static instance: AutoConfiguration + private cachedConfig: AutoConfigResult | null = null + private datasetStats: DatasetAnalysis = { estimatedSize: 0 } + + private constructor() {} + + public static getInstance(): AutoConfiguration { + if (!AutoConfiguration.instance) { + AutoConfiguration.instance = new AutoConfiguration() + } + return AutoConfiguration.instance + } + + /** + * Detect environment and generate optimal configuration + */ + public async detectAndConfigure(hints?: { + expectedDataSize?: number + s3Available?: boolean + memoryBudget?: number + }): Promise { + if (this.cachedConfig && !hints) { + return this.cachedConfig + } + + const environment = this.detectEnvironment() + const resources = await this.detectResources() + const storage = await this.detectStorageCapabilities(hints?.s3Available) + + const config: AutoConfigResult = { + environment, + ...resources, + ...storage, + recommendedConfig: this.generateRecommendedConfig(environment, resources, hints), + optimizationFlags: this.generateOptimizationFlags(environment, resources) + } + + this.cachedConfig = config + return config + } + + /** + * Update configuration based on runtime dataset analysis + */ + public async adaptToDataset(analysis: DatasetAnalysis): Promise { + this.datasetStats = analysis + + // Regenerate configuration with dataset insights + const currentConfig = await this.detectAndConfigure() + const adaptedConfig = this.adaptConfigurationToData(currentConfig, analysis) + + this.cachedConfig = adaptedConfig + return adaptedConfig + } + + /** + * Learn from performance metrics and adjust configuration + */ + public async learnFromPerformance(metrics: { + averageSearchTime: number + memoryUsage: number + cacheHitRate: number + errorRate: number + }): Promise> { + const adjustments: Partial = {} + + // Learn from search performance + if (metrics.averageSearchTime > 200) { + // Too slow - optimize for speed + adjustments.enableDistributedSearch = true + adjustments.maxNodesPerPartition = Math.max(10000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 0.8) + } else if (metrics.averageSearchTime < 50) { + // Very fast - can optimize for quality + adjustments.maxNodesPerPartition = Math.min(100000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 1.2) + } + + // Learn from memory usage + if (metrics.memoryUsage > (this.cachedConfig?.recommendedConfig.maxMemoryUsage || 0) * 0.9) { + // High memory usage - enable compression + adjustments.enableCompression = true + } + + // Learn from cache performance + if (metrics.cacheHitRate < 0.7) { + // Poor cache performance - enable predictive caching + adjustments.enablePredictiveCaching = true + } + + // Update cached config with learned adjustments + if (this.cachedConfig) { + this.cachedConfig.recommendedConfig = { + ...this.cachedConfig.recommendedConfig, + ...adjustments + } + } + + return adjustments + } + + /** + * Get minimal configuration for quick setup + */ + public async getQuickSetupConfig(scenario: 'small' | 'medium' | 'large' | 'enterprise'): Promise<{ + expectedDatasetSize: number + maxMemoryUsage: number + targetSearchLatency: number + s3Required: boolean + }> { + const environment = this.detectEnvironment() + const resources = await this.detectResources() + + switch (scenario) { + case 'small': + return { + expectedDatasetSize: 10000, + maxMemoryUsage: Math.min(resources.availableMemory * 0.3, 1024 * 1024 * 1024), // 1GB max + targetSearchLatency: 100, + s3Required: false + } + + case 'medium': + return { + expectedDatasetSize: 100000, + maxMemoryUsage: Math.min(resources.availableMemory * 0.5, 4 * 1024 * 1024 * 1024), // 4GB max + targetSearchLatency: 150, + s3Required: environment === 'serverless' + } + + case 'large': + return { + expectedDatasetSize: 1000000, + maxMemoryUsage: Math.min(resources.availableMemory * 0.7, 8 * 1024 * 1024 * 1024), // 8GB max + targetSearchLatency: 200, + s3Required: true + } + + case 'enterprise': + return { + expectedDatasetSize: 10000000, + maxMemoryUsage: Math.min(resources.availableMemory * 0.8, 32 * 1024 * 1024 * 1024), // 32GB max + targetSearchLatency: 300, + s3Required: true + } + } + } + + /** + * Detect the current runtime environment + */ + private detectEnvironment(): 'browser' | 'nodejs' | 'serverless' | 'unknown' { + if (isBrowser()) { + return 'browser' + } + + if (isNode()) { + // Check for serverless environment indicators + if (process.env.AWS_LAMBDA_FUNCTION_NAME || + process.env.VERCEL || + process.env.NETLIFY || + process.env.CLOUDFLARE_WORKERS) { + return 'serverless' + } + return 'nodejs' + } + + return 'unknown' + } + + /** + * Detect available system resources + */ + private async detectResources(): Promise<{ + availableMemory: number + cpuCores: number + threadingAvailable: boolean + }> { + let availableMemory = 2 * 1024 * 1024 * 1024 // Default 2GB + let cpuCores = 4 // Default 4 cores + + // Browser memory detection + if (isBrowser()) { + // @ts-ignore - navigator.deviceMemory is experimental + if (navigator.deviceMemory) { + // @ts-ignore + availableMemory = navigator.deviceMemory * 1024 * 1024 * 1024 * 0.3 // Use 30% of device memory + } else { + availableMemory = 512 * 1024 * 1024 // Conservative 512MB for browsers + } + + cpuCores = navigator.hardwareConcurrency || 4 + } + + // Node.js memory detection + if (isNode()) { + try { + const os = await import('os') + availableMemory = os.totalmem() * 0.7 // Use 70% of total memory + cpuCores = os.cpus().length + } catch (error) { + // Fallback to defaults + } + } + + return { + availableMemory, + cpuCores, + threadingAvailable: isThreadingAvailable() + } + } + + /** + * Detect available storage capabilities + */ + private async detectStorageCapabilities(s3Hint?: boolean): Promise<{ + persistentStorageAvailable: boolean + s3StorageDetected: boolean + }> { + let persistentStorageAvailable = false + let s3StorageDetected = s3Hint || false + + if (isBrowser()) { + // Check for OPFS support + persistentStorageAvailable = 'navigator' in globalThis && + 'storage' in navigator && + 'getDirectory' in navigator.storage + } + + if (isNode()) { + persistentStorageAvailable = true // Always available in Node.js + + // Check for AWS SDK or S3 environment variables + s3StorageDetected = s3Hint || + !!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) || + !!(process.env.S3_BUCKET_NAME) + } + + return { + persistentStorageAvailable, + s3StorageDetected + } + } + + /** + * Generate recommended configuration based on detected environment and resources + */ + private generateRecommendedConfig( + environment: string, + resources: { availableMemory: number; cpuCores: number }, + hints?: { expectedDataSize?: number; memoryBudget?: number } + ): AutoConfigResult['recommendedConfig'] { + const datasetSize = hints?.expectedDataSize || this.estimateDatasetSize() + const memoryBudget = hints?.memoryBudget || Math.floor(resources.availableMemory * 0.6) + + // Base configuration + let config = { + expectedDatasetSize: datasetSize, + maxMemoryUsage: memoryBudget, + targetSearchLatency: 150, + enablePartitioning: datasetSize > 25000, + enableCompression: environment === 'browser' || memoryBudget < 2 * 1024 * 1024 * 1024, + enableDistributedSearch: resources.cpuCores > 2 && datasetSize > 50000, + enablePredictiveCaching: true, + partitionStrategy: 'semantic' as const, + maxNodesPerPartition: 50000, + semanticClusters: 8 + } + + // Environment-specific adjustments + switch (environment) { + case 'browser': + config = { + ...config, + maxMemoryUsage: Math.min(memoryBudget, 1024 * 1024 * 1024), // Cap at 1GB + targetSearchLatency: 200, // More lenient for browsers + enableCompression: true, // Always enable for browsers + maxNodesPerPartition: 25000, // Smaller partitions + semanticClusters: 4 // Fewer clusters to save memory + } + break + + case 'serverless': + config = { + ...config, + targetSearchLatency: 500, // Account for cold starts + enablePredictiveCaching: false, // Avoid background processes + maxNodesPerPartition: 30000 // Moderate partition size + } + break + + case 'nodejs': + config = { + ...config, + targetSearchLatency: 100, // Aggressive for Node.js + maxNodesPerPartition: Math.min(100000, Math.floor(datasetSize / 10)), // Larger partitions + semanticClusters: Math.min(16, Math.max(4, Math.floor(datasetSize / 50000))) // Scale clusters with data + } + break + } + + // Dataset size adjustments + if (datasetSize > 1000000) { + config.semanticClusters = Math.min(32, Math.floor(datasetSize / 100000)) + config.maxNodesPerPartition = 100000 + } else if (datasetSize < 10000) { + config.enablePartitioning = false + config.enableDistributedSearch = false + config.partitionStrategy = 'semantic' // Keep semantic but disable partitioning + } + + return config + } + + /** + * Generate optimization flags based on environment and resources + */ + private generateOptimizationFlags( + environment: string, + resources: { availableMemory: number; cpuCores: number } + ): AutoConfigResult['optimizationFlags'] { + return { + useMemoryMapping: environment === 'nodejs' && resources.availableMemory > 4 * 1024 * 1024 * 1024, + aggressiveCaching: resources.availableMemory > 2 * 1024 * 1024 * 1024, + backgroundOptimization: environment !== 'serverless' && resources.cpuCores > 2, + compressionLevel: resources.availableMemory < 1024 * 1024 * 1024 ? 'aggressive' : + resources.availableMemory < 4 * 1024 * 1024 * 1024 ? 'light' : 'none' + } + } + + /** + * Adapt configuration based on actual dataset analysis + */ + private adaptConfigurationToData( + baseConfig: AutoConfigResult, + analysis: DatasetAnalysis + ): AutoConfigResult { + const updatedConfig = { ...baseConfig } + + // Adjust based on actual dataset size + if (analysis.estimatedSize !== baseConfig.recommendedConfig.expectedDatasetSize) { + const sizeRatio = analysis.estimatedSize / baseConfig.recommendedConfig.expectedDatasetSize + + updatedConfig.recommendedConfig.expectedDatasetSize = analysis.estimatedSize + + // Scale partition size with dataset + if (sizeRatio > 2) { + updatedConfig.recommendedConfig.maxNodesPerPartition = Math.min( + 100000, + Math.floor(updatedConfig.recommendedConfig.maxNodesPerPartition * 1.5) + ) + updatedConfig.recommendedConfig.semanticClusters = Math.min( + 32, + Math.floor(updatedConfig.recommendedConfig.semanticClusters * 1.5) + ) + } + } + + // Adjust based on vector dimension + if (analysis.vectorDimension) { + if (analysis.vectorDimension > 1024) { + // High-dimensional vectors - optimize for compression + updatedConfig.recommendedConfig.enableCompression = true + updatedConfig.optimizationFlags.compressionLevel = 'aggressive' + } + } + + // Adjust based on access patterns + if (analysis.accessPatterns === 'read-heavy') { + updatedConfig.recommendedConfig.enablePredictiveCaching = true + updatedConfig.optimizationFlags.aggressiveCaching = true + } else if (analysis.accessPatterns === 'write-heavy') { + updatedConfig.recommendedConfig.enablePredictiveCaching = false + updatedConfig.optimizationFlags.backgroundOptimization = false + } + + return updatedConfig + } + + /** + * Estimate dataset size if not provided + */ + private estimateDatasetSize(): number { + // Start with conservative estimate + const environment = this.detectEnvironment() + + switch (environment) { + case 'browser': return 10000 + case 'serverless': return 50000 + case 'nodejs': return 100000 + default: return 25000 + } + } + + /** + * Reset cached configuration (for testing or manual refresh) + */ + public resetCache(): void { + this.cachedConfig = null + this.datasetStats = { estimatedSize: 0 } + } +} + +/** + * Convenience function for quick auto-configuration + */ +export async function autoConfigureBrainy(hints?: { + expectedDataSize?: number + s3Available?: boolean + memoryBudget?: number +}): Promise { + const autoConfig = AutoConfiguration.getInstance() + return autoConfig.detectAndConfigure(hints) +} + +/** + * Get quick setup configuration for common scenarios + */ +export async function getQuickSetup(scenario: 'small' | 'medium' | 'large' | 'enterprise') { + const autoConfig = AutoConfiguration.getInstance() + return autoConfig.getQuickSetupConfig(scenario) +} \ No newline at end of file