feat(auto-configuration): implement automatic configuration system for optimal settings
This commit is contained in:
parent
fd3cfdda6b
commit
8f6f657ba0
1 changed files with 474 additions and 0 deletions
474
src/utils/autoConfiguration.ts
Normal file
474
src/utils/autoConfiguration.ts
Normal file
|
|
@ -0,0 +1,474 @@
|
|||
/**
|
||||
* Automatic Configuration System for Brainy Vector Database
|
||||
* Detects environment, resources, and data patterns to provide optimal settings
|
||||
*/
|
||||
|
||||
import { isBrowser, isNode, isThreadingAvailable } from './environment.js'
|
||||
|
||||
export interface AutoConfigResult {
|
||||
// Environment details
|
||||
environment: 'browser' | 'nodejs' | 'serverless' | 'unknown'
|
||||
|
||||
// Resource detection
|
||||
availableMemory: number // bytes
|
||||
cpuCores: number
|
||||
threadingAvailable: boolean
|
||||
|
||||
// Storage capabilities
|
||||
persistentStorageAvailable: boolean
|
||||
s3StorageDetected: boolean
|
||||
|
||||
// Recommended configuration
|
||||
recommendedConfig: {
|
||||
expectedDatasetSize: number
|
||||
maxMemoryUsage: number
|
||||
targetSearchLatency: number
|
||||
enablePartitioning: boolean
|
||||
enableCompression: boolean
|
||||
enableDistributedSearch: boolean
|
||||
enablePredictiveCaching: boolean
|
||||
partitionStrategy: 'semantic' | 'hash'
|
||||
maxNodesPerPartition: number
|
||||
semanticClusters: number
|
||||
}
|
||||
|
||||
// Performance optimization flags
|
||||
optimizationFlags: {
|
||||
useMemoryMapping: boolean
|
||||
aggressiveCaching: boolean
|
||||
backgroundOptimization: boolean
|
||||
compressionLevel: 'none' | 'light' | 'aggressive'
|
||||
}
|
||||
}
|
||||
|
||||
export interface DatasetAnalysis {
|
||||
estimatedSize: number
|
||||
vectorDimension?: number
|
||||
growthRate?: number // vectors per second
|
||||
accessPatterns?: 'read-heavy' | 'write-heavy' | 'balanced'
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic configuration system that detects environment and optimizes settings
|
||||
*/
|
||||
export class AutoConfiguration {
|
||||
private static instance: AutoConfiguration
|
||||
private cachedConfig: AutoConfigResult | null = null
|
||||
private datasetStats: DatasetAnalysis = { estimatedSize: 0 }
|
||||
|
||||
private constructor() {}
|
||||
|
||||
public static getInstance(): AutoConfiguration {
|
||||
if (!AutoConfiguration.instance) {
|
||||
AutoConfiguration.instance = new AutoConfiguration()
|
||||
}
|
||||
return AutoConfiguration.instance
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect environment and generate optimal configuration
|
||||
*/
|
||||
public async detectAndConfigure(hints?: {
|
||||
expectedDataSize?: number
|
||||
s3Available?: boolean
|
||||
memoryBudget?: number
|
||||
}): Promise<AutoConfigResult> {
|
||||
if (this.cachedConfig && !hints) {
|
||||
return this.cachedConfig
|
||||
}
|
||||
|
||||
const environment = this.detectEnvironment()
|
||||
const resources = await this.detectResources()
|
||||
const storage = await this.detectStorageCapabilities(hints?.s3Available)
|
||||
|
||||
const config: AutoConfigResult = {
|
||||
environment,
|
||||
...resources,
|
||||
...storage,
|
||||
recommendedConfig: this.generateRecommendedConfig(environment, resources, hints),
|
||||
optimizationFlags: this.generateOptimizationFlags(environment, resources)
|
||||
}
|
||||
|
||||
this.cachedConfig = config
|
||||
return config
|
||||
}
|
||||
|
||||
/**
|
||||
* Update configuration based on runtime dataset analysis
|
||||
*/
|
||||
public async adaptToDataset(analysis: DatasetAnalysis): Promise<AutoConfigResult> {
|
||||
this.datasetStats = analysis
|
||||
|
||||
// Regenerate configuration with dataset insights
|
||||
const currentConfig = await this.detectAndConfigure()
|
||||
const adaptedConfig = this.adaptConfigurationToData(currentConfig, analysis)
|
||||
|
||||
this.cachedConfig = adaptedConfig
|
||||
return adaptedConfig
|
||||
}
|
||||
|
||||
/**
|
||||
* Learn from performance metrics and adjust configuration
|
||||
*/
|
||||
public async learnFromPerformance(metrics: {
|
||||
averageSearchTime: number
|
||||
memoryUsage: number
|
||||
cacheHitRate: number
|
||||
errorRate: number
|
||||
}): Promise<Partial<AutoConfigResult['recommendedConfig']>> {
|
||||
const adjustments: Partial<AutoConfigResult['recommendedConfig']> = {}
|
||||
|
||||
// Learn from search performance
|
||||
if (metrics.averageSearchTime > 200) {
|
||||
// Too slow - optimize for speed
|
||||
adjustments.enableDistributedSearch = true
|
||||
adjustments.maxNodesPerPartition = Math.max(10000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 0.8)
|
||||
} else if (metrics.averageSearchTime < 50) {
|
||||
// Very fast - can optimize for quality
|
||||
adjustments.maxNodesPerPartition = Math.min(100000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 1.2)
|
||||
}
|
||||
|
||||
// Learn from memory usage
|
||||
if (metrics.memoryUsage > (this.cachedConfig?.recommendedConfig.maxMemoryUsage || 0) * 0.9) {
|
||||
// High memory usage - enable compression
|
||||
adjustments.enableCompression = true
|
||||
}
|
||||
|
||||
// Learn from cache performance
|
||||
if (metrics.cacheHitRate < 0.7) {
|
||||
// Poor cache performance - enable predictive caching
|
||||
adjustments.enablePredictiveCaching = true
|
||||
}
|
||||
|
||||
// Update cached config with learned adjustments
|
||||
if (this.cachedConfig) {
|
||||
this.cachedConfig.recommendedConfig = {
|
||||
...this.cachedConfig.recommendedConfig,
|
||||
...adjustments
|
||||
}
|
||||
}
|
||||
|
||||
return adjustments
|
||||
}
|
||||
|
||||
/**
|
||||
* Get minimal configuration for quick setup
|
||||
*/
|
||||
public async getQuickSetupConfig(scenario: 'small' | 'medium' | 'large' | 'enterprise'): Promise<{
|
||||
expectedDatasetSize: number
|
||||
maxMemoryUsage: number
|
||||
targetSearchLatency: number
|
||||
s3Required: boolean
|
||||
}> {
|
||||
const environment = this.detectEnvironment()
|
||||
const resources = await this.detectResources()
|
||||
|
||||
switch (scenario) {
|
||||
case 'small':
|
||||
return {
|
||||
expectedDatasetSize: 10000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.3, 1024 * 1024 * 1024), // 1GB max
|
||||
targetSearchLatency: 100,
|
||||
s3Required: false
|
||||
}
|
||||
|
||||
case 'medium':
|
||||
return {
|
||||
expectedDatasetSize: 100000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.5, 4 * 1024 * 1024 * 1024), // 4GB max
|
||||
targetSearchLatency: 150,
|
||||
s3Required: environment === 'serverless'
|
||||
}
|
||||
|
||||
case 'large':
|
||||
return {
|
||||
expectedDatasetSize: 1000000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.7, 8 * 1024 * 1024 * 1024), // 8GB max
|
||||
targetSearchLatency: 200,
|
||||
s3Required: true
|
||||
}
|
||||
|
||||
case 'enterprise':
|
||||
return {
|
||||
expectedDatasetSize: 10000000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.8, 32 * 1024 * 1024 * 1024), // 32GB max
|
||||
targetSearchLatency: 300,
|
||||
s3Required: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect the current runtime environment
|
||||
*/
|
||||
private detectEnvironment(): 'browser' | 'nodejs' | 'serverless' | 'unknown' {
|
||||
if (isBrowser()) {
|
||||
return 'browser'
|
||||
}
|
||||
|
||||
if (isNode()) {
|
||||
// Check for serverless environment indicators
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.CLOUDFLARE_WORKERS) {
|
||||
return 'serverless'
|
||||
}
|
||||
return 'nodejs'
|
||||
}
|
||||
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect available system resources
|
||||
*/
|
||||
private async detectResources(): Promise<{
|
||||
availableMemory: number
|
||||
cpuCores: number
|
||||
threadingAvailable: boolean
|
||||
}> {
|
||||
let availableMemory = 2 * 1024 * 1024 * 1024 // Default 2GB
|
||||
let cpuCores = 4 // Default 4 cores
|
||||
|
||||
// Browser memory detection
|
||||
if (isBrowser()) {
|
||||
// @ts-ignore - navigator.deviceMemory is experimental
|
||||
if (navigator.deviceMemory) {
|
||||
// @ts-ignore
|
||||
availableMemory = navigator.deviceMemory * 1024 * 1024 * 1024 * 0.3 // Use 30% of device memory
|
||||
} else {
|
||||
availableMemory = 512 * 1024 * 1024 // Conservative 512MB for browsers
|
||||
}
|
||||
|
||||
cpuCores = navigator.hardwareConcurrency || 4
|
||||
}
|
||||
|
||||
// Node.js memory detection
|
||||
if (isNode()) {
|
||||
try {
|
||||
const os = await import('os')
|
||||
availableMemory = os.totalmem() * 0.7 // Use 70% of total memory
|
||||
cpuCores = os.cpus().length
|
||||
} catch (error) {
|
||||
// Fallback to defaults
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
availableMemory,
|
||||
cpuCores,
|
||||
threadingAvailable: isThreadingAvailable()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect available storage capabilities
|
||||
*/
|
||||
private async detectStorageCapabilities(s3Hint?: boolean): Promise<{
|
||||
persistentStorageAvailable: boolean
|
||||
s3StorageDetected: boolean
|
||||
}> {
|
||||
let persistentStorageAvailable = false
|
||||
let s3StorageDetected = s3Hint || false
|
||||
|
||||
if (isBrowser()) {
|
||||
// Check for OPFS support
|
||||
persistentStorageAvailable = 'navigator' in globalThis &&
|
||||
'storage' in navigator &&
|
||||
'getDirectory' in navigator.storage
|
||||
}
|
||||
|
||||
if (isNode()) {
|
||||
persistentStorageAvailable = true // Always available in Node.js
|
||||
|
||||
// Check for AWS SDK or S3 environment variables
|
||||
s3StorageDetected = s3Hint ||
|
||||
!!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
||||
!!(process.env.S3_BUCKET_NAME)
|
||||
}
|
||||
|
||||
return {
|
||||
persistentStorageAvailable,
|
||||
s3StorageDetected
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate recommended configuration based on detected environment and resources
|
||||
*/
|
||||
private generateRecommendedConfig(
|
||||
environment: string,
|
||||
resources: { availableMemory: number; cpuCores: number },
|
||||
hints?: { expectedDataSize?: number; memoryBudget?: number }
|
||||
): AutoConfigResult['recommendedConfig'] {
|
||||
const datasetSize = hints?.expectedDataSize || this.estimateDatasetSize()
|
||||
const memoryBudget = hints?.memoryBudget || Math.floor(resources.availableMemory * 0.6)
|
||||
|
||||
// Base configuration
|
||||
let config = {
|
||||
expectedDatasetSize: datasetSize,
|
||||
maxMemoryUsage: memoryBudget,
|
||||
targetSearchLatency: 150,
|
||||
enablePartitioning: datasetSize > 25000,
|
||||
enableCompression: environment === 'browser' || memoryBudget < 2 * 1024 * 1024 * 1024,
|
||||
enableDistributedSearch: resources.cpuCores > 2 && datasetSize > 50000,
|
||||
enablePredictiveCaching: true,
|
||||
partitionStrategy: 'semantic' as const,
|
||||
maxNodesPerPartition: 50000,
|
||||
semanticClusters: 8
|
||||
}
|
||||
|
||||
// Environment-specific adjustments
|
||||
switch (environment) {
|
||||
case 'browser':
|
||||
config = {
|
||||
...config,
|
||||
maxMemoryUsage: Math.min(memoryBudget, 1024 * 1024 * 1024), // Cap at 1GB
|
||||
targetSearchLatency: 200, // More lenient for browsers
|
||||
enableCompression: true, // Always enable for browsers
|
||||
maxNodesPerPartition: 25000, // Smaller partitions
|
||||
semanticClusters: 4 // Fewer clusters to save memory
|
||||
}
|
||||
break
|
||||
|
||||
case 'serverless':
|
||||
config = {
|
||||
...config,
|
||||
targetSearchLatency: 500, // Account for cold starts
|
||||
enablePredictiveCaching: false, // Avoid background processes
|
||||
maxNodesPerPartition: 30000 // Moderate partition size
|
||||
}
|
||||
break
|
||||
|
||||
case 'nodejs':
|
||||
config = {
|
||||
...config,
|
||||
targetSearchLatency: 100, // Aggressive for Node.js
|
||||
maxNodesPerPartition: Math.min(100000, Math.floor(datasetSize / 10)), // Larger partitions
|
||||
semanticClusters: Math.min(16, Math.max(4, Math.floor(datasetSize / 50000))) // Scale clusters with data
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Dataset size adjustments
|
||||
if (datasetSize > 1000000) {
|
||||
config.semanticClusters = Math.min(32, Math.floor(datasetSize / 100000))
|
||||
config.maxNodesPerPartition = 100000
|
||||
} else if (datasetSize < 10000) {
|
||||
config.enablePartitioning = false
|
||||
config.enableDistributedSearch = false
|
||||
config.partitionStrategy = 'semantic' // Keep semantic but disable partitioning
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate optimization flags based on environment and resources
|
||||
*/
|
||||
private generateOptimizationFlags(
|
||||
environment: string,
|
||||
resources: { availableMemory: number; cpuCores: number }
|
||||
): AutoConfigResult['optimizationFlags'] {
|
||||
return {
|
||||
useMemoryMapping: environment === 'nodejs' && resources.availableMemory > 4 * 1024 * 1024 * 1024,
|
||||
aggressiveCaching: resources.availableMemory > 2 * 1024 * 1024 * 1024,
|
||||
backgroundOptimization: environment !== 'serverless' && resources.cpuCores > 2,
|
||||
compressionLevel: resources.availableMemory < 1024 * 1024 * 1024 ? 'aggressive' :
|
||||
resources.availableMemory < 4 * 1024 * 1024 * 1024 ? 'light' : 'none'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapt configuration based on actual dataset analysis
|
||||
*/
|
||||
private adaptConfigurationToData(
|
||||
baseConfig: AutoConfigResult,
|
||||
analysis: DatasetAnalysis
|
||||
): AutoConfigResult {
|
||||
const updatedConfig = { ...baseConfig }
|
||||
|
||||
// Adjust based on actual dataset size
|
||||
if (analysis.estimatedSize !== baseConfig.recommendedConfig.expectedDatasetSize) {
|
||||
const sizeRatio = analysis.estimatedSize / baseConfig.recommendedConfig.expectedDatasetSize
|
||||
|
||||
updatedConfig.recommendedConfig.expectedDatasetSize = analysis.estimatedSize
|
||||
|
||||
// Scale partition size with dataset
|
||||
if (sizeRatio > 2) {
|
||||
updatedConfig.recommendedConfig.maxNodesPerPartition = Math.min(
|
||||
100000,
|
||||
Math.floor(updatedConfig.recommendedConfig.maxNodesPerPartition * 1.5)
|
||||
)
|
||||
updatedConfig.recommendedConfig.semanticClusters = Math.min(
|
||||
32,
|
||||
Math.floor(updatedConfig.recommendedConfig.semanticClusters * 1.5)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Adjust based on vector dimension
|
||||
if (analysis.vectorDimension) {
|
||||
if (analysis.vectorDimension > 1024) {
|
||||
// High-dimensional vectors - optimize for compression
|
||||
updatedConfig.recommendedConfig.enableCompression = true
|
||||
updatedConfig.optimizationFlags.compressionLevel = 'aggressive'
|
||||
}
|
||||
}
|
||||
|
||||
// Adjust based on access patterns
|
||||
if (analysis.accessPatterns === 'read-heavy') {
|
||||
updatedConfig.recommendedConfig.enablePredictiveCaching = true
|
||||
updatedConfig.optimizationFlags.aggressiveCaching = true
|
||||
} else if (analysis.accessPatterns === 'write-heavy') {
|
||||
updatedConfig.recommendedConfig.enablePredictiveCaching = false
|
||||
updatedConfig.optimizationFlags.backgroundOptimization = false
|
||||
}
|
||||
|
||||
return updatedConfig
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate dataset size if not provided
|
||||
*/
|
||||
private estimateDatasetSize(): number {
|
||||
// Start with conservative estimate
|
||||
const environment = this.detectEnvironment()
|
||||
|
||||
switch (environment) {
|
||||
case 'browser': return 10000
|
||||
case 'serverless': return 50000
|
||||
case 'nodejs': return 100000
|
||||
default: return 25000
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset cached configuration (for testing or manual refresh)
|
||||
*/
|
||||
public resetCache(): void {
|
||||
this.cachedConfig = null
|
||||
this.datasetStats = { estimatedSize: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience function for quick auto-configuration
|
||||
*/
|
||||
export async function autoConfigureBrainy(hints?: {
|
||||
expectedDataSize?: number
|
||||
s3Available?: boolean
|
||||
memoryBudget?: number
|
||||
}): Promise<AutoConfigResult> {
|
||||
const autoConfig = AutoConfiguration.getInstance()
|
||||
return autoConfig.detectAndConfigure(hints)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get quick setup configuration for common scenarios
|
||||
*/
|
||||
export async function getQuickSetup(scenario: 'small' | 'medium' | 'large' | 'enterprise') {
|
||||
const autoConfig = AutoConfiguration.getInstance()
|
||||
return autoConfig.getQuickSetupConfig(scenario)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue