chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,341 @@
|
|||
/**
|
||||
* Automatic Configuration System for Brainy Vector Database
|
||||
* Detects environment, resources, and data patterns to provide optimal settings
|
||||
*/
|
||||
import { isBrowser, isNode, isThreadingAvailable } from './environment.js';
|
||||
/**
|
||||
* Automatic configuration system that detects environment and optimizes settings
|
||||
*/
|
||||
export class AutoConfiguration {
|
||||
constructor() {
|
||||
this.cachedConfig = null;
|
||||
this.datasetStats = { estimatedSize: 0 };
|
||||
}
|
||||
static getInstance() {
|
||||
if (!AutoConfiguration.instance) {
|
||||
AutoConfiguration.instance = new AutoConfiguration();
|
||||
}
|
||||
return AutoConfiguration.instance;
|
||||
}
|
||||
/**
|
||||
* Detect environment and generate optimal configuration
|
||||
*/
|
||||
async detectAndConfigure(hints) {
|
||||
if (this.cachedConfig && !hints) {
|
||||
return this.cachedConfig;
|
||||
}
|
||||
const environment = this.detectEnvironment();
|
||||
const resources = await this.detectResources();
|
||||
const storage = await this.detectStorageCapabilities(hints?.s3Available);
|
||||
const config = {
|
||||
environment,
|
||||
...resources,
|
||||
...storage,
|
||||
recommendedConfig: this.generateRecommendedConfig(environment, resources, hints),
|
||||
optimizationFlags: this.generateOptimizationFlags(environment, resources)
|
||||
};
|
||||
this.cachedConfig = config;
|
||||
return config;
|
||||
}
|
||||
/**
|
||||
* Update configuration based on runtime dataset analysis
|
||||
*/
|
||||
async adaptToDataset(analysis) {
|
||||
this.datasetStats = analysis;
|
||||
// Regenerate configuration with dataset insights
|
||||
const currentConfig = await this.detectAndConfigure();
|
||||
const adaptedConfig = this.adaptConfigurationToData(currentConfig, analysis);
|
||||
this.cachedConfig = adaptedConfig;
|
||||
return adaptedConfig;
|
||||
}
|
||||
/**
|
||||
* Learn from performance metrics and adjust configuration
|
||||
*/
|
||||
async learnFromPerformance(metrics) {
|
||||
const adjustments = {};
|
||||
// Learn from search performance
|
||||
if (metrics.averageSearchTime > 200) {
|
||||
// Too slow - optimize for speed
|
||||
adjustments.enableDistributedSearch = true;
|
||||
adjustments.maxNodesPerPartition = Math.max(10000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 0.8);
|
||||
}
|
||||
else if (metrics.averageSearchTime < 50) {
|
||||
// Very fast - can optimize for quality
|
||||
adjustments.maxNodesPerPartition = Math.min(100000, (this.cachedConfig?.recommendedConfig.maxNodesPerPartition || 50000) * 1.2);
|
||||
}
|
||||
// Learn from memory usage
|
||||
if (metrics.memoryUsage > (this.cachedConfig?.recommendedConfig.maxMemoryUsage || 0) * 0.9) {
|
||||
// High memory usage - enable compression
|
||||
adjustments.enableCompression = true;
|
||||
}
|
||||
// Learn from cache performance
|
||||
if (metrics.cacheHitRate < 0.7) {
|
||||
// Poor cache performance - enable predictive caching
|
||||
adjustments.enablePredictiveCaching = true;
|
||||
}
|
||||
// Update cached config with learned adjustments
|
||||
if (this.cachedConfig) {
|
||||
this.cachedConfig.recommendedConfig = {
|
||||
...this.cachedConfig.recommendedConfig,
|
||||
...adjustments
|
||||
};
|
||||
}
|
||||
return adjustments;
|
||||
}
|
||||
/**
|
||||
* Get minimal configuration for quick setup
|
||||
*/
|
||||
async getQuickSetupConfig(scenario) {
|
||||
const environment = this.detectEnvironment();
|
||||
const resources = await this.detectResources();
|
||||
switch (scenario) {
|
||||
case 'small':
|
||||
return {
|
||||
expectedDatasetSize: 10000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.3, 1024 * 1024 * 1024), // 1GB max
|
||||
targetSearchLatency: 100,
|
||||
s3Required: false
|
||||
};
|
||||
case 'medium':
|
||||
return {
|
||||
expectedDatasetSize: 100000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.5, 4 * 1024 * 1024 * 1024), // 4GB max
|
||||
targetSearchLatency: 150,
|
||||
s3Required: environment === 'serverless'
|
||||
};
|
||||
case 'large':
|
||||
return {
|
||||
expectedDatasetSize: 1000000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.7, 8 * 1024 * 1024 * 1024), // 8GB max
|
||||
targetSearchLatency: 200,
|
||||
s3Required: true
|
||||
};
|
||||
case 'enterprise':
|
||||
return {
|
||||
expectedDatasetSize: 10000000,
|
||||
maxMemoryUsage: Math.min(resources.availableMemory * 0.8, 32 * 1024 * 1024 * 1024), // 32GB max
|
||||
targetSearchLatency: 300,
|
||||
s3Required: true
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Detect the current runtime environment
|
||||
*/
|
||||
detectEnvironment() {
|
||||
if (isBrowser()) {
|
||||
return 'browser';
|
||||
}
|
||||
if (isNode()) {
|
||||
// Check for serverless environment indicators
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
||||
process.env.VERCEL ||
|
||||
process.env.NETLIFY ||
|
||||
process.env.CLOUDFLARE_WORKERS) {
|
||||
return 'serverless';
|
||||
}
|
||||
return 'nodejs';
|
||||
}
|
||||
return 'unknown';
|
||||
}
|
||||
/**
|
||||
* Detect available system resources
|
||||
*/
|
||||
async detectResources() {
|
||||
let availableMemory = 2 * 1024 * 1024 * 1024; // Default 2GB
|
||||
let cpuCores = 4; // Default 4 cores
|
||||
// Browser memory detection
|
||||
if (isBrowser()) {
|
||||
// @ts-ignore - navigator.deviceMemory is experimental
|
||||
if (navigator.deviceMemory) {
|
||||
// @ts-ignore
|
||||
availableMemory = navigator.deviceMemory * 1024 * 1024 * 1024 * 0.3; // Use 30% of device memory
|
||||
}
|
||||
else {
|
||||
availableMemory = 512 * 1024 * 1024; // Conservative 512MB for browsers
|
||||
}
|
||||
cpuCores = navigator.hardwareConcurrency || 4;
|
||||
}
|
||||
// Node.js memory detection
|
||||
if (isNode()) {
|
||||
try {
|
||||
const os = await import('os');
|
||||
availableMemory = os.totalmem() * 0.7; // Use 70% of total memory
|
||||
cpuCores = os.cpus().length;
|
||||
}
|
||||
catch (error) {
|
||||
// Fallback to defaults
|
||||
}
|
||||
}
|
||||
return {
|
||||
availableMemory,
|
||||
cpuCores,
|
||||
threadingAvailable: isThreadingAvailable()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Detect available storage capabilities
|
||||
*/
|
||||
async detectStorageCapabilities(s3Hint) {
|
||||
let persistentStorageAvailable = false;
|
||||
let s3StorageDetected = s3Hint || false;
|
||||
if (isBrowser()) {
|
||||
// Check for OPFS support
|
||||
persistentStorageAvailable = 'navigator' in globalThis &&
|
||||
'storage' in navigator &&
|
||||
'getDirectory' in navigator.storage;
|
||||
}
|
||||
if (isNode()) {
|
||||
persistentStorageAvailable = true; // Always available in Node.js
|
||||
// Check for AWS SDK or S3 environment variables
|
||||
s3StorageDetected = s3Hint ||
|
||||
!!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ||
|
||||
!!(process.env.S3_BUCKET_NAME);
|
||||
}
|
||||
return {
|
||||
persistentStorageAvailable,
|
||||
s3StorageDetected
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Generate recommended configuration based on detected environment and resources
|
||||
*/
|
||||
generateRecommendedConfig(environment, resources, hints) {
|
||||
const datasetSize = hints?.expectedDataSize || this.estimateDatasetSize();
|
||||
const memoryBudget = hints?.memoryBudget || Math.floor(resources.availableMemory * 0.6);
|
||||
// Base configuration
|
||||
let config = {
|
||||
expectedDatasetSize: datasetSize,
|
||||
maxMemoryUsage: memoryBudget,
|
||||
targetSearchLatency: 150,
|
||||
enablePartitioning: datasetSize > 25000,
|
||||
enableCompression: environment === 'browser' || memoryBudget < 2 * 1024 * 1024 * 1024,
|
||||
enableDistributedSearch: resources.cpuCores > 2 && datasetSize > 50000,
|
||||
enablePredictiveCaching: true,
|
||||
partitionStrategy: 'semantic',
|
||||
maxNodesPerPartition: 50000,
|
||||
semanticClusters: 8
|
||||
};
|
||||
// Environment-specific adjustments
|
||||
switch (environment) {
|
||||
case 'browser':
|
||||
config = {
|
||||
...config,
|
||||
maxMemoryUsage: Math.min(memoryBudget, 1024 * 1024 * 1024), // Cap at 1GB
|
||||
targetSearchLatency: 200, // More lenient for browsers
|
||||
enableCompression: true, // Always enable for browsers
|
||||
maxNodesPerPartition: 25000, // Smaller partitions
|
||||
semanticClusters: 4 // Fewer clusters to save memory
|
||||
};
|
||||
break;
|
||||
case 'serverless':
|
||||
config = {
|
||||
...config,
|
||||
targetSearchLatency: 500, // Account for cold starts
|
||||
enablePredictiveCaching: false, // Avoid background processes
|
||||
maxNodesPerPartition: 30000 // Moderate partition size
|
||||
};
|
||||
break;
|
||||
case 'nodejs':
|
||||
config = {
|
||||
...config,
|
||||
targetSearchLatency: 100, // Aggressive for Node.js
|
||||
maxNodesPerPartition: Math.min(100000, Math.floor(datasetSize / 10)), // Larger partitions
|
||||
semanticClusters: Math.min(16, Math.max(4, Math.floor(datasetSize / 50000))) // Scale clusters with data
|
||||
};
|
||||
break;
|
||||
}
|
||||
// Dataset size adjustments
|
||||
if (datasetSize > 1000000) {
|
||||
config.semanticClusters = Math.min(32, Math.floor(datasetSize / 100000));
|
||||
config.maxNodesPerPartition = 100000;
|
||||
}
|
||||
else if (datasetSize < 10000) {
|
||||
config.enablePartitioning = false;
|
||||
config.enableDistributedSearch = false;
|
||||
config.partitionStrategy = 'semantic'; // Keep semantic but disable partitioning
|
||||
}
|
||||
return config;
|
||||
}
|
||||
/**
|
||||
* Generate optimization flags based on environment and resources
|
||||
*/
|
||||
generateOptimizationFlags(environment, resources) {
|
||||
return {
|
||||
useMemoryMapping: environment === 'nodejs' && resources.availableMemory > 4 * 1024 * 1024 * 1024,
|
||||
aggressiveCaching: resources.availableMemory > 2 * 1024 * 1024 * 1024,
|
||||
backgroundOptimization: environment !== 'serverless' && resources.cpuCores > 2,
|
||||
compressionLevel: resources.availableMemory < 1024 * 1024 * 1024 ? 'aggressive' :
|
||||
resources.availableMemory < 4 * 1024 * 1024 * 1024 ? 'light' : 'none'
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Adapt configuration based on actual dataset analysis
|
||||
*/
|
||||
adaptConfigurationToData(baseConfig, analysis) {
|
||||
const updatedConfig = { ...baseConfig };
|
||||
// Adjust based on actual dataset size
|
||||
if (analysis.estimatedSize !== baseConfig.recommendedConfig.expectedDatasetSize) {
|
||||
const sizeRatio = analysis.estimatedSize / baseConfig.recommendedConfig.expectedDatasetSize;
|
||||
updatedConfig.recommendedConfig.expectedDatasetSize = analysis.estimatedSize;
|
||||
// Scale partition size with dataset
|
||||
if (sizeRatio > 2) {
|
||||
updatedConfig.recommendedConfig.maxNodesPerPartition = Math.min(100000, Math.floor(updatedConfig.recommendedConfig.maxNodesPerPartition * 1.5));
|
||||
updatedConfig.recommendedConfig.semanticClusters = Math.min(32, Math.floor(updatedConfig.recommendedConfig.semanticClusters * 1.5));
|
||||
}
|
||||
}
|
||||
// Adjust based on vector dimension
|
||||
if (analysis.vectorDimension) {
|
||||
if (analysis.vectorDimension > 1024) {
|
||||
// High-dimensional vectors - optimize for compression
|
||||
updatedConfig.recommendedConfig.enableCompression = true;
|
||||
updatedConfig.optimizationFlags.compressionLevel = 'aggressive';
|
||||
}
|
||||
}
|
||||
// Adjust based on access patterns
|
||||
if (analysis.accessPatterns === 'read-heavy') {
|
||||
updatedConfig.recommendedConfig.enablePredictiveCaching = true;
|
||||
updatedConfig.optimizationFlags.aggressiveCaching = true;
|
||||
}
|
||||
else if (analysis.accessPatterns === 'write-heavy') {
|
||||
updatedConfig.recommendedConfig.enablePredictiveCaching = false;
|
||||
updatedConfig.optimizationFlags.backgroundOptimization = false;
|
||||
}
|
||||
return updatedConfig;
|
||||
}
|
||||
/**
|
||||
* Estimate dataset size if not provided
|
||||
*/
|
||||
estimateDatasetSize() {
|
||||
// Start with conservative estimate
|
||||
const environment = this.detectEnvironment();
|
||||
switch (environment) {
|
||||
case 'browser': return 10000;
|
||||
case 'serverless': return 50000;
|
||||
case 'nodejs': return 100000;
|
||||
default: return 25000;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Reset cached configuration (for testing or manual refresh)
|
||||
*/
|
||||
resetCache() {
|
||||
this.cachedConfig = null;
|
||||
this.datasetStats = { estimatedSize: 0 };
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Convenience function for quick auto-configuration
|
||||
*/
|
||||
export async function autoConfigureBrainy(hints) {
|
||||
const autoConfig = AutoConfiguration.getInstance();
|
||||
return autoConfig.detectAndConfigure(hints);
|
||||
}
|
||||
/**
|
||||
* Get quick setup configuration for common scenarios
|
||||
*/
|
||||
export async function getQuickSetup(scenario) {
|
||||
const autoConfig = AutoConfiguration.getInstance();
|
||||
return autoConfig.getQuickSetupConfig(scenario);
|
||||
}
|
||||
//# sourceMappingURL=autoConfiguration.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue