chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,215 @@
|
|||
/**
|
||||
* Shared Configuration Manager
|
||||
* Ensures configuration consistency across multiple instances using shared storage
|
||||
*/
|
||||
/**
|
||||
* Manages configuration consistency for shared storage
|
||||
*/
|
||||
export class SharedConfigManager {
|
||||
/**
|
||||
* Load or create shared configuration
|
||||
* When connecting to existing data, this OVERRIDES auto-configuration!
|
||||
*/
|
||||
static async loadOrCreateSharedConfig(storage, localConfig) {
|
||||
const warnings = [];
|
||||
try {
|
||||
// Check if we're using shared storage
|
||||
if (!this.isSharedStorage(localConfig.storageType)) {
|
||||
// Local storage - use local config
|
||||
return { config: localConfig, warnings: [] };
|
||||
}
|
||||
// Try to load existing configuration from shared storage
|
||||
const existingConfig = await this.loadConfigFromStorage(storage);
|
||||
if (existingConfig) {
|
||||
// EXISTING SHARED DATA - Must use its configuration!
|
||||
console.log('📁 Found existing shared data configuration');
|
||||
// Check for critical mismatches
|
||||
const mismatches = this.checkCriticalMismatches(localConfig, existingConfig);
|
||||
if (mismatches.length > 0) {
|
||||
console.warn('⚠️ Configuration override required for shared storage:');
|
||||
mismatches.forEach(m => {
|
||||
console.warn(` - ${m.param}: ${m.local} → ${m.shared} (using shared)`);
|
||||
warnings.push(`${m.param} overridden: ${m.local} → ${m.shared}`);
|
||||
});
|
||||
}
|
||||
// Override critical parameters with shared values
|
||||
const mergedConfig = this.mergeWithSharedConfig(localConfig, existingConfig);
|
||||
// Update last accessed
|
||||
await this.updateAccessInfo(storage, existingConfig);
|
||||
return { config: mergedConfig, warnings };
|
||||
}
|
||||
else {
|
||||
// NEW SHARED STORAGE - Save our configuration
|
||||
console.log('📝 Initializing new shared storage with configuration');
|
||||
const sharedConfig = this.createSharedConfig(localConfig);
|
||||
await this.saveConfigToStorage(storage, sharedConfig);
|
||||
return { config: localConfig, warnings: [] };
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to manage shared configuration:', error);
|
||||
warnings.push('Could not verify shared configuration - proceeding with caution');
|
||||
return { config: localConfig, warnings };
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if storage type is shared (multi-instance)
|
||||
*/
|
||||
static isSharedStorage(storageType) {
|
||||
return ['s3', 'gcs', 'r2'].includes(storageType);
|
||||
}
|
||||
/**
|
||||
* Load configuration from shared storage
|
||||
*/
|
||||
static async loadConfigFromStorage(storage) {
|
||||
try {
|
||||
const configData = await storage.get(this.CONFIG_FILE);
|
||||
if (!configData)
|
||||
return null;
|
||||
const config = JSON.parse(configData);
|
||||
return config;
|
||||
}
|
||||
catch (error) {
|
||||
// Config doesn't exist yet
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Save configuration to shared storage
|
||||
*/
|
||||
static async saveConfigToStorage(storage, config) {
|
||||
try {
|
||||
await storage.set(this.CONFIG_FILE, JSON.stringify(config, null, 2));
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to save shared configuration:', error);
|
||||
throw new Error('Cannot initialize shared storage without saving configuration');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Create shared configuration from local config
|
||||
*/
|
||||
static createSharedConfig(localConfig) {
|
||||
return {
|
||||
version: '2.10.0', // Brainy version
|
||||
precision: localConfig.embeddingOptions?.precision || 'fp32',
|
||||
dimensions: 384, // Fixed for all-MiniLM-L6-v2
|
||||
hnswM: localConfig.hnsw?.M || 16,
|
||||
hnswEfConstruction: localConfig.hnsw?.efConstruction || 200,
|
||||
distanceFunction: localConfig.distanceFunction || 'cosine',
|
||||
createdAt: new Date().toISOString(),
|
||||
lastUpdated: new Date().toISOString(),
|
||||
instanceCount: 1,
|
||||
lastAccessedBy: this.getInstanceIdentifier()
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Check for critical configuration mismatches
|
||||
*/
|
||||
static checkCriticalMismatches(localConfig, sharedConfig) {
|
||||
const mismatches = [];
|
||||
// Model precision - CRITICAL!
|
||||
const localPrecision = localConfig.embeddingOptions?.precision || 'fp32';
|
||||
if (localPrecision !== sharedConfig.precision) {
|
||||
mismatches.push({
|
||||
param: 'Model Precision',
|
||||
local: localPrecision,
|
||||
shared: sharedConfig.precision
|
||||
});
|
||||
}
|
||||
// HNSW parameters - Important for index consistency
|
||||
const localM = localConfig.hnsw?.M || 16;
|
||||
if (localM !== sharedConfig.hnswM) {
|
||||
mismatches.push({
|
||||
param: 'HNSW M',
|
||||
local: localM,
|
||||
shared: sharedConfig.hnswM
|
||||
});
|
||||
}
|
||||
const localEf = localConfig.hnsw?.efConstruction || 200;
|
||||
if (localEf !== sharedConfig.hnswEfConstruction) {
|
||||
mismatches.push({
|
||||
param: 'HNSW efConstruction',
|
||||
local: localEf,
|
||||
shared: sharedConfig.hnswEfConstruction
|
||||
});
|
||||
}
|
||||
// Distance function
|
||||
const localDistance = localConfig.distanceFunction || 'cosine';
|
||||
if (localDistance !== sharedConfig.distanceFunction) {
|
||||
mismatches.push({
|
||||
param: 'Distance Function',
|
||||
local: localDistance,
|
||||
shared: sharedConfig.distanceFunction
|
||||
});
|
||||
}
|
||||
return mismatches;
|
||||
}
|
||||
/**
|
||||
* Merge local config with shared config (shared takes precedence for critical params)
|
||||
*/
|
||||
static mergeWithSharedConfig(localConfig, sharedConfig) {
|
||||
return {
|
||||
...localConfig,
|
||||
// Override critical parameters with shared values
|
||||
embeddingOptions: {
|
||||
...localConfig.embeddingOptions,
|
||||
precision: sharedConfig.precision // MUST use shared precision!
|
||||
},
|
||||
hnsw: {
|
||||
...localConfig.hnsw,
|
||||
M: sharedConfig.hnswM,
|
||||
efConstruction: sharedConfig.hnswEfConstruction
|
||||
},
|
||||
distanceFunction: sharedConfig.distanceFunction,
|
||||
// Add metadata about shared configuration
|
||||
_sharedConfig: {
|
||||
loaded: true,
|
||||
version: sharedConfig.version,
|
||||
createdAt: sharedConfig.createdAt,
|
||||
precision: sharedConfig.precision
|
||||
}
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Update access information in shared config
|
||||
*/
|
||||
static async updateAccessInfo(storage, config) {
|
||||
try {
|
||||
config.lastUpdated = new Date().toISOString();
|
||||
config.instanceCount = (config.instanceCount || 0) + 1;
|
||||
config.lastAccessedBy = this.getInstanceIdentifier();
|
||||
await this.saveConfigToStorage(storage, config);
|
||||
}
|
||||
catch {
|
||||
// Non-critical - don't fail if we can't update access info
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get unique identifier for this instance
|
||||
*/
|
||||
static getInstanceIdentifier() {
|
||||
if (process.env.HOSTNAME)
|
||||
return process.env.HOSTNAME;
|
||||
if (process.env.CONTAINER_ID)
|
||||
return process.env.CONTAINER_ID;
|
||||
if (process.env.K_SERVICE)
|
||||
return process.env.K_SERVICE;
|
||||
if (process.env.AWS_LAMBDA_FUNCTION_NAME)
|
||||
return process.env.AWS_LAMBDA_FUNCTION_NAME;
|
||||
// Generate a random identifier
|
||||
return `instance-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
}
|
||||
/**
|
||||
* Validate that a configuration is compatible with shared data
|
||||
*/
|
||||
static validateCompatibility(config1, config2) {
|
||||
return (config1.precision === config2.precision &&
|
||||
config1.dimensions === config2.dimensions &&
|
||||
config1.hnswM === config2.hnswM &&
|
||||
config1.hnswEfConstruction === config2.hnswEfConstruction &&
|
||||
config1.distanceFunction === config2.distanceFunction);
|
||||
}
|
||||
}
|
||||
SharedConfigManager.CONFIG_FILE = '.brainy/config.json';
|
||||
//# sourceMappingURL=sharedConfigManager.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue