chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,441 @@
|
|||
/**
|
||||
* Distributed Configuration Manager
|
||||
* Manages shared configuration in S3 for distributed Brainy instances
|
||||
*/
|
||||
import { v4 as uuidv4 } from '../universal/uuid.js';
|
||||
// Constants for config storage locations
|
||||
const DISTRIBUTED_CONFIG_KEY = 'distributed_config';
|
||||
const LEGACY_CONFIG_KEY = '_distributed_config';
|
||||
export class DistributedConfigManager {
|
||||
constructor(storage, distributedConfig, brainyMode) {
|
||||
this.config = null;
|
||||
this.lastConfigVersion = 0;
|
||||
this.hasMigrated = false;
|
||||
this.storage = storage;
|
||||
this.instanceId = distributedConfig?.instanceId || `instance-${uuidv4()}`;
|
||||
// Updated default path to use _system instead of _brainy
|
||||
this.configPath = distributedConfig?.configPath || '_system/distributed_config.json';
|
||||
this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000;
|
||||
this.configCheckInterval = distributedConfig?.configCheckInterval || 10000;
|
||||
this.instanceTimeout = distributedConfig?.instanceTimeout || 60000;
|
||||
// Set role from distributed config if provided
|
||||
if (distributedConfig?.role) {
|
||||
this.role = distributedConfig.role;
|
||||
}
|
||||
// Infer role from Brainy's read/write mode if not explicitly set
|
||||
else if (brainyMode) {
|
||||
if (brainyMode.writeOnly) {
|
||||
this.role = 'writer';
|
||||
}
|
||||
else if (brainyMode.readOnly) {
|
||||
this.role = 'reader';
|
||||
}
|
||||
// If neither readOnly nor writeOnly, role must be explicitly set
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Initialize the distributed configuration
|
||||
*/
|
||||
async initialize() {
|
||||
// Load or create configuration
|
||||
this.config = await this.loadOrCreateConfig();
|
||||
// Determine role if not explicitly set
|
||||
if (!this.role) {
|
||||
this.role = await this.determineRole();
|
||||
}
|
||||
// Register this instance
|
||||
await this.registerInstance();
|
||||
// Start heartbeat and config watching
|
||||
this.startHeartbeat();
|
||||
this.startConfigWatch();
|
||||
return this.config;
|
||||
}
|
||||
/**
|
||||
* Load existing config or create new one
|
||||
*/
|
||||
async loadOrCreateConfig() {
|
||||
// First, try to load from the new location in index folder
|
||||
try {
|
||||
const configData = await this.storage.getStatistics();
|
||||
if (configData && configData.distributedConfig) {
|
||||
this.lastConfigVersion = configData.distributedConfig.version;
|
||||
return configData.distributedConfig;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
// Config doesn't exist in new location yet
|
||||
}
|
||||
// Check if we need to migrate from old location
|
||||
if (!this.hasMigrated) {
|
||||
const migrated = await this.migrateConfigFromLegacyLocation();
|
||||
if (migrated) {
|
||||
return migrated;
|
||||
}
|
||||
}
|
||||
// Legacy fallback - try old location
|
||||
try {
|
||||
const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
||||
if (configData) {
|
||||
// Migrate to new location
|
||||
await this.migrateConfig(configData);
|
||||
this.lastConfigVersion = configData.version;
|
||||
return configData;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
// Config doesn't exist yet
|
||||
}
|
||||
// Create default config
|
||||
const newConfig = {
|
||||
version: 1,
|
||||
updated: new Date().toISOString(),
|
||||
settings: {
|
||||
partitionStrategy: 'hash',
|
||||
partitionCount: 100,
|
||||
embeddingModel: 'text-embedding-ada-002',
|
||||
dimensions: 1536,
|
||||
distanceMetric: 'cosine',
|
||||
hnswParams: {
|
||||
M: 16,
|
||||
efConstruction: 200
|
||||
}
|
||||
},
|
||||
instances: {}
|
||||
};
|
||||
await this.saveConfig(newConfig);
|
||||
return newConfig;
|
||||
}
|
||||
/**
|
||||
* Determine role based on configuration
|
||||
* IMPORTANT: Role must be explicitly set - no automatic assignment based on order
|
||||
*/
|
||||
async determineRole() {
|
||||
// Check environment variable first
|
||||
if (process.env.BRAINY_ROLE) {
|
||||
const role = process.env.BRAINY_ROLE.toLowerCase();
|
||||
if (role === 'writer' || role === 'reader' || role === 'hybrid') {
|
||||
return role;
|
||||
}
|
||||
throw new Error(`Invalid BRAINY_ROLE: ${process.env.BRAINY_ROLE}. Must be 'writer', 'reader', or 'hybrid'`);
|
||||
}
|
||||
// Check if explicitly passed in distributed config
|
||||
if (this.role) {
|
||||
return this.role;
|
||||
}
|
||||
// DO NOT auto-assign roles based on deployment order or existing instances
|
||||
// This is dangerous and can lead to data corruption or loss
|
||||
throw new Error('Distributed mode requires explicit role configuration. ' +
|
||||
'Set BRAINY_ROLE environment variable or pass role in distributed config. ' +
|
||||
'Valid roles: "writer", "reader", "hybrid"');
|
||||
}
|
||||
/**
|
||||
* Check if an instance is still alive
|
||||
*/
|
||||
isInstanceAlive(instance) {
|
||||
const lastSeen = new Date(instance.lastHeartbeat).getTime();
|
||||
const now = Date.now();
|
||||
return (now - lastSeen) < this.instanceTimeout;
|
||||
}
|
||||
/**
|
||||
* Register this instance in the shared config
|
||||
*/
|
||||
async registerInstance() {
|
||||
if (!this.config)
|
||||
return;
|
||||
// Role must be set by this point
|
||||
if (!this.role) {
|
||||
throw new Error('Cannot register instance without a role');
|
||||
}
|
||||
const instanceInfo = {
|
||||
role: this.role,
|
||||
status: 'active',
|
||||
lastHeartbeat: new Date().toISOString(),
|
||||
metrics: {
|
||||
memoryUsage: process.memoryUsage().heapUsed
|
||||
}
|
||||
};
|
||||
// Add endpoint if available
|
||||
if (process.env.SERVICE_ENDPOINT) {
|
||||
instanceInfo.endpoint = process.env.SERVICE_ENDPOINT;
|
||||
}
|
||||
this.config.instances[this.instanceId] = instanceInfo;
|
||||
await this.saveConfig(this.config);
|
||||
}
|
||||
/**
|
||||
* Migrate config from legacy location to new location
|
||||
*/
|
||||
async migrateConfigFromLegacyLocation() {
|
||||
try {
|
||||
// Try to load from old location
|
||||
const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
||||
if (legacyConfig) {
|
||||
console.log('Migrating distributed config from legacy location to index folder...');
|
||||
// Save to new location
|
||||
await this.migrateConfig(legacyConfig);
|
||||
// Delete from old location (optional - we can keep it for rollback)
|
||||
// await this.storage.deleteMetadata(LEGACY_CONFIG_KEY)
|
||||
this.hasMigrated = true;
|
||||
this.lastConfigVersion = legacyConfig.version;
|
||||
return legacyConfig;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error during config migration:', error);
|
||||
}
|
||||
this.hasMigrated = true;
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Migrate config to new location in index folder
|
||||
*/
|
||||
async migrateConfig(config) {
|
||||
// Get existing statistics or create new
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Add distributed config to statistics
|
||||
stats.distributedConfig = config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
}
|
||||
/**
|
||||
* Save configuration with version increment
|
||||
*/
|
||||
async saveConfig(config) {
|
||||
config.version++;
|
||||
config.updated = new Date().toISOString();
|
||||
this.lastConfigVersion = config.version;
|
||||
// Save to new location in index folder along with statistics
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Update distributed config in statistics
|
||||
stats.distributedConfig = config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
this.config = config;
|
||||
}
|
||||
/**
|
||||
* Start heartbeat to keep instance alive in config
|
||||
*/
|
||||
startHeartbeat() {
|
||||
this.heartbeatTimer = setInterval(async () => {
|
||||
await this.updateHeartbeat();
|
||||
}, this.heartbeatInterval);
|
||||
}
|
||||
/**
|
||||
* Update heartbeat and clean stale instances
|
||||
*/
|
||||
async updateHeartbeat() {
|
||||
if (!this.config)
|
||||
return;
|
||||
// Reload config to get latest state
|
||||
try {
|
||||
const latestConfig = await this.loadConfig();
|
||||
if (latestConfig) {
|
||||
this.config = latestConfig;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to reload config:', error);
|
||||
}
|
||||
// Update our heartbeat
|
||||
if (this.config.instances[this.instanceId]) {
|
||||
this.config.instances[this.instanceId].lastHeartbeat = new Date().toISOString();
|
||||
this.config.instances[this.instanceId].status = 'active';
|
||||
// Update metrics if available
|
||||
this.config.instances[this.instanceId].metrics = {
|
||||
memoryUsage: process.memoryUsage().heapUsed
|
||||
};
|
||||
}
|
||||
else {
|
||||
// Re-register if we were removed
|
||||
await this.registerInstance();
|
||||
return;
|
||||
}
|
||||
// Clean up stale instances
|
||||
const now = Date.now();
|
||||
let hasChanges = false;
|
||||
for (const [id, instance] of Object.entries(this.config.instances)) {
|
||||
if (id === this.instanceId)
|
||||
continue;
|
||||
const lastSeen = new Date(instance.lastHeartbeat).getTime();
|
||||
if (now - lastSeen > this.instanceTimeout) {
|
||||
delete this.config.instances[id];
|
||||
hasChanges = true;
|
||||
}
|
||||
}
|
||||
// Save if there were changes
|
||||
if (hasChanges) {
|
||||
await this.saveConfig(this.config);
|
||||
}
|
||||
else {
|
||||
// Just update our heartbeat without version increment
|
||||
// Get existing statistics
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Update distributed config in statistics without version increment
|
||||
stats.distributedConfig = this.config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Start watching for config changes
|
||||
*/
|
||||
startConfigWatch() {
|
||||
this.configWatchTimer = setInterval(async () => {
|
||||
await this.checkForConfigUpdates();
|
||||
}, this.configCheckInterval);
|
||||
}
|
||||
/**
|
||||
* Check for configuration updates
|
||||
*/
|
||||
async checkForConfigUpdates() {
|
||||
try {
|
||||
const latestConfig = await this.loadConfig();
|
||||
if (!latestConfig)
|
||||
return;
|
||||
if (latestConfig.version > this.lastConfigVersion) {
|
||||
this.config = latestConfig;
|
||||
this.lastConfigVersion = latestConfig.version;
|
||||
// Notify listeners of config update
|
||||
if (this.onConfigUpdate) {
|
||||
this.onConfigUpdate(latestConfig);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to check config updates:', error);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Load configuration from storage
|
||||
*/
|
||||
async loadConfig() {
|
||||
try {
|
||||
// Try new location first
|
||||
const stats = await this.storage.getStatistics();
|
||||
if (stats && stats.distributedConfig) {
|
||||
return stats.distributedConfig;
|
||||
}
|
||||
// Fallback to legacy location if not migrated yet
|
||||
if (!this.hasMigrated) {
|
||||
const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
|
||||
if (configData) {
|
||||
// Trigger migration on next save
|
||||
return configData;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to load config:', error);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Get current configuration
|
||||
*/
|
||||
getConfig() {
|
||||
return this.config;
|
||||
}
|
||||
/**
|
||||
* Get instance role
|
||||
*/
|
||||
getRole() {
|
||||
if (!this.role) {
|
||||
throw new Error('Role not initialized');
|
||||
}
|
||||
return this.role;
|
||||
}
|
||||
/**
|
||||
* Get instance ID
|
||||
*/
|
||||
getInstanceId() {
|
||||
return this.instanceId;
|
||||
}
|
||||
/**
|
||||
* Set config update callback
|
||||
*/
|
||||
setOnConfigUpdate(callback) {
|
||||
this.onConfigUpdate = callback;
|
||||
}
|
||||
/**
|
||||
* Get all active instances of a specific role
|
||||
*/
|
||||
getInstancesByRole(role) {
|
||||
if (!this.config)
|
||||
return [];
|
||||
return Object.entries(this.config.instances)
|
||||
.filter(([_, instance]) => instance.role === role &&
|
||||
this.isInstanceAlive(instance))
|
||||
.map(([_, instance]) => instance);
|
||||
}
|
||||
/**
|
||||
* Update instance metrics
|
||||
*/
|
||||
async updateMetrics(metrics) {
|
||||
if (!this.config || !this.config.instances[this.instanceId])
|
||||
return;
|
||||
this.config.instances[this.instanceId].metrics = {
|
||||
...this.config.instances[this.instanceId].metrics,
|
||||
...metrics
|
||||
};
|
||||
// Don't increment version for metric updates
|
||||
// Get existing statistics
|
||||
let stats = await this.storage.getStatistics();
|
||||
if (!stats) {
|
||||
stats = {
|
||||
nounCount: {},
|
||||
verbCount: {},
|
||||
metadataCount: {},
|
||||
hnswIndexSize: 0,
|
||||
lastUpdated: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
// Update distributed config in statistics without version increment
|
||||
stats.distributedConfig = this.config;
|
||||
// Save updated statistics
|
||||
await this.storage.saveStatistics(stats);
|
||||
}
|
||||
/**
|
||||
* Cleanup resources
|
||||
*/
|
||||
async cleanup() {
|
||||
// Stop timers
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
}
|
||||
if (this.configWatchTimer) {
|
||||
clearInterval(this.configWatchTimer);
|
||||
}
|
||||
// Mark instance as inactive
|
||||
if (this.config && this.config.instances[this.instanceId]) {
|
||||
this.config.instances[this.instanceId].status = 'inactive';
|
||||
await this.saveConfig(this.config);
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=configManager.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue