chore: recovery checkpoint - v3.0 API successfully recovered

CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB

Recovery Status:
- Successfully recovered brainy.ts from compiled JavaScript
- All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.)
- Neural subsystem intact (562KB embedded patterns, NLP working)
- Augmentation pipeline operational (20+ augmentations)
- HNSW clustering system complete
- Triple Intelligence compiled (needs constructor fix)
- Test suite validates functionality

Changes preserved:
- 898 files with changes from last 3 days
- 144,475 insertions
- All augmentation improvements
- All test coverage enhancements
- Complete v3.0 feature set

This is a LOCAL checkpoint only - contains recovered work after corruption incident.
Created backup in .backups/brainy-full-20250910-151314.tar.gz

Branch: recovery-checkpoint-20250910-151433
Date: Wed Sep 10 03:18:04 PM PDT 2025

.recovery-workspace/dist-backup-20250910-141917/distributed/cacheSync.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Distributed Cache Synchronization
+ * Provides cache coherence across multiple Brainy instances
+ */
+import { EventEmitter } from 'events';
+export interface CacheSyncConfig {
+    nodeId: string;
+    syncInterval?: number;
+    maxSyncBatchSize?: number;
+    compressionEnabled?: boolean;
+}
+export interface CacheEntry {
+    key: string;
+    value: any;
+    version: number;
+    timestamp: number;
+    ttl?: number;
+    nodeId: string;
+}
+export interface SyncMessage {
+    type: 'invalidate' | 'update' | 'delete' | 'batch';
+    entries: CacheEntry[];
+    source: string;
+    timestamp: number;
+}
+/**
+ * Distributed Cache Synchronizer
+ */
+export declare class CacheSync extends EventEmitter {
+    private nodeId;
+    private localCache;
+    private versionVector;
+    private syncQueue;
+    private syncInterval;
+    private maxSyncBatchSize;
+    private syncTimer?;
+    private isRunning;
+    constructor(config: CacheSyncConfig);
+    /**
+     * Start cache synchronization
+     */
+    start(): void;
+    /**
+     * Stop cache synchronization
+     */
+    stop(): void;
+    /**
+     * Get a value from cache
+     */
+    get(key: string): any | undefined;
+    /**
+     * Set a value in cache and propagate
+     */
+    set(key: string, value: any, ttl?: number): void;
+    /**
+     * Delete a value from cache and propagate
+     */
+    delete(key: string): boolean;
+    /**
+     * Invalidate a cache entry across all nodes
+     */
+    invalidate(key: string): void;
+    /**
+     * Clear all cache entries
+     */
+    clear(): void;
+    /**
+     * Handle incoming sync message from another node
+     */
+    handleSyncMessage(message: SyncMessage): void;
+    /**
+     * Handle a remote cache entry
+     */
+    private handleRemoteEntry;
+    /**
+     * Queue a sync message
+     */
+    private queueSync;
+    /**
+     * Start sync timer
+     */
+    private startSyncTimer;
+    /**
+     * Perform sync operation
+     */
+    private performSync;
+    /**
+     * Increment version for a key
+     */
+    private incrementVersion;
+    /**
+     * Get cache statistics
+     */
+    getStats(): {
+        entries: number;
+        pendingSync: number;
+        versionedKeys: number;
+        memoryUsage: number;
+    };
+    /**
+     * Get cache entries for debugging
+     */
+    getEntries(): CacheEntry[];
+    /**
+     * Merge cache state from another node (for recovery)
+     */
+    mergeState(entries: CacheEntry[]): void;
+}
+/**
+ * Create a cache sync instance
+ */
+export declare function createCacheSync(config: CacheSyncConfig): CacheSync;

.recovery-workspace/dist-backup-20250910-141917/distributed/cacheSync.js

@@ -0,0 +1,265 @@
+/**
+ * Distributed Cache Synchronization
+ * Provides cache coherence across multiple Brainy instances
+ */
+import { EventEmitter } from 'events';
+/**
+ * Distributed Cache Synchronizer
+ */
+export class CacheSync extends EventEmitter {
+    constructor(config) {
+        super();
+        this.localCache = new Map();
+        this.versionVector = new Map();
+        this.syncQueue = [];
+        this.isRunning = false;
+        this.nodeId = config.nodeId;
+        this.syncInterval = config.syncInterval || 1000;
+        this.maxSyncBatchSize = config.maxSyncBatchSize || 100;
+    }
+    /**
+     * Start cache synchronization
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        this.startSyncTimer();
+        this.emit('started', { nodeId: this.nodeId });
+    }
+    /**
+     * Stop cache synchronization
+     */
+    stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.syncTimer) {
+            clearInterval(this.syncTimer);
+            this.syncTimer = undefined;
+        }
+        this.emit('stopped', { nodeId: this.nodeId });
+    }
+    /**
+     * Get a value from cache
+     */
+    get(key) {
+        const entry = this.localCache.get(key);
+        if (!entry)
+            return undefined;
+        // Check TTL
+        if (entry.ttl && Date.now() - entry.timestamp > entry.ttl) {
+            this.localCache.delete(key);
+            return undefined;
+        }
+        return entry.value;
+    }
+    /**
+     * Set a value in cache and propagate
+     */
+    set(key, value, ttl) {
+        const version = this.incrementVersion(key);
+        const entry = {
+            key,
+            value,
+            version,
+            timestamp: Date.now(),
+            ttl,
+            nodeId: this.nodeId
+        };
+        this.localCache.set(key, entry);
+        // Queue for sync
+        this.queueSync('update', [entry]);
+    }
+    /**
+     * Delete a value from cache and propagate
+     */
+    delete(key) {
+        const existed = this.localCache.has(key);
+        if (existed) {
+            const version = this.incrementVersion(key);
+            this.localCache.delete(key);
+            // Queue deletion for sync
+            this.queueSync('delete', [{
+                    key,
+                    value: null,
+                    version,
+                    timestamp: Date.now(),
+                    nodeId: this.nodeId
+                }]);
+        }
+        return existed;
+    }
+    /**
+     * Invalidate a cache entry across all nodes
+     */
+    invalidate(key) {
+        const version = this.incrementVersion(key);
+        this.localCache.delete(key);
+        // Queue invalidation
+        this.queueSync('invalidate', [{
+                key,
+                value: null,
+                version,
+                timestamp: Date.now(),
+                nodeId: this.nodeId
+            }]);
+    }
+    /**
+     * Clear all cache entries
+     */
+    clear() {
+        const entries = [];
+        for (const key of this.localCache.keys()) {
+            const version = this.incrementVersion(key);
+            entries.push({
+                key,
+                value: null,
+                version,
+                timestamp: Date.now(),
+                nodeId: this.nodeId
+            });
+        }
+        this.localCache.clear();
+        if (entries.length > 0) {
+            this.queueSync('delete', entries);
+        }
+    }
+    /**
+     * Handle incoming sync message from another node
+     */
+    handleSyncMessage(message) {
+        if (message.source === this.nodeId)
+            return; // Ignore own messages
+        for (const entry of message.entries) {
+            this.handleRemoteEntry(message.type, entry);
+        }
+        this.emit('synced', {
+            type: message.type,
+            entries: message.entries.length,
+            source: message.source
+        });
+    }
+    /**
+     * Handle a remote cache entry
+     */
+    handleRemoteEntry(type, entry) {
+        const localEntry = this.localCache.get(entry.key);
+        const localVersion = this.versionVector.get(entry.key) || 0;
+        // Version vector check - only accept if remote version is newer
+        if (entry.version <= localVersion) {
+            return; // Our version is newer or same, ignore
+        }
+        // Update version vector
+        this.versionVector.set(entry.key, entry.version);
+        switch (type) {
+            case 'update':
+            case 'batch':
+                // Update local cache with remote value
+                this.localCache.set(entry.key, entry);
+                break;
+            case 'delete':
+            case 'invalidate':
+                // Remove from local cache
+                this.localCache.delete(entry.key);
+                break;
+        }
+    }
+    /**
+     * Queue a sync message
+     */
+    queueSync(type, entries) {
+        const message = {
+            type,
+            entries,
+            source: this.nodeId,
+            timestamp: Date.now()
+        };
+        this.syncQueue.push(message);
+        // If queue is getting large, sync immediately
+        if (this.syncQueue.length >= this.maxSyncBatchSize) {
+            this.performSync();
+        }
+    }
+    /**
+     * Start sync timer
+     */
+    startSyncTimer() {
+        this.syncTimer = setInterval(() => {
+            this.performSync();
+        }, this.syncInterval);
+    }
+    /**
+     * Perform sync operation
+     */
+    performSync() {
+        if (this.syncQueue.length === 0)
+            return;
+        // Batch multiple messages if possible
+        const messages = this.syncQueue.splice(0, this.maxSyncBatchSize);
+        if (messages.length === 1) {
+            // Single message
+            this.emit('sync', messages[0]);
+        }
+        else {
+            // Batch multiple messages
+            const batchedEntries = [];
+            for (const msg of messages) {
+                batchedEntries.push(...msg.entries);
+            }
+            const batchMessage = {
+                type: 'batch',
+                entries: batchedEntries,
+                source: this.nodeId,
+                timestamp: Date.now()
+            };
+            this.emit('sync', batchMessage);
+        }
+    }
+    /**
+     * Increment version for a key
+     */
+    incrementVersion(key) {
+        const current = this.versionVector.get(key) || 0;
+        const next = current + 1;
+        this.versionVector.set(key, next);
+        return next;
+    }
+    /**
+     * Get cache statistics
+     */
+    getStats() {
+        // Estimate memory usage (rough approximation)
+        let memoryUsage = 0;
+        for (const entry of this.localCache.values()) {
+            memoryUsage += JSON.stringify(entry).length;
+        }
+        return {
+            entries: this.localCache.size,
+            pendingSync: this.syncQueue.length,
+            versionedKeys: this.versionVector.size,
+            memoryUsage
+        };
+    }
+    /**
+     * Get cache entries for debugging
+     */
+    getEntries() {
+        return Array.from(this.localCache.values());
+    }
+    /**
+     * Merge cache state from another node (for recovery)
+     */
+    mergeState(entries) {
+        for (const entry of entries) {
+            this.handleRemoteEntry('update', entry);
+        }
+    }
+}
+/**
+ * Create a cache sync instance
+ */
+export function createCacheSync(config) {
+    return new CacheSync(config);
+}
+//# sourceMappingURL=cacheSync.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/configManager.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Distributed Configuration Manager
+ * Manages shared configuration in S3 for distributed Brainy instances
+ */
+import { DistributedConfig, SharedConfig, InstanceInfo, InstanceRole } from '../types/distributedTypes.js';
+import { StorageAdapter } from '../coreTypes.js';
+export declare class DistributedConfigManager {
+    private config;
+    private instanceId;
+    private role;
+    private configPath;
+    private heartbeatInterval;
+    private configCheckInterval;
+    private instanceTimeout;
+    private storage;
+    private heartbeatTimer?;
+    private configWatchTimer?;
+    private lastConfigVersion;
+    private onConfigUpdate?;
+    private hasMigrated;
+    constructor(storage: StorageAdapter, distributedConfig?: DistributedConfig, brainyMode?: {
+        readOnly?: boolean;
+        writeOnly?: boolean;
+    });
+    /**
+     * Initialize the distributed configuration
+     */
+    initialize(): Promise<SharedConfig>;
+    /**
+     * Load existing config or create new one
+     */
+    private loadOrCreateConfig;
+    /**
+     * Determine role based on configuration
+     * IMPORTANT: Role must be explicitly set - no automatic assignment based on order
+     */
+    private determineRole;
+    /**
+     * Check if an instance is still alive
+     */
+    private isInstanceAlive;
+    /**
+     * Register this instance in the shared config
+     */
+    private registerInstance;
+    /**
+     * Migrate config from legacy location to new location
+     */
+    private migrateConfigFromLegacyLocation;
+    /**
+     * Migrate config to new location in index folder
+     */
+    private migrateConfig;
+    /**
+     * Save configuration with version increment
+     */
+    private saveConfig;
+    /**
+     * Start heartbeat to keep instance alive in config
+     */
+    private startHeartbeat;
+    /**
+     * Update heartbeat and clean stale instances
+     */
+    private updateHeartbeat;
+    /**
+     * Start watching for config changes
+     */
+    private startConfigWatch;
+    /**
+     * Check for configuration updates
+     */
+    private checkForConfigUpdates;
+    /**
+     * Load configuration from storage
+     */
+    private loadConfig;
+    /**
+     * Get current configuration
+     */
+    getConfig(): SharedConfig | null;
+    /**
+     * Get instance role
+     */
+    getRole(): InstanceRole;
+    /**
+     * Get instance ID
+     */
+    getInstanceId(): string;
+    /**
+     * Set config update callback
+     */
+    setOnConfigUpdate(callback: (config: SharedConfig) => void): void;
+    /**
+     * Get all active instances of a specific role
+     */
+    getInstancesByRole(role: InstanceRole): InstanceInfo[];
+    /**
+     * Update instance metrics
+     */
+    updateMetrics(metrics: Partial<InstanceInfo['metrics']>): Promise<void>;
+    /**
+     * Cleanup resources
+     */
+    cleanup(): Promise<void>;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/configManager.js

@@ -0,0 +1,441 @@
+/**
+ * Distributed Configuration Manager
+ * Manages shared configuration in S3 for distributed Brainy instances
+ */
+import { v4 as uuidv4 } from '../universal/uuid.js';
+// Constants for config storage locations
+const DISTRIBUTED_CONFIG_KEY = 'distributed_config';
+const LEGACY_CONFIG_KEY = '_distributed_config';
+export class DistributedConfigManager {
+    constructor(storage, distributedConfig, brainyMode) {
+        this.config = null;
+        this.lastConfigVersion = 0;
+        this.hasMigrated = false;
+        this.storage = storage;
+        this.instanceId = distributedConfig?.instanceId || `instance-${uuidv4()}`;
+        // Updated default path to use _system instead of _brainy
+        this.configPath = distributedConfig?.configPath || '_system/distributed_config.json';
+        this.heartbeatInterval = distributedConfig?.heartbeatInterval || 30000;
+        this.configCheckInterval = distributedConfig?.configCheckInterval || 10000;
+        this.instanceTimeout = distributedConfig?.instanceTimeout || 60000;
+        // Set role from distributed config if provided
+        if (distributedConfig?.role) {
+            this.role = distributedConfig.role;
+        }
+        // Infer role from Brainy's read/write mode if not explicitly set
+        else if (brainyMode) {
+            if (brainyMode.writeOnly) {
+                this.role = 'writer';
+            }
+            else if (brainyMode.readOnly) {
+                this.role = 'reader';
+            }
+            // If neither readOnly nor writeOnly, role must be explicitly set
+        }
+    }
+    /**
+     * Initialize the distributed configuration
+     */
+    async initialize() {
+        // Load or create configuration
+        this.config = await this.loadOrCreateConfig();
+        // Determine role if not explicitly set
+        if (!this.role) {
+            this.role = await this.determineRole();
+        }
+        // Register this instance
+        await this.registerInstance();
+        // Start heartbeat and config watching
+        this.startHeartbeat();
+        this.startConfigWatch();
+        return this.config;
+    }
+    /**
+     * Load existing config or create new one
+     */
+    async loadOrCreateConfig() {
+        // First, try to load from the new location in index folder
+        try {
+            const configData = await this.storage.getStatistics();
+            if (configData && configData.distributedConfig) {
+                this.lastConfigVersion = configData.distributedConfig.version;
+                return configData.distributedConfig;
+            }
+        }
+        catch (error) {
+            // Config doesn't exist in new location yet
+        }
+        // Check if we need to migrate from old location
+        if (!this.hasMigrated) {
+            const migrated = await this.migrateConfigFromLegacyLocation();
+            if (migrated) {
+                return migrated;
+            }
+        }
+        // Legacy fallback - try old location
+        try {
+            const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
+            if (configData) {
+                // Migrate to new location
+                await this.migrateConfig(configData);
+                this.lastConfigVersion = configData.version;
+                return configData;
+            }
+        }
+        catch (error) {
+            // Config doesn't exist yet
+        }
+        // Create default config
+        const newConfig = {
+            version: 1,
+            updated: new Date().toISOString(),
+            settings: {
+                partitionStrategy: 'hash',
+                partitionCount: 100,
+                embeddingModel: 'text-embedding-ada-002',
+                dimensions: 1536,
+                distanceMetric: 'cosine',
+                hnswParams: {
+                    M: 16,
+                    efConstruction: 200
+                }
+            },
+            instances: {}
+        };
+        await this.saveConfig(newConfig);
+        return newConfig;
+    }
+    /**
+     * Determine role based on configuration
+     * IMPORTANT: Role must be explicitly set - no automatic assignment based on order
+     */
+    async determineRole() {
+        // Check environment variable first
+        if (process.env.BRAINY_ROLE) {
+            const role = process.env.BRAINY_ROLE.toLowerCase();
+            if (role === 'writer' || role === 'reader' || role === 'hybrid') {
+                return role;
+            }
+            throw new Error(`Invalid BRAINY_ROLE: ${process.env.BRAINY_ROLE}. Must be 'writer', 'reader', or 'hybrid'`);
+        }
+        // Check if explicitly passed in distributed config
+        if (this.role) {
+            return this.role;
+        }
+        // DO NOT auto-assign roles based on deployment order or existing instances
+        // This is dangerous and can lead to data corruption or loss
+        throw new Error('Distributed mode requires explicit role configuration. ' +
+            'Set BRAINY_ROLE environment variable or pass role in distributed config. ' +
+            'Valid roles: "writer", "reader", "hybrid"');
+    }
+    /**
+     * Check if an instance is still alive
+     */
+    isInstanceAlive(instance) {
+        const lastSeen = new Date(instance.lastHeartbeat).getTime();
+        const now = Date.now();
+        return (now - lastSeen) < this.instanceTimeout;
+    }
+    /**
+     * Register this instance in the shared config
+     */
+    async registerInstance() {
+        if (!this.config)
+            return;
+        // Role must be set by this point
+        if (!this.role) {
+            throw new Error('Cannot register instance without a role');
+        }
+        const instanceInfo = {
+            role: this.role,
+            status: 'active',
+            lastHeartbeat: new Date().toISOString(),
+            metrics: {
+                memoryUsage: process.memoryUsage().heapUsed
+            }
+        };
+        // Add endpoint if available
+        if (process.env.SERVICE_ENDPOINT) {
+            instanceInfo.endpoint = process.env.SERVICE_ENDPOINT;
+        }
+        this.config.instances[this.instanceId] = instanceInfo;
+        await this.saveConfig(this.config);
+    }
+    /**
+     * Migrate config from legacy location to new location
+     */
+    async migrateConfigFromLegacyLocation() {
+        try {
+            // Try to load from old location
+            const legacyConfig = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
+            if (legacyConfig) {
+                console.log('Migrating distributed config from legacy location to index folder...');
+                // Save to new location
+                await this.migrateConfig(legacyConfig);
+                // Delete from old location (optional - we can keep it for rollback)
+                // await this.storage.deleteMetadata(LEGACY_CONFIG_KEY)
+                this.hasMigrated = true;
+                this.lastConfigVersion = legacyConfig.version;
+                return legacyConfig;
+            }
+        }
+        catch (error) {
+            console.error('Error during config migration:', error);
+        }
+        this.hasMigrated = true;
+        return null;
+    }
+    /**
+     * Migrate config to new location in index folder
+     */
+    async migrateConfig(config) {
+        // Get existing statistics or create new
+        let stats = await this.storage.getStatistics();
+        if (!stats) {
+            stats = {
+                nounCount: {},
+                verbCount: {},
+                metadataCount: {},
+                hnswIndexSize: 0,
+                lastUpdated: new Date().toISOString()
+            };
+        }
+        // Add distributed config to statistics
+        stats.distributedConfig = config;
+        // Save updated statistics
+        await this.storage.saveStatistics(stats);
+    }
+    /**
+     * Save configuration with version increment
+     */
+    async saveConfig(config) {
+        config.version++;
+        config.updated = new Date().toISOString();
+        this.lastConfigVersion = config.version;
+        // Save to new location in index folder along with statistics
+        let stats = await this.storage.getStatistics();
+        if (!stats) {
+            stats = {
+                nounCount: {},
+                verbCount: {},
+                metadataCount: {},
+                hnswIndexSize: 0,
+                lastUpdated: new Date().toISOString()
+            };
+        }
+        // Update distributed config in statistics
+        stats.distributedConfig = config;
+        // Save updated statistics
+        await this.storage.saveStatistics(stats);
+        this.config = config;
+    }
+    /**
+     * Start heartbeat to keep instance alive in config
+     */
+    startHeartbeat() {
+        this.heartbeatTimer = setInterval(async () => {
+            await this.updateHeartbeat();
+        }, this.heartbeatInterval);
+    }
+    /**
+     * Update heartbeat and clean stale instances
+     */
+    async updateHeartbeat() {
+        if (!this.config)
+            return;
+        // Reload config to get latest state
+        try {
+            const latestConfig = await this.loadConfig();
+            if (latestConfig) {
+                this.config = latestConfig;
+            }
+        }
+        catch (error) {
+            console.error('Failed to reload config:', error);
+        }
+        // Update our heartbeat
+        if (this.config.instances[this.instanceId]) {
+            this.config.instances[this.instanceId].lastHeartbeat = new Date().toISOString();
+            this.config.instances[this.instanceId].status = 'active';
+            // Update metrics if available
+            this.config.instances[this.instanceId].metrics = {
+                memoryUsage: process.memoryUsage().heapUsed
+            };
+        }
+        else {
+            // Re-register if we were removed
+            await this.registerInstance();
+            return;
+        }
+        // Clean up stale instances
+        const now = Date.now();
+        let hasChanges = false;
+        for (const [id, instance] of Object.entries(this.config.instances)) {
+            if (id === this.instanceId)
+                continue;
+            const lastSeen = new Date(instance.lastHeartbeat).getTime();
+            if (now - lastSeen > this.instanceTimeout) {
+                delete this.config.instances[id];
+                hasChanges = true;
+            }
+        }
+        // Save if there were changes
+        if (hasChanges) {
+            await this.saveConfig(this.config);
+        }
+        else {
+            // Just update our heartbeat without version increment
+            // Get existing statistics
+            let stats = await this.storage.getStatistics();
+            if (!stats) {
+                stats = {
+                    nounCount: {},
+                    verbCount: {},
+                    metadataCount: {},
+                    hnswIndexSize: 0,
+                    lastUpdated: new Date().toISOString()
+                };
+            }
+            // Update distributed config in statistics without version increment
+            stats.distributedConfig = this.config;
+            // Save updated statistics
+            await this.storage.saveStatistics(stats);
+        }
+    }
+    /**
+     * Start watching for config changes
+     */
+    startConfigWatch() {
+        this.configWatchTimer = setInterval(async () => {
+            await this.checkForConfigUpdates();
+        }, this.configCheckInterval);
+    }
+    /**
+     * Check for configuration updates
+     */
+    async checkForConfigUpdates() {
+        try {
+            const latestConfig = await this.loadConfig();
+            if (!latestConfig)
+                return;
+            if (latestConfig.version > this.lastConfigVersion) {
+                this.config = latestConfig;
+                this.lastConfigVersion = latestConfig.version;
+                // Notify listeners of config update
+                if (this.onConfigUpdate) {
+                    this.onConfigUpdate(latestConfig);
+                }
+            }
+        }
+        catch (error) {
+            console.error('Failed to check config updates:', error);
+        }
+    }
+    /**
+     * Load configuration from storage
+     */
+    async loadConfig() {
+        try {
+            // Try new location first
+            const stats = await this.storage.getStatistics();
+            if (stats && stats.distributedConfig) {
+                return stats.distributedConfig;
+            }
+            // Fallback to legacy location if not migrated yet
+            if (!this.hasMigrated) {
+                const configData = await this.storage.getMetadata(LEGACY_CONFIG_KEY);
+                if (configData) {
+                    // Trigger migration on next save
+                    return configData;
+                }
+            }
+        }
+        catch (error) {
+            console.error('Failed to load config:', error);
+        }
+        return null;
+    }
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+        return this.config;
+    }
+    /**
+     * Get instance role
+     */
+    getRole() {
+        if (!this.role) {
+            throw new Error('Role not initialized');
+        }
+        return this.role;
+    }
+    /**
+     * Get instance ID
+     */
+    getInstanceId() {
+        return this.instanceId;
+    }
+    /**
+     * Set config update callback
+     */
+    setOnConfigUpdate(callback) {
+        this.onConfigUpdate = callback;
+    }
+    /**
+     * Get all active instances of a specific role
+     */
+    getInstancesByRole(role) {
+        if (!this.config)
+            return [];
+        return Object.entries(this.config.instances)
+            .filter(([_, instance]) => instance.role === role &&
+            this.isInstanceAlive(instance))
+            .map(([_, instance]) => instance);
+    }
+    /**
+     * Update instance metrics
+     */
+    async updateMetrics(metrics) {
+        if (!this.config || !this.config.instances[this.instanceId])
+            return;
+        this.config.instances[this.instanceId].metrics = {
+            ...this.config.instances[this.instanceId].metrics,
+            ...metrics
+        };
+        // Don't increment version for metric updates
+        // Get existing statistics
+        let stats = await this.storage.getStatistics();
+        if (!stats) {
+            stats = {
+                nounCount: {},
+                verbCount: {},
+                metadataCount: {},
+                hnswIndexSize: 0,
+                lastUpdated: new Date().toISOString()
+            };
+        }
+        // Update distributed config in statistics without version increment
+        stats.distributedConfig = this.config;
+        // Save updated statistics
+        await this.storage.saveStatistics(stats);
+    }
+    /**
+     * Cleanup resources
+     */
+    async cleanup() {
+        // Stop timers
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+        }
+        if (this.configWatchTimer) {
+            clearInterval(this.configWatchTimer);
+        }
+        // Mark instance as inactive
+        if (this.config && this.config.instances[this.instanceId]) {
+            this.config.instances[this.instanceId].status = 'inactive';
+            await this.saveConfig(this.config);
+        }
+    }
+}
+//# sourceMappingURL=configManager.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/coordinator.d.ts

@@ -0,0 +1,193 @@
+/**
+ * Distributed Coordinator for Brainy 3.0
+ * Provides leader election, consensus, and coordination for distributed instances
+ */
+import { EventEmitter } from 'events';
+import { NetworkTransport } from './networkTransport.js';
+export interface NodeInfo {
+    id: string;
+    address: string;
+    port: number;
+    lastSeen: number;
+    status: 'active' | 'inactive' | 'suspected';
+    state?: 'follower' | 'candidate' | 'leader';
+    lastHeartbeat?: number;
+}
+export interface CoordinatorConfig {
+    nodeId?: string;
+    address?: string;
+    port?: number;
+    heartbeatInterval?: number;
+    electionTimeout?: number;
+    nodes?: string[];
+}
+export interface ConsensusState {
+    term: number;
+    votedFor: string | null;
+    leader: string | null;
+    state: 'follower' | 'candidate' | 'leader';
+}
+export interface RaftMessage {
+    type: 'requestVote' | 'voteResponse' | 'appendEntries' | 'appendResponse';
+    term: number;
+    from: string;
+    to?: string;
+    data?: any;
+}
+/**
+ * Distributed Coordinator implementing Raft-like consensus
+ */
+export declare class DistributedCoordinator extends EventEmitter {
+    private nodeId;
+    private nodes;
+    private consensusState;
+    private heartbeatInterval;
+    private electionTimeout;
+    private electionTimer?;
+    private heartbeatTimer?;
+    private isRunning;
+    private networkTransport?;
+    private votesReceived;
+    private currentTerm;
+    private lastLogIndex;
+    private lastLogTerm;
+    private logEntries;
+    private transport;
+    private pendingMigrations;
+    private committedMigrations;
+    constructor(config?: CoordinatorConfig);
+    /**
+     * Start the coordinator
+     */
+    start(networkTransport?: NetworkTransport): Promise<void>;
+    /**
+     * Setup network message handlers
+     */
+    private setupNetworkHandlers;
+    /**
+     * Stop the coordinator
+     */
+    stop(): Promise<void>;
+    /**
+     * Register additional nodes
+     */
+    registerNodes(nodes: string[]): void;
+    /**
+     * Become a follower
+     */
+    private becomeFollower;
+    /**
+     * Become a candidate and start election
+     */
+    private becomeCandidate;
+    /**
+     * Become the leader
+     */
+    private becomeLeader;
+    /**
+     * Request votes from all nodes
+     */
+    private requestVotes;
+    /**
+     * Handle vote request from another node
+     */
+    private handleVoteRequest;
+    /**
+     * Handle vote response
+     */
+    private handleVoteResponse;
+    /**
+     * Check if we have majority votes
+     */
+    private checkVoteMajority;
+    /**
+     * Handle append entries (heartbeat) from leader
+     */
+    private handleAppendEntries;
+    /**
+     * Handle append response from follower
+     */
+    private handleAppendResponse;
+    /**
+     * Send heartbeat to followers
+     */
+    private sendHeartbeat;
+    /**
+     * Start heartbeat timer as leader
+     */
+    private startHeartbeat;
+    /**
+     * Reset election timeout
+     */
+    private resetElectionTimeout;
+    /**
+     * Check if log is up to date
+     */
+    private isLogUpToDate;
+    /**
+     * Check log consistency
+     */
+    private checkLogConsistency;
+    /**
+     * Append log entries
+     */
+    private appendLogEntries;
+    /**
+     * Get last log index
+     */
+    private getLastLogIndex;
+    /**
+     * Get last log term
+     */
+    private getLastLogTerm;
+    /**
+     * Generate a unique node ID
+     */
+    private generateNodeId;
+    /**
+     * Get current leader
+     */
+    getLeader(): string | null;
+    /**
+     * Propose a shard migration to the cluster
+     */
+    proposeMigration(migration: {
+        shardId: string;
+        fromNode: string;
+        toNode: string;
+        migrationId: string;
+    }): Promise<void>;
+    /**
+     * Get migration status
+     */
+    getMigrationStatus(migrationId: string): Promise<'pending' | 'committed' | 'rejected'>;
+    /**
+     * Check if this node is the leader
+     */
+    isLeader(): boolean;
+    /**
+     * Get all nodes in the cluster
+     */
+    getNodes(): NodeInfo[];
+    /**
+     * Get cluster health status
+     */
+    getHealth(): {
+        healthy: boolean;
+        leader: string | null;
+        nodes: number;
+        activeNodes: number;
+    };
+    /**
+     * Propose a command to the cluster
+     */
+    proposeCommand(command: any): Promise<void>;
+    /**
+     * Get current state
+     */
+    getState(): ConsensusState;
+}
+/**
+ * Create a coordinator instance
+ */
+export declare function createCoordinator(config?: CoordinatorConfig): DistributedCoordinator;

.recovery-workspace/dist-backup-20250910-141917/distributed/coordinator.js

@@ -0,0 +1,548 @@
+/**
+ * Distributed Coordinator for Brainy 3.0
+ * Provides leader election, consensus, and coordination for distributed instances
+ */
+import { EventEmitter } from 'events';
+import { createHash } from 'crypto';
+/**
+ * Distributed Coordinator implementing Raft-like consensus
+ */
+export class DistributedCoordinator extends EventEmitter {
+    constructor(config = {}) {
+        super();
+        this.nodes = new Map();
+        this.isRunning = false;
+        this.votesReceived = new Set();
+        this.currentTerm = 0;
+        this.lastLogIndex = -1;
+        this.lastLogTerm = 0;
+        this.logEntries = [];
+        this.transport = null; // For migration proposals
+        this.pendingMigrations = new Map();
+        this.committedMigrations = new Set();
+        // Generate node ID if not provided
+        this.nodeId = config.nodeId || this.generateNodeId();
+        // Configuration
+        this.heartbeatInterval = config.heartbeatInterval || 1000;
+        this.electionTimeout = config.electionTimeout || 5000;
+        // Initialize consensus state
+        this.consensusState = {
+            term: 0,
+            votedFor: null,
+            leader: null,
+            state: 'follower'
+        };
+        // Register this node
+        this.nodes.set(this.nodeId, {
+            id: this.nodeId,
+            address: config.address || 'localhost',
+            port: config.port || 3000,
+            lastSeen: Date.now(),
+            status: 'active',
+            state: 'follower',
+            lastHeartbeat: Date.now()
+        });
+        // Register other nodes if provided
+        if (config.nodes) {
+            this.registerNodes(config.nodes);
+        }
+    }
+    /**
+     * Start the coordinator
+     */
+    async start(networkTransport) {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        this.networkTransport = networkTransport;
+        // Setup network message handlers if transport is provided
+        if (this.networkTransport) {
+            this.setupNetworkHandlers();
+        }
+        this.emit('started', { nodeId: this.nodeId });
+        // Start as follower
+        this.becomeFollower();
+    }
+    /**
+     * Setup network message handlers
+     */
+    setupNetworkHandlers() {
+        if (!this.networkTransport)
+            return;
+        // Register message handlers directly on the messageHandlers map
+        const handlers = this.networkTransport.messageHandlers;
+        // Handle vote requests
+        handlers.set('requestVote', async (msg) => {
+            const response = await this.handleVoteRequest(msg);
+            // Send response back
+            if (this.networkTransport) {
+                await this.networkTransport.sendToNode(msg.from, 'voteResponse', response);
+            }
+            return response;
+        });
+        // Handle vote responses
+        handlers.set('voteResponse', async (msg) => {
+            this.handleVoteResponse(msg);
+        });
+        // Handle heartbeats/append entries
+        handlers.set('appendEntries', async (msg) => {
+            const response = await this.handleAppendEntries(msg);
+            // Send response back
+            if (this.networkTransport) {
+                await this.networkTransport.sendToNode(msg.from, 'appendResponse', response);
+            }
+            return response;
+        });
+        // Handle append responses
+        handlers.set('appendResponse', async (msg) => {
+            this.handleAppendResponse(msg);
+        });
+    }
+    /**
+     * Stop the coordinator
+     */
+    async stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.electionTimer) {
+            clearTimeout(this.electionTimer);
+            this.electionTimer = undefined;
+        }
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = undefined;
+        }
+        this.emit('stopped');
+    }
+    /**
+     * Register additional nodes
+     */
+    registerNodes(nodes) {
+        for (const node of nodes) {
+            const [address, port] = node.split(':');
+            const nodeId = this.generateNodeId(node);
+            if (!this.nodes.has(nodeId)) {
+                this.nodes.set(nodeId, {
+                    id: nodeId,
+                    address,
+                    port: parseInt(port || '3000'),
+                    lastSeen: Date.now(),
+                    status: 'active',
+                    state: 'follower'
+                });
+            }
+        }
+    }
+    /**
+     * Become a follower
+     */
+    becomeFollower() {
+        this.consensusState.state = 'follower';
+        const node = this.nodes.get(this.nodeId);
+        if (node) {
+            node.state = 'follower';
+        }
+        this.resetElectionTimeout();
+        this.emit('stateChange', 'follower');
+    }
+    /**
+     * Become a candidate and start election
+     */
+    async becomeCandidate() {
+        this.consensusState.state = 'candidate';
+        this.currentTerm++;
+        this.consensusState.term = this.currentTerm;
+        this.consensusState.votedFor = this.nodeId;
+        this.votesReceived = new Set([this.nodeId]);
+        const node = this.nodes.get(this.nodeId);
+        if (node) {
+            node.state = 'candidate';
+        }
+        this.emit('stateChange', 'candidate');
+        // Request votes from all other nodes
+        await this.requestVotes();
+        // Reset election timeout
+        this.resetElectionTimeout();
+    }
+    /**
+     * Become the leader
+     */
+    becomeLeader() {
+        this.consensusState.state = 'leader';
+        this.consensusState.leader = this.nodeId;
+        const node = this.nodes.get(this.nodeId);
+        if (node) {
+            node.state = 'leader';
+        }
+        // Stop election timer as leader
+        if (this.electionTimer) {
+            clearTimeout(this.electionTimer);
+            this.electionTimer = undefined;
+        }
+        this.emit('stateChange', 'leader');
+        this.emit('leaderElected', this.nodeId);
+        // Start sending heartbeats
+        this.startHeartbeat();
+    }
+    /**
+     * Request votes from all nodes
+     */
+    async requestVotes() {
+        if (!this.networkTransport) {
+            // Simulate vote for testing
+            this.checkVoteMajority();
+            return;
+        }
+        const voteRequest = {
+            type: 'requestVote',
+            term: this.currentTerm,
+            candidateId: this.nodeId,
+            lastLogIndex: this.getLastLogIndex(),
+            lastLogTerm: this.getLastLogTerm()
+        };
+        // Send vote requests to all other nodes
+        for (const [nodeId] of this.nodes) {
+            if (nodeId !== this.nodeId) {
+                try {
+                    await this.networkTransport.sendToNode(nodeId, 'requestVote', voteRequest);
+                }
+                catch (err) {
+                    console.error(`Failed to request vote from ${nodeId}:`, err);
+                }
+            }
+        }
+    }
+    /**
+     * Handle vote request from another node
+     */
+    async handleVoteRequest(msg) {
+        const { term, candidateId, lastLogIndex, lastLogTerm } = msg.data;
+        // Update term if necessary
+        if (term > this.currentTerm) {
+            this.currentTerm = term;
+            this.consensusState.term = term;
+            this.consensusState.votedFor = null;
+            this.becomeFollower();
+        }
+        // Grant vote if conditions are met
+        let voteGranted = false;
+        if (term >= this.currentTerm &&
+            (!this.consensusState.votedFor || this.consensusState.votedFor === candidateId) &&
+            this.isLogUpToDate(lastLogIndex, lastLogTerm)) {
+            this.consensusState.votedFor = candidateId;
+            voteGranted = true;
+            this.resetElectionTimeout();
+        }
+        return {
+            type: 'voteResponse',
+            term: this.currentTerm,
+            voteGranted
+        };
+    }
+    /**
+     * Handle vote response
+     */
+    handleVoteResponse(msg) {
+        const { term, voteGranted } = msg.data;
+        // Ignore old responses
+        if (term < this.currentTerm)
+            return;
+        // Update term if necessary
+        if (term > this.currentTerm) {
+            this.currentTerm = term;
+            this.consensusState.term = term;
+            this.becomeFollower();
+            return;
+        }
+        // Count vote if granted
+        if (voteGranted && this.consensusState.state === 'candidate') {
+            this.votesReceived.add(msg.from);
+            this.checkVoteMajority();
+        }
+    }
+    /**
+     * Check if we have majority votes
+     */
+    checkVoteMajority() {
+        const majority = Math.floor(this.nodes.size / 2) + 1;
+        if (this.votesReceived.size >= majority) {
+            this.becomeLeader();
+        }
+    }
+    /**
+     * Handle append entries (heartbeat) from leader
+     */
+    async handleAppendEntries(msg) {
+        const { term, leaderId, prevLogIndex, prevLogTerm, entries, leaderCommit } = msg.data;
+        // Update term if necessary
+        if (term > this.currentTerm) {
+            this.currentTerm = term;
+            this.consensusState.term = term;
+            this.consensusState.votedFor = null;
+            this.becomeFollower();
+        }
+        // Reset election timeout when receiving valid heartbeat
+        if (term >= this.currentTerm) {
+            this.consensusState.leader = leaderId;
+            this.resetElectionTimeout();
+            // Update leader node's last heartbeat
+            const leaderNode = this.nodes.get(leaderId);
+            if (leaderNode) {
+                leaderNode.lastHeartbeat = Date.now();
+                leaderNode.lastSeen = Date.now();
+            }
+        }
+        // Check log consistency
+        let success = false;
+        if (term >= this.currentTerm) {
+            if (this.checkLogConsistency(prevLogIndex, prevLogTerm)) {
+                // Append new entries if any
+                if (entries && entries.length > 0) {
+                    this.appendLogEntries(prevLogIndex, entries);
+                }
+                success = true;
+            }
+        }
+        return {
+            type: 'appendResponse',
+            term: this.currentTerm,
+            success
+        };
+    }
+    /**
+     * Handle append response from follower
+     */
+    handleAppendResponse(msg) {
+        const { term, success } = msg.data;
+        // Update term if necessary
+        if (term > this.currentTerm) {
+            this.currentTerm = term;
+            this.consensusState.term = term;
+            this.becomeFollower();
+        }
+        // Process successful append
+        if (success && this.consensusState.state === 'leader') {
+            // Update follower's match index
+            // In a real implementation, this would track replication progress
+        }
+    }
+    /**
+     * Send heartbeat to followers
+     */
+    async sendHeartbeat() {
+        if (!this.networkTransport) {
+            // Fallback for testing
+            await new Promise(resolve => setTimeout(resolve, 10));
+            return;
+        }
+        const heartbeat = {
+            type: 'appendEntries',
+            term: this.currentTerm,
+            leaderId: this.nodeId,
+            prevLogIndex: this.getLastLogIndex(),
+            prevLogTerm: this.getLastLogTerm(),
+            entries: [],
+            leaderCommit: 0
+        };
+        // Send heartbeat to all followers
+        for (const [nodeId] of this.nodes) {
+            if (nodeId !== this.nodeId) {
+                try {
+                    await this.networkTransport.sendToNode(nodeId, 'appendEntries', heartbeat);
+                }
+                catch (err) {
+                    // Node might be down, mark as suspected
+                    const node = this.nodes.get(nodeId);
+                    if (node) {
+                        node.status = 'suspected';
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Start heartbeat timer as leader
+     */
+    startHeartbeat() {
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+        }
+        this.heartbeatTimer = setInterval(() => {
+            this.sendHeartbeat();
+        }, this.heartbeatInterval);
+        // Send immediate heartbeat
+        this.sendHeartbeat();
+    }
+    /**
+     * Reset election timeout
+     */
+    resetElectionTimeout() {
+        if (this.electionTimer) {
+            clearTimeout(this.electionTimer);
+        }
+        // Randomize timeout to prevent split votes
+        const timeout = this.electionTimeout + Math.random() * this.electionTimeout;
+        this.electionTimer = setTimeout(() => {
+            if (this.consensusState.state === 'follower') {
+                this.becomeCandidate();
+            }
+        }, timeout);
+    }
+    /**
+     * Check if log is up to date
+     */
+    isLogUpToDate(lastLogIndex, lastLogTerm) {
+        const myLastLogTerm = this.getLastLogTerm();
+        const myLastLogIndex = this.getLastLogIndex();
+        if (lastLogTerm > myLastLogTerm)
+            return true;
+        if (lastLogTerm < myLastLogTerm)
+            return false;
+        return lastLogIndex >= myLastLogIndex;
+    }
+    /**
+     * Check log consistency
+     */
+    checkLogConsistency(prevLogIndex, prevLogTerm) {
+        if (prevLogIndex === -1)
+            return true;
+        if (prevLogIndex >= this.logEntries.length)
+            return false;
+        const entry = this.logEntries[prevLogIndex];
+        return entry && entry.term === prevLogTerm;
+    }
+    /**
+     * Append log entries
+     */
+    appendLogEntries(prevLogIndex, entries) {
+        // Remove conflicting entries
+        this.logEntries = this.logEntries.slice(0, prevLogIndex + 1);
+        // Append new entries
+        for (const entry of entries) {
+            this.logEntries.push({
+                term: entry.term,
+                index: this.logEntries.length,
+                data: entry.data
+            });
+        }
+        this.lastLogIndex = this.logEntries.length - 1;
+        if (this.lastLogIndex >= 0) {
+            this.lastLogTerm = this.logEntries[this.lastLogIndex].term;
+        }
+    }
+    /**
+     * Get last log index
+     */
+    getLastLogIndex() {
+        return this.lastLogIndex;
+    }
+    /**
+     * Get last log term
+     */
+    getLastLogTerm() {
+        return this.lastLogTerm;
+    }
+    /**
+     * Generate a unique node ID
+     */
+    generateNodeId(seed) {
+        const source = seed || `${process.pid}-${Date.now()}-${Math.random()}`;
+        return createHash('sha256').update(source).digest('hex').substring(0, 16);
+    }
+    /**
+     * Get current leader
+     */
+    getLeader() {
+        return this.consensusState.leader;
+    }
+    /**
+     * Propose a shard migration to the cluster
+     */
+    async proposeMigration(migration) {
+        if (!this.isLeader()) {
+            throw new Error('Only leader can propose migrations');
+        }
+        // Broadcast migration proposal to all nodes
+        const message = {
+            type: 'migration-proposal',
+            migration,
+            timestamp: Date.now()
+        };
+        await this.transport.broadcast('migration', message);
+        // Store migration as pending
+        this.pendingMigrations.set(migration.migrationId, {
+            ...migration,
+            status: 'pending'
+        });
+    }
+    /**
+     * Get migration status
+     */
+    async getMigrationStatus(migrationId) {
+        const migration = this.pendingMigrations.get(migrationId);
+        if (!migration) {
+            // Check if it was committed
+            return this.committedMigrations.has(migrationId) ? 'committed' : 'rejected';
+        }
+        return migration.status || 'pending';
+    }
+    /**
+     * Check if this node is the leader
+     */
+    isLeader() {
+        return this.consensusState.state === 'leader';
+    }
+    /**
+     * Get all nodes in the cluster
+     */
+    getNodes() {
+        return Array.from(this.nodes.values());
+    }
+    /**
+     * Get cluster health status
+     */
+    getHealth() {
+        const now = Date.now();
+        const activeNodes = Array.from(this.nodes.values()).filter(node => now - node.lastSeen < this.electionTimeout).length;
+        return {
+            healthy: this.consensusState.leader !== null && activeNodes > this.nodes.size / 2,
+            leader: this.consensusState.leader,
+            nodes: this.nodes.size,
+            activeNodes
+        };
+    }
+    /**
+     * Propose a command to the cluster
+     */
+    async proposeCommand(command) {
+        if (this.consensusState.state !== 'leader') {
+            throw new Error(`Not the leader. Current leader: ${this.consensusState.leader}`);
+        }
+        // Add to log
+        const entry = {
+            term: this.currentTerm,
+            index: this.logEntries.length,
+            data: command
+        };
+        this.logEntries.push(entry);
+        this.lastLogIndex = entry.index;
+        this.lastLogTerm = entry.term;
+        // Replicate to followers
+        await this.sendHeartbeat();
+        this.emit('commandProposed', command);
+    }
+    /**
+     * Get current state
+     */
+    getState() {
+        return { ...this.consensusState };
+    }
+}
+/**
+ * Create a coordinator instance
+ */
+export function createCoordinator(config) {
+    return new DistributedCoordinator(config);
+}
+//# sourceMappingURL=coordinator.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/domainDetector.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Domain Detector
+ * Automatically detects and manages data domains for logical separation
+ */
+import { DomainMetadata } from '../types/distributedTypes.js';
+export interface DomainPattern {
+    domain: string;
+    patterns: {
+        fields?: string[];
+        keywords?: string[];
+        regex?: RegExp;
+    };
+    priority?: number;
+}
+export declare class DomainDetector {
+    private domainPatterns;
+    private customPatterns;
+    private domainStats;
+    /**
+     * Detect domain from data object
+     * @param data - The data object to analyze
+     * @returns The detected domain and metadata
+     */
+    detectDomain(data: any): DomainMetadata;
+    /**
+     * Score a data object against a domain pattern
+     */
+    private scorePattern;
+    /**
+     * Extract domain-specific metadata
+     */
+    private extractDomainMetadata;
+    /**
+     * Calculate detection confidence
+     */
+    private calculateConfidence;
+    /**
+     * Categorize price ranges
+     */
+    private getPriceRange;
+    /**
+     * Categorize customer value
+     */
+    private getValueCategory;
+    /**
+     * Categorize amount ranges
+     */
+    private getAmountRange;
+    /**
+     * Add custom domain pattern
+     * @param pattern - Custom domain pattern to add
+     */
+    addCustomPattern(pattern: DomainPattern): void;
+    /**
+     * Remove custom domain pattern
+     * @param domain - Domain to remove pattern for
+     */
+    removeCustomPattern(domain: string): void;
+    /**
+     * Update domain statistics
+     */
+    private updateStats;
+    /**
+     * Get domain statistics
+     * @returns Map of domain to count
+     */
+    getDomainStats(): Map<string, number>;
+    /**
+     * Clear domain statistics
+     */
+    clearStats(): void;
+    /**
+     * Get all configured domains
+     * @returns Array of domain names
+     */
+    getConfiguredDomains(): string[];
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/domainDetector.js

@@ -0,0 +1,307 @@
+/**
+ * Domain Detector
+ * Automatically detects and manages data domains for logical separation
+ */
+export class DomainDetector {
+    constructor() {
+        this.domainPatterns = [
+            {
+                domain: 'medical',
+                patterns: {
+                    fields: ['symptoms', 'diagnosis', 'treatment', 'medication', 'patient'],
+                    keywords: ['medical', 'health', 'disease', 'symptom', 'treatment', 'doctor', 'patient']
+                },
+                priority: 1
+            },
+            {
+                domain: 'legal',
+                patterns: {
+                    fields: ['contract', 'clause', 'litigation', 'statute', 'jurisdiction'],
+                    keywords: ['legal', 'law', 'contract', 'court', 'attorney', 'litigation', 'statute']
+                },
+                priority: 1
+            },
+            {
+                domain: 'product',
+                patterns: {
+                    fields: ['price', 'sku', 'inventory', 'category', 'brand'],
+                    keywords: ['product', 'price', 'sale', 'inventory', 'catalog', 'item', 'sku']
+                },
+                priority: 1
+            },
+            {
+                domain: 'customer',
+                patterns: {
+                    fields: ['customerId', 'email', 'phone', 'address', 'orders'],
+                    keywords: ['customer', 'client', 'user', 'account', 'profile', 'contact']
+                },
+                priority: 1
+            },
+            {
+                domain: 'financial',
+                patterns: {
+                    fields: ['amount', 'currency', 'transaction', 'balance', 'account'],
+                    keywords: ['financial', 'money', 'payment', 'transaction', 'bank', 'credit', 'debit']
+                },
+                priority: 1
+            },
+            {
+                domain: 'technical',
+                patterns: {
+                    fields: ['code', 'function', 'error', 'stack', 'api'],
+                    keywords: ['code', 'software', 'api', 'error', 'debug', 'function', 'class', 'method']
+                },
+                priority: 2
+            }
+        ];
+        this.customPatterns = [];
+        this.domainStats = new Map();
+    }
+    /**
+     * Detect domain from data object
+     * @param data - The data object to analyze
+     * @returns The detected domain and metadata
+     */
+    detectDomain(data) {
+        if (!data || typeof data !== 'object') {
+            return { domain: 'general' };
+        }
+        // Check for explicit domain field
+        if (data.domain && typeof data.domain === 'string') {
+            this.updateStats(data.domain);
+            return {
+                domain: data.domain,
+                domainMetadata: this.extractDomainMetadata(data, data.domain)
+            };
+        }
+        // Score each domain pattern
+        const scores = new Map();
+        // Check custom patterns first (higher priority)
+        for (const pattern of this.customPatterns) {
+            const score = this.scorePattern(data, pattern);
+            if (score > 0) {
+                scores.set(pattern.domain, score * (pattern.priority || 1));
+            }
+        }
+        // Check default patterns
+        for (const pattern of this.domainPatterns) {
+            const score = this.scorePattern(data, pattern);
+            if (score > 0) {
+                const currentScore = scores.get(pattern.domain) || 0;
+                scores.set(pattern.domain, currentScore + score * (pattern.priority || 1));
+            }
+        }
+        // Find highest scoring domain
+        let bestDomain = 'general';
+        let bestScore = 0;
+        for (const [domain, score] of scores.entries()) {
+            if (score > bestScore) {
+                bestDomain = domain;
+                bestScore = score;
+            }
+        }
+        this.updateStats(bestDomain);
+        return {
+            domain: bestDomain,
+            domainMetadata: this.extractDomainMetadata(data, bestDomain)
+        };
+    }
+    /**
+     * Score a data object against a domain pattern
+     */
+    scorePattern(data, pattern) {
+        let score = 0;
+        // Check field matches
+        if (pattern.patterns.fields) {
+            const dataKeys = Object.keys(data);
+            for (const field of pattern.patterns.fields) {
+                if (dataKeys.some(key => key.toLowerCase().includes(field.toLowerCase()))) {
+                    score += 2; // Field match is strong signal
+                }
+            }
+        }
+        // Check keyword matches in values
+        if (pattern.patterns.keywords) {
+            const dataStr = JSON.stringify(data).toLowerCase();
+            for (const keyword of pattern.patterns.keywords) {
+                if (dataStr.includes(keyword.toLowerCase())) {
+                    score += 1;
+                }
+            }
+        }
+        // Check regex patterns
+        if (pattern.patterns.regex) {
+            const dataStr = JSON.stringify(data);
+            if (pattern.patterns.regex.test(dataStr)) {
+                score += 3; // Regex match is very specific
+            }
+        }
+        return score;
+    }
+    /**
+     * Extract domain-specific metadata
+     */
+    extractDomainMetadata(data, domain) {
+        const metadata = {};
+        switch (domain) {
+            case 'medical':
+                if (data.patientId)
+                    metadata.patientId = data.patientId;
+                if (data.condition)
+                    metadata.condition = data.condition;
+                if (data.severity)
+                    metadata.severity = data.severity;
+                break;
+            case 'legal':
+                if (data.caseId)
+                    metadata.caseId = data.caseId;
+                if (data.jurisdiction)
+                    metadata.jurisdiction = data.jurisdiction;
+                if (data.documentType)
+                    metadata.documentType = data.documentType;
+                break;
+            case 'product':
+                if (data.sku)
+                    metadata.sku = data.sku;
+                if (data.category)
+                    metadata.category = data.category;
+                if (data.brand)
+                    metadata.brand = data.brand;
+                if (data.price)
+                    metadata.priceRange = this.getPriceRange(data.price);
+                break;
+            case 'customer':
+                if (data.customerId)
+                    metadata.customerId = data.customerId;
+                if (data.segment)
+                    metadata.segment = data.segment;
+                if (data.lifetime_value)
+                    metadata.valueCategory = this.getValueCategory(data.lifetime_value);
+                break;
+            case 'financial':
+                if (data.accountId)
+                    metadata.accountId = data.accountId;
+                if (data.transactionType)
+                    metadata.transactionType = data.transactionType;
+                if (data.amount)
+                    metadata.amountRange = this.getAmountRange(data.amount);
+                break;
+            case 'technical':
+                if (data.service)
+                    metadata.service = data.service;
+                if (data.environment)
+                    metadata.environment = data.environment;
+                if (data.severity)
+                    metadata.severity = data.severity;
+                break;
+        }
+        // Add detection confidence
+        metadata.detectionConfidence = this.calculateConfidence(data, domain);
+        return metadata;
+    }
+    /**
+     * Calculate detection confidence
+     */
+    calculateConfidence(data, domain) {
+        // If domain was explicitly specified
+        if (data.domain === domain)
+            return 'high';
+        // Check how many patterns matched
+        const pattern = [...this.customPatterns, ...this.domainPatterns]
+            .find(p => p.domain === domain);
+        if (!pattern)
+            return 'low';
+        const score = this.scorePattern(data, pattern);
+        if (score >= 5)
+            return 'high';
+        if (score >= 2)
+            return 'medium';
+        return 'low';
+    }
+    /**
+     * Categorize price ranges
+     */
+    getPriceRange(price) {
+        if (price < 10)
+            return 'low';
+        if (price < 100)
+            return 'medium';
+        if (price < 1000)
+            return 'high';
+        return 'premium';
+    }
+    /**
+     * Categorize customer value
+     */
+    getValueCategory(value) {
+        if (value < 100)
+            return 'low';
+        if (value < 1000)
+            return 'medium';
+        if (value < 10000)
+            return 'high';
+        return 'vip';
+    }
+    /**
+     * Categorize amount ranges
+     */
+    getAmountRange(amount) {
+        if (amount < 100)
+            return 'micro';
+        if (amount < 1000)
+            return 'small';
+        if (amount < 10000)
+            return 'medium';
+        if (amount < 100000)
+            return 'large';
+        return 'enterprise';
+    }
+    /**
+     * Add custom domain pattern
+     * @param pattern - Custom domain pattern to add
+     */
+    addCustomPattern(pattern) {
+        // Remove existing pattern for same domain if exists
+        this.customPatterns = this.customPatterns.filter(p => p.domain !== pattern.domain);
+        this.customPatterns.push(pattern);
+    }
+    /**
+     * Remove custom domain pattern
+     * @param domain - Domain to remove pattern for
+     */
+    removeCustomPattern(domain) {
+        this.customPatterns = this.customPatterns.filter(p => p.domain !== domain);
+    }
+    /**
+     * Update domain statistics
+     */
+    updateStats(domain) {
+        const count = this.domainStats.get(domain) || 0;
+        this.domainStats.set(domain, count + 1);
+    }
+    /**
+     * Get domain statistics
+     * @returns Map of domain to count
+     */
+    getDomainStats() {
+        return new Map(this.domainStats);
+    }
+    /**
+     * Clear domain statistics
+     */
+    clearStats() {
+        this.domainStats.clear();
+    }
+    /**
+     * Get all configured domains
+     * @returns Array of domain names
+     */
+    getConfiguredDomains() {
+        const domains = new Set();
+        for (const pattern of [...this.domainPatterns, ...this.customPatterns]) {
+            domains.add(pattern.domain);
+        }
+        return Array.from(domains).sort();
+    }
+}
+//# sourceMappingURL=domainDetector.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/hashPartitioner.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Hash-based Partitioner
+ * Provides deterministic partitioning for distributed writes
+ */
+import { SharedConfig } from '../types/distributedTypes.js';
+export declare class HashPartitioner {
+    private partitionCount;
+    private partitionPrefix;
+    constructor(config: SharedConfig);
+    /**
+     * Get partition for a given vector ID using deterministic hashing
+     * @param vectorId - The unique identifier of the vector
+     * @returns The partition path
+     */
+    getPartition(vectorId: string): string;
+    /**
+     * Get partition with domain metadata (domain stored as metadata, not in path)
+     * @param vectorId - The unique identifier of the vector
+     * @param domain - The domain identifier (for metadata only)
+     * @returns The partition path
+     */
+    getPartitionWithDomain(vectorId: string, domain?: string): string;
+    /**
+     * Get all partition paths
+     * @returns Array of all partition paths
+     */
+    getAllPartitions(): string[];
+    /**
+     * Get partition index from partition path
+     * @param partitionPath - The partition path
+     * @returns The partition index
+     */
+    getPartitionIndex(partitionPath: string): number;
+    /**
+     * Hash a string to a number for consistent partitioning
+     * @param str - The string to hash
+     * @returns A positive integer hash
+     */
+    private hashString;
+    /**
+     * Get partitions for batch operations
+     * Groups vector IDs by their target partition
+     * @param vectorIds - Array of vector IDs
+     * @returns Map of partition to vector IDs
+     */
+    getPartitionsForBatch(vectorIds: string[]): Map<string, string[]>;
+}
+/**
+ * Affinity-based Partitioner
+ * Extends HashPartitioner to prefer certain partitions for a writer
+ * while maintaining correctness
+ */
+export declare class AffinityPartitioner extends HashPartitioner {
+    private preferredPartitions;
+    private instanceId;
+    constructor(config: SharedConfig, instanceId: string);
+    /**
+     * Calculate preferred partitions for this instance
+     */
+    private calculatePreferredPartitions;
+    /**
+     * Check if a partition is preferred for this instance
+     * @param partitionPath - The partition path
+     * @returns Whether this partition is preferred
+     */
+    isPreferredPartition(partitionPath: string): boolean;
+    /**
+     * Get all preferred partitions for this instance
+     * @returns Array of preferred partition paths
+     */
+    getPreferredPartitions(): string[];
+    /**
+     * Update preferred partitions based on new config
+     * @param config - The updated shared configuration
+     */
+    updatePreferences(config: SharedConfig): void;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/hashPartitioner.js

@@ -0,0 +1,146 @@
+/**
+ * Hash-based Partitioner
+ * Provides deterministic partitioning for distributed writes
+ */
+import { getPartitionHash } from '../utils/crypto.js';
+export class HashPartitioner {
+    constructor(config) {
+        this.partitionPrefix = 'vectors/p';
+        this.partitionCount = config.settings.partitionCount || 100;
+    }
+    /**
+     * Get partition for a given vector ID using deterministic hashing
+     * @param vectorId - The unique identifier of the vector
+     * @returns The partition path
+     */
+    getPartition(vectorId) {
+        const hash = this.hashString(vectorId);
+        const partitionIndex = hash % this.partitionCount;
+        return `${this.partitionPrefix}${partitionIndex.toString().padStart(3, '0')}`;
+    }
+    /**
+     * Get partition with domain metadata (domain stored as metadata, not in path)
+     * @param vectorId - The unique identifier of the vector
+     * @param domain - The domain identifier (for metadata only)
+     * @returns The partition path
+     */
+    getPartitionWithDomain(vectorId, domain) {
+        // Domain doesn't affect partitioning - it's just metadata
+        return this.getPartition(vectorId);
+    }
+    /**
+     * Get all partition paths
+     * @returns Array of all partition paths
+     */
+    getAllPartitions() {
+        const partitions = [];
+        for (let i = 0; i < this.partitionCount; i++) {
+            partitions.push(`${this.partitionPrefix}${i.toString().padStart(3, '0')}`);
+        }
+        return partitions;
+    }
+    /**
+     * Get partition index from partition path
+     * @param partitionPath - The partition path
+     * @returns The partition index
+     */
+    getPartitionIndex(partitionPath) {
+        const match = partitionPath.match(/p(\d+)$/);
+        if (match) {
+            return parseInt(match[1], 10);
+        }
+        throw new Error(`Invalid partition path: ${partitionPath}`);
+    }
+    /**
+     * Hash a string to a number for consistent partitioning
+     * @param str - The string to hash
+     * @returns A positive integer hash
+     */
+    hashString(str) {
+        // Use our cross-platform hash function
+        return getPartitionHash(str);
+    }
+    /**
+     * Get partitions for batch operations
+     * Groups vector IDs by their target partition
+     * @param vectorIds - Array of vector IDs
+     * @returns Map of partition to vector IDs
+     */
+    getPartitionsForBatch(vectorIds) {
+        const partitionMap = new Map();
+        for (const id of vectorIds) {
+            const partition = this.getPartition(id);
+            if (!partitionMap.has(partition)) {
+                partitionMap.set(partition, []);
+            }
+            partitionMap.get(partition).push(id);
+        }
+        return partitionMap;
+    }
+}
+/**
+ * Affinity-based Partitioner
+ * Extends HashPartitioner to prefer certain partitions for a writer
+ * while maintaining correctness
+ */
+export class AffinityPartitioner extends HashPartitioner {
+    constructor(config, instanceId) {
+        super(config);
+        this.instanceId = instanceId;
+        this.preferredPartitions = this.calculatePreferredPartitions(config);
+    }
+    /**
+     * Calculate preferred partitions for this instance
+     */
+    calculatePreferredPartitions(config) {
+        const partitionCount = config.settings.partitionCount || 100;
+        const writers = Object.entries(config.instances)
+            .filter(([_, inst]) => inst.role === 'writer')
+            .map(([id, _]) => id)
+            .sort(); // Ensure consistent ordering
+        const writerIndex = writers.indexOf(this.instanceId);
+        if (writerIndex === -1) {
+            // Not a writer or not found, no preferences
+            return new Set();
+        }
+        const writerCount = writers.length;
+        const partitionsPerWriter = Math.ceil(partitionCount / writerCount);
+        const preferred = new Set();
+        const start = writerIndex * partitionsPerWriter;
+        const end = Math.min(start + partitionsPerWriter, partitionCount);
+        for (let i = start; i < end; i++) {
+            preferred.add(i);
+        }
+        return preferred;
+    }
+    /**
+     * Check if a partition is preferred for this instance
+     * @param partitionPath - The partition path
+     * @returns Whether this partition is preferred
+     */
+    isPreferredPartition(partitionPath) {
+        try {
+            const index = this.getPartitionIndex(partitionPath);
+            return this.preferredPartitions.has(index);
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get all preferred partitions for this instance
+     * @returns Array of preferred partition paths
+     */
+    getPreferredPartitions() {
+        return Array.from(this.preferredPartitions)
+            .map(index => `vectors/p${index.toString().padStart(3, '0')}`);
+    }
+    /**
+     * Update preferred partitions based on new config
+     * @param config - The updated shared configuration
+     */
+    updatePreferences(config) {
+        this.preferredPartitions = this.calculatePreferredPartitions(config);
+    }
+}
+//# sourceMappingURL=hashPartitioner.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/hashPartitioner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hashPartitioner.js","sourceRoot":"","sources":["../../src/distributed/hashPartitioner.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AAGrD,MAAM,OAAO,eAAe;IAI1B,YAAY,MAAoB;QAFxB,oBAAe,GAAW,WAAW,CAAA;QAG3C,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,cAAc,IAAI,GAAG,CAAA;IAC7D,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,QAAgB;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,cAAc,CAAA;QACjD,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED;;;;;OAKG;IACH,sBAAsB,CAAC,QAAgB,EAAE,MAAe;QACtD,0DAA0D;QAC1D,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IACpC,CAAC;IAED;;;OAGG;IACH,gBAAgB;QACd,MAAM,UAAU,GAAa,EAAE,CAAA;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QACD,OAAO,UAAU,CAAA;IACnB,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,aAAqB;QACrC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAC5C,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC/B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,aAAa,EAAE,CAAC,CAAA;IAC7D,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAW;QAC5B,uCAAuC;QACvC,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAA;IAC9B,CAAC;IAED;;;;;OAKG;IACH,qBAAqB,CAAC,SAAmB;QACvC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEhD,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAA;YACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YACjC,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACvC,CAAC;QAED,OAAO,YAAY,CAAA;IACrB,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,mBAAoB,SAAQ,eAAe;IAItD,YAAY,MAAoB,EAAE,UAAkB;QAClD,KAAK,CAAC,MAAM,CAAC,CAAA;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAA;IACtE,CAAC;IAED;;OAEG;IACK,4BAA4B,CAAC,MAAoB;QACvD,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,cAAc,IAAI,GAAG,CAAA;QAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;aAC7C,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;aAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;aACpB,IAAI,EAAE,CAAA,CAAC,6BAA6B;QAEvC,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACpD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,4CAA4C;YAC5C,OAAO,IAAI,GAAG,EAAE,CAAA;QAClB,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAA;QAClC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,CAAA;QAEnE,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;QACnC,MAAM,KAAK,GAAG,WAAW,GAAG,mBAAmB,CAAA;QAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,mBAAmB,EAAE,cAAc,CAAC,CAAA;QAEjE,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAClB,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,aAAqB;QACxC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YACnD,OAAO,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,sBAAsB;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC;aACxC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;IAClE,CAAC;IAED;;;OAGG;IACH,iBAAiB,CAAC,MAAoB;QACpC,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAA;IACtE,CAAC;CACF"}

.recovery-workspace/dist-backup-20250910-141917/distributed/healthMonitor.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Health Monitor
+ * Monitors and reports instance health in distributed deployments
+ */
+import { DistributedConfigManager } from './configManager.js';
+export interface HealthMetrics {
+    vectorCount: number;
+    cacheHitRate: number;
+    memoryUsage: number;
+    cpuUsage?: number;
+    requestsPerSecond?: number;
+    averageLatency?: number;
+    errorRate?: number;
+}
+export interface HealthStatus {
+    status: 'healthy' | 'degraded' | 'unhealthy';
+    instanceId: string;
+    role: string;
+    uptime: number;
+    lastCheck: string;
+    metrics: HealthMetrics;
+    warnings?: string[];
+    errors?: string[];
+}
+export declare class HealthMonitor {
+    private configManager;
+    private startTime;
+    private requestCount;
+    private errorCount;
+    private totalLatency;
+    private cacheHits;
+    private cacheMisses;
+    private vectorCount;
+    private checkInterval;
+    private healthCheckTimer?;
+    private metricsWindow;
+    private latencyWindow;
+    private windowSize;
+    constructor(configManager: DistributedConfigManager);
+    /**
+     * Start health monitoring
+     */
+    start(): void;
+    /**
+     * Stop health monitoring
+     */
+    stop(): void;
+    /**
+     * Update health status and metrics
+     */
+    private updateHealth;
+    /**
+     * Collect current metrics
+     */
+    private collectMetrics;
+    /**
+     * Calculate cache hit rate
+     */
+    private calculateCacheHitRate;
+    /**
+     * Calculate requests per second
+     */
+    private calculateRPS;
+    /**
+     * Calculate average latency
+     */
+    private calculateAverageLatency;
+    /**
+     * Calculate error rate
+     */
+    private calculateErrorRate;
+    /**
+     * Get CPU usage (simplified)
+     */
+    private getCPUUsage;
+    /**
+     * Clean old entries from sliding windows
+     */
+    private cleanWindows;
+    /**
+     * Record a request
+     * @param latency - Request latency in milliseconds
+     * @param error - Whether the request resulted in an error
+     */
+    recordRequest(latency: number, error?: boolean): void;
+    /**
+     * Record cache access
+     * @param hit - Whether it was a cache hit
+     */
+    recordCacheAccess(hit: boolean): void;
+    /**
+     * Update vector count
+     * @param count - New vector count
+     */
+    updateVectorCount(count: number): void;
+    /**
+     * Get current health status
+     * @returns Health status object
+     */
+    getHealthStatus(): HealthStatus;
+    /**
+     * Get health check endpoint data
+     * @returns JSON-serializable health data
+     */
+    getHealthEndpointData(): Record<string, any>;
+    /**
+     * Reset metrics (useful for testing)
+     */
+    resetMetrics(): void;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/healthMonitor.js

@@ -0,0 +1,244 @@
+/**
+ * Health Monitor
+ * Monitors and reports instance health in distributed deployments
+ */
+export class HealthMonitor {
+    constructor(configManager) {
+        this.requestCount = 0;
+        this.errorCount = 0;
+        this.totalLatency = 0;
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        this.vectorCount = 0;
+        this.checkInterval = 30000; // 30 seconds
+        this.metricsWindow = []; // Sliding window for RPS calculation
+        this.latencyWindow = []; // Sliding window for latency
+        this.windowSize = 60000; // 1 minute window
+        this.configManager = configManager;
+        this.startTime = Date.now();
+    }
+    /**
+     * Start health monitoring
+     */
+    start() {
+        // Initial health update
+        this.updateHealth();
+        // Schedule periodic health checks
+        this.healthCheckTimer = setInterval(() => {
+            this.updateHealth();
+        }, this.checkInterval);
+    }
+    /**
+     * Stop health monitoring
+     */
+    stop() {
+        if (this.healthCheckTimer) {
+            clearInterval(this.healthCheckTimer);
+            this.healthCheckTimer = undefined;
+        }
+    }
+    /**
+     * Update health status and metrics
+     */
+    async updateHealth() {
+        const metrics = this.collectMetrics();
+        // Update config with latest metrics
+        await this.configManager.updateMetrics({
+            vectorCount: metrics.vectorCount,
+            cacheHitRate: metrics.cacheHitRate,
+            memoryUsage: metrics.memoryUsage,
+            cpuUsage: metrics.cpuUsage
+        });
+        // Clean sliding windows
+        this.cleanWindows();
+    }
+    /**
+     * Collect current metrics
+     */
+    collectMetrics() {
+        const memUsage = process.memoryUsage();
+        return {
+            vectorCount: this.vectorCount,
+            cacheHitRate: this.calculateCacheHitRate(),
+            memoryUsage: memUsage.heapUsed,
+            cpuUsage: this.getCPUUsage(),
+            requestsPerSecond: this.calculateRPS(),
+            averageLatency: this.calculateAverageLatency(),
+            errorRate: this.calculateErrorRate()
+        };
+    }
+    /**
+     * Calculate cache hit rate
+     */
+    calculateCacheHitRate() {
+        const total = this.cacheHits + this.cacheMisses;
+        if (total === 0)
+            return 0;
+        return this.cacheHits / total;
+    }
+    /**
+     * Calculate requests per second
+     */
+    calculateRPS() {
+        const now = Date.now();
+        const recentRequests = this.metricsWindow.filter(timestamp => now - timestamp < this.windowSize);
+        return recentRequests.length / (this.windowSize / 1000);
+    }
+    /**
+     * Calculate average latency
+     */
+    calculateAverageLatency() {
+        if (this.latencyWindow.length === 0)
+            return 0;
+        const sum = this.latencyWindow.reduce((a, b) => a + b, 0);
+        return sum / this.latencyWindow.length;
+    }
+    /**
+     * Calculate error rate
+     */
+    calculateErrorRate() {
+        if (this.requestCount === 0)
+            return 0;
+        return this.errorCount / this.requestCount;
+    }
+    /**
+     * Get CPU usage (simplified)
+     */
+    getCPUUsage() {
+        // Simplified CPU usage based on process time
+        const usage = process.cpuUsage();
+        const total = usage.user + usage.system;
+        const seconds = (Date.now() - this.startTime) / 1000;
+        return Math.min(100, (total / 1000000 / seconds) * 100);
+    }
+    /**
+     * Clean old entries from sliding windows
+     */
+    cleanWindows() {
+        const now = Date.now();
+        const cutoff = now - this.windowSize;
+        this.metricsWindow = this.metricsWindow.filter(t => t > cutoff);
+        // Keep only recent latency measurements
+        if (this.latencyWindow.length > 100) {
+            this.latencyWindow = this.latencyWindow.slice(-100);
+        }
+    }
+    /**
+     * Record a request
+     * @param latency - Request latency in milliseconds
+     * @param error - Whether the request resulted in an error
+     */
+    recordRequest(latency, error = false) {
+        this.requestCount++;
+        this.metricsWindow.push(Date.now());
+        this.latencyWindow.push(latency);
+        if (error) {
+            this.errorCount++;
+        }
+    }
+    /**
+     * Record cache access
+     * @param hit - Whether it was a cache hit
+     */
+    recordCacheAccess(hit) {
+        if (hit) {
+            this.cacheHits++;
+        }
+        else {
+            this.cacheMisses++;
+        }
+    }
+    /**
+     * Update vector count
+     * @param count - New vector count
+     */
+    updateVectorCount(count) {
+        this.vectorCount = count;
+    }
+    /**
+     * Get current health status
+     * @returns Health status object
+     */
+    getHealthStatus() {
+        const metrics = this.collectMetrics();
+        const uptime = Date.now() - this.startTime;
+        const warnings = [];
+        const errors = [];
+        // Check for warnings
+        if (metrics.memoryUsage > 1024 * 1024 * 1024) { // > 1GB
+            warnings.push('High memory usage detected');
+        }
+        if (metrics.cacheHitRate < 0.5) {
+            warnings.push('Low cache hit rate');
+        }
+        if (metrics.errorRate && metrics.errorRate > 0.05) {
+            warnings.push('High error rate detected');
+        }
+        if (metrics.averageLatency && metrics.averageLatency > 1000) {
+            warnings.push('High latency detected');
+        }
+        // Check for errors
+        if (metrics.memoryUsage > 2 * 1024 * 1024 * 1024) { // > 2GB
+            errors.push('Critical memory usage');
+        }
+        if (metrics.errorRate && metrics.errorRate > 0.2) {
+            errors.push('Critical error rate');
+        }
+        // Determine overall status
+        let status = 'healthy';
+        if (errors.length > 0) {
+            status = 'unhealthy';
+        }
+        else if (warnings.length > 0) {
+            status = 'degraded';
+        }
+        return {
+            status,
+            instanceId: this.configManager.getInstanceId(),
+            role: this.configManager.getRole(),
+            uptime,
+            lastCheck: new Date().toISOString(),
+            metrics,
+            warnings: warnings.length > 0 ? warnings : undefined,
+            errors: errors.length > 0 ? errors : undefined
+        };
+    }
+    /**
+     * Get health check endpoint data
+     * @returns JSON-serializable health data
+     */
+    getHealthEndpointData() {
+        const status = this.getHealthStatus();
+        return {
+            status: status.status,
+            instanceId: status.instanceId,
+            role: status.role,
+            uptime: Math.floor(status.uptime / 1000), // Convert to seconds
+            lastCheck: status.lastCheck,
+            metrics: {
+                vectorCount: status.metrics.vectorCount,
+                cacheHitRate: Math.round(status.metrics.cacheHitRate * 100) / 100,
+                memoryUsageMB: Math.round(status.metrics.memoryUsage / 1024 / 1024),
+                cpuUsagePercent: Math.round(status.metrics.cpuUsage || 0),
+                requestsPerSecond: Math.round(status.metrics.requestsPerSecond || 0),
+                averageLatencyMs: Math.round(status.metrics.averageLatency || 0),
+                errorRate: Math.round((status.metrics.errorRate || 0) * 100) / 100
+            },
+            warnings: status.warnings,
+            errors: status.errors
+        };
+    }
+    /**
+     * Reset metrics (useful for testing)
+     */
+    resetMetrics() {
+        this.requestCount = 0;
+        this.errorCount = 0;
+        this.totalLatency = 0;
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        this.metricsWindow = [];
+        this.latencyWindow = [];
+    }
+}
+//# sourceMappingURL=healthMonitor.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/httpTransport.d.ts

@@ -0,0 +1,120 @@
+/**
+ * HTTP + SSE Transport for Zero-Config Distributed Brainy
+ * Simple, reliable, works everywhere - no WebSocket complexity!
+ * REAL PRODUCTION CODE - Handles millions of operations
+ */
+import * as http from 'http';
+import { EventEmitter } from 'events';
+export interface TransportMessage {
+    id: string;
+    method: string;
+    params: any;
+    timestamp: number;
+    from: string;
+    to?: string;
+}
+export interface TransportResponse {
+    id: string;
+    result?: any;
+    error?: {
+        code: number;
+        message: string;
+        data?: any;
+    };
+    timestamp: number;
+}
+export interface SSEClient {
+    id: string;
+    response: http.ServerResponse;
+    lastPing: number;
+}
+export declare class HTTPTransport extends EventEmitter {
+    private server;
+    private port;
+    private nodeId;
+    private endpoints;
+    private pendingRequests;
+    private sseClients;
+    private messageHandlers;
+    private isRunning;
+    private readonly REQUEST_TIMEOUT;
+    private readonly SSE_HEARTBEAT_INTERVAL;
+    private sseHeartbeatTimer;
+    constructor(nodeId: string);
+    /**
+     * Start HTTP server with automatic port selection
+     */
+    start(): Promise<number>;
+    /**
+     * Stop HTTP server
+     */
+    stop(): Promise<void>;
+    /**
+     * Register a node endpoint
+     */
+    registerEndpoint(nodeId: string, endpoint: string): void;
+    /**
+     * Register RPC method handler
+     */
+    registerHandler(method: string, handler: (params: any, from: string) => Promise<any>): void;
+    /**
+     * Call RPC method on remote node
+     */
+    call(nodeId: string, method: string, params: any): Promise<any>;
+    /**
+     * Broadcast to all SSE clients
+     */
+    broadcast(event: string, data: any): void;
+    /**
+     * Handle health check
+     */
+    private handleHealth;
+    /**
+     * Handle RPC requests
+     */
+    private handleRPC;
+    /**
+     * Handle SSE connections for real-time updates
+     */
+    private handleSSE;
+    /**
+     * Handle streaming data (for shard migration)
+     */
+    private handleStream;
+    /**
+     * Handle stream upload (receiving data)
+     */
+    private handleStreamUpload;
+    /**
+     * Handle stream download (sending data)
+     */
+    private handleStreamDownload;
+    /**
+     * Send HTTP request to another node
+     */
+    private sendHTTPRequest;
+    /**
+     * Read request body
+     */
+    private readBody;
+    /**
+     * Find an available port
+     */
+    private findAvailablePort;
+    /**
+     * Start SSE heartbeat to keep connections alive
+     */
+    private startSSEHeartbeat;
+    /**
+     * Generate unique ID
+     */
+    private generateId;
+    /**
+     * Get connected nodes count
+     */
+    getConnectionCount(): number;
+    /**
+     * Get SSE client count
+     */
+    getSSEClientCount(): number;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/httpTransport.js

@@ -0,0 +1,446 @@
+/**
+ * HTTP + SSE Transport for Zero-Config Distributed Brainy
+ * Simple, reliable, works everywhere - no WebSocket complexity!
+ * REAL PRODUCTION CODE - Handles millions of operations
+ */
+import * as http from 'http';
+import * as https from 'https';
+import { EventEmitter } from 'events';
+import * as net from 'net';
+import { URL } from 'url';
+export class HTTPTransport extends EventEmitter {
+    constructor(nodeId) {
+        super();
+        this.server = null;
+        this.port = 0;
+        this.endpoints = new Map(); // nodeId -> endpoint
+        this.pendingRequests = new Map();
+        this.sseClients = new Map();
+        this.messageHandlers = new Map();
+        this.isRunning = false;
+        this.REQUEST_TIMEOUT = 30000; // 30 seconds
+        this.SSE_HEARTBEAT_INTERVAL = 15000; // 15 seconds
+        this.sseHeartbeatTimer = null;
+        this.nodeId = nodeId;
+    }
+    /**
+     * Start HTTP server with automatic port selection
+     */
+    async start() {
+        if (this.isRunning)
+            return this.port;
+        // Create HTTP server with all handlers
+        this.server = http.createServer(async (req, res) => {
+            // Enable CORS for browser compatibility
+            res.setHeader('Access-Control-Allow-Origin', '*');
+            res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+            res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+            if (req.method === 'OPTIONS') {
+                res.writeHead(204);
+                res.end();
+                return;
+            }
+            const url = new URL(req.url || '/', `http://${req.headers.host}`);
+            try {
+                // Route requests
+                if (url.pathname === '/health') {
+                    await this.handleHealth(req, res);
+                }
+                else if (url.pathname === '/rpc') {
+                    await this.handleRPC(req, res);
+                }
+                else if (url.pathname === '/events') {
+                    await this.handleSSE(req, res);
+                }
+                else if (url.pathname.startsWith('/stream/')) {
+                    await this.handleStream(req, res, url);
+                }
+                else {
+                    res.writeHead(404);
+                    res.end('Not Found');
+                }
+            }
+            catch (err) {
+                console.error(`[${this.nodeId}] Request error:`, err);
+                res.writeHead(500);
+                res.end('Internal Server Error');
+            }
+        });
+        // Find available port
+        this.port = await this.findAvailablePort();
+        // Start server
+        await new Promise((resolve, reject) => {
+            this.server.listen(this.port, () => {
+                console.log(`[${this.nodeId}] HTTP transport listening on port ${this.port}`);
+                resolve();
+            });
+            this.server.on('error', reject);
+        });
+        this.isRunning = true;
+        // Start SSE heartbeat
+        this.startSSEHeartbeat();
+        this.emit('started', this.port);
+        return this.port;
+    }
+    /**
+     * Stop HTTP server
+     */
+    async stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        // Stop SSE heartbeat
+        if (this.sseHeartbeatTimer) {
+            clearInterval(this.sseHeartbeatTimer);
+            this.sseHeartbeatTimer = null;
+        }
+        // Close all SSE connections
+        for (const client of this.sseClients.values()) {
+            client.response.end();
+        }
+        this.sseClients.clear();
+        // Cancel pending requests
+        for (const pending of this.pendingRequests.values()) {
+            clearTimeout(pending.timeout);
+            pending.reject(new Error('Transport shutting down'));
+        }
+        this.pendingRequests.clear();
+        // Close server
+        if (this.server) {
+            await new Promise(resolve => {
+                this.server.close(() => resolve());
+            });
+            this.server = null;
+        }
+        this.emit('stopped');
+    }
+    /**
+     * Register a node endpoint
+     */
+    registerEndpoint(nodeId, endpoint) {
+        this.endpoints.set(nodeId, endpoint);
+        this.emit('endpointRegistered', { nodeId, endpoint });
+    }
+    /**
+     * Register RPC method handler
+     */
+    registerHandler(method, handler) {
+        this.messageHandlers.set(method, handler);
+    }
+    /**
+     * Call RPC method on remote node
+     */
+    async call(nodeId, method, params) {
+        const endpoint = this.endpoints.get(nodeId);
+        if (!endpoint) {
+            throw new Error(`No endpoint registered for node ${nodeId}`);
+        }
+        const message = {
+            id: this.generateId(),
+            method,
+            params,
+            timestamp: Date.now(),
+            from: this.nodeId,
+            to: nodeId
+        };
+        // Send HTTP request
+        const response = await this.sendHTTPRequest(endpoint, '/rpc', message);
+        if (response.error) {
+            throw new Error(response.error.message);
+        }
+        return response.result;
+    }
+    /**
+     * Broadcast to all SSE clients
+     */
+    broadcast(event, data) {
+        const message = JSON.stringify({ event, data, timestamp: Date.now() });
+        for (const [clientId, client] of this.sseClients.entries()) {
+            try {
+                client.response.write(`data: ${message}\n\n`);
+            }
+            catch (err) {
+                // Client disconnected
+                console.debug(`[${this.nodeId}] SSE client ${clientId} disconnected`);
+                this.sseClients.delete(clientId);
+            }
+        }
+    }
+    /**
+     * Handle health check
+     */
+    async handleHealth(req, res) {
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({
+            status: 'healthy',
+            nodeId: this.nodeId,
+            uptime: process.uptime(),
+            memory: process.memoryUsage(),
+            connections: this.sseClients.size
+        }));
+    }
+    /**
+     * Handle RPC requests
+     */
+    async handleRPC(req, res) {
+        if (req.method !== 'POST') {
+            res.writeHead(405);
+            res.end('Method Not Allowed');
+            return;
+        }
+        // Read body
+        const body = await this.readBody(req);
+        let message;
+        try {
+            message = JSON.parse(body);
+        }
+        catch (err) {
+            res.writeHead(400);
+            res.end('Invalid JSON');
+            return;
+        }
+        // Handle the RPC call
+        const response = {
+            id: message.id,
+            timestamp: Date.now()
+        };
+        try {
+            const handler = this.messageHandlers.get(message.method);
+            if (!handler) {
+                throw new Error(`Method ${message.method} not found`);
+            }
+            response.result = await handler(message.params, message.from);
+        }
+        catch (err) {
+            response.error = {
+                code: -32603,
+                message: err.message,
+                data: err.stack
+            };
+        }
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify(response));
+    }
+    /**
+     * Handle SSE connections for real-time updates
+     */
+    async handleSSE(req, res) {
+        // Setup SSE headers
+        res.writeHead(200, {
+            'Content-Type': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+            'Connection': 'keep-alive',
+            'X-Accel-Buffering': 'no' // Disable Nginx buffering
+        });
+        // Send initial connection event
+        const clientId = this.generateId();
+        res.write(`data: ${JSON.stringify({
+            event: 'connected',
+            clientId,
+            nodeId: this.nodeId
+        })}\n\n`);
+        // Store client
+        this.sseClients.set(clientId, {
+            id: clientId,
+            response: res,
+            lastPing: Date.now()
+        });
+        // Handle client disconnect
+        req.on('close', () => {
+            this.sseClients.delete(clientId);
+            this.emit('sseDisconnected', clientId);
+        });
+        this.emit('sseConnected', clientId);
+    }
+    /**
+     * Handle streaming data (for shard migration)
+     */
+    async handleStream(req, res, url) {
+        const streamId = url.pathname.split('/').pop();
+        if (req.method === 'POST') {
+            // Receiving stream
+            await this.handleStreamUpload(req, res, streamId);
+        }
+        else if (req.method === 'GET') {
+            // Sending stream
+            await this.handleStreamDownload(req, res, streamId);
+        }
+        else {
+            res.writeHead(405);
+            res.end('Method Not Allowed');
+        }
+    }
+    /**
+     * Handle stream upload (receiving data)
+     */
+    async handleStreamUpload(req, res, streamId) {
+        const chunks = [];
+        let totalSize = 0;
+        req.on('data', (chunk) => {
+            chunks.push(chunk);
+            totalSize += chunk.length;
+            // Emit progress
+            this.emit('streamProgress', {
+                streamId,
+                type: 'upload',
+                bytes: totalSize
+            });
+        });
+        req.on('end', () => {
+            const data = Buffer.concat(chunks);
+            // Process the streamed data
+            this.emit('streamComplete', {
+                streamId,
+                type: 'upload',
+                data,
+                size: totalSize
+            });
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify({
+                success: true,
+                streamId,
+                size: totalSize
+            }));
+        });
+        req.on('error', (err) => {
+            console.error(`[${this.nodeId}] Stream upload error:`, err);
+            res.writeHead(500);
+            res.end('Stream Error');
+        });
+    }
+    /**
+     * Handle stream download (sending data)
+     */
+    async handleStreamDownload(req, res, streamId) {
+        // Stream download not yet implemented
+        // Return error response instead of fake data
+        res.writeHead(501, {
+            'Content-Type': 'application/json'
+        });
+        res.end(JSON.stringify({
+            error: 'Stream download not implemented',
+            message: 'This feature is not yet available in the current version',
+            streamId
+        }));
+        this.emit('streamError', {
+            streamId,
+            type: 'download',
+            error: 'Not implemented'
+        });
+    }
+    /**
+     * Send HTTP request to another node
+     */
+    async sendHTTPRequest(endpoint, path, data) {
+        return new Promise((resolve, reject) => {
+            const url = new URL(path, endpoint);
+            const options = {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                }
+            };
+            const proto = url.protocol === 'https:' ? https : http;
+            const req = proto.request(url, options, (res) => {
+                let body = '';
+                res.on('data', chunk => body += chunk);
+                res.on('end', () => {
+                    try {
+                        const response = JSON.parse(body);
+                        resolve(response);
+                    }
+                    catch (err) {
+                        reject(new Error(`Invalid response: ${body}`));
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.on('timeout', () => {
+                req.destroy();
+                reject(new Error('Request timeout'));
+            });
+            req.setTimeout(this.REQUEST_TIMEOUT);
+            req.write(JSON.stringify(data));
+            req.end();
+        });
+    }
+    /**
+     * Read request body
+     */
+    readBody(req) {
+        return new Promise((resolve, reject) => {
+            let body = '';
+            req.on('data', chunk => body += chunk);
+            req.on('end', () => resolve(body));
+            req.on('error', reject);
+        });
+    }
+    /**
+     * Find an available port
+     */
+    async findAvailablePort(startPort = 7947) {
+        const checkPort = (port) => {
+            return new Promise(resolve => {
+                const server = net.createServer();
+                server.once('error', () => resolve(false));
+                server.once('listening', () => {
+                    server.close();
+                    resolve(true);
+                });
+                server.listen(port);
+            });
+        };
+        // Try preferred port first
+        if (await checkPort(startPort)) {
+            return startPort;
+        }
+        // Find random available port
+        return new Promise((resolve, reject) => {
+            const server = net.createServer();
+            server.once('error', reject);
+            server.once('listening', () => {
+                const port = server.address().port;
+                server.close(() => resolve(port));
+            });
+            server.listen(0); // 0 = random available port
+        });
+    }
+    /**
+     * Start SSE heartbeat to keep connections alive
+     */
+    startSSEHeartbeat() {
+        this.sseHeartbeatTimer = setInterval(() => {
+            const now = Date.now();
+            const ping = JSON.stringify({ event: 'ping', timestamp: now });
+            for (const [clientId, client] of this.sseClients.entries()) {
+                try {
+                    client.response.write(`: ping\n\n`); // SSE comment for keep-alive
+                    client.lastPing = now;
+                }
+                catch (err) {
+                    // Client disconnected
+                    this.sseClients.delete(clientId);
+                }
+            }
+        }, this.SSE_HEARTBEAT_INTERVAL);
+    }
+    /**
+     * Generate unique ID
+     */
+    generateId() {
+        return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+    }
+    /**
+     * Get connected nodes count
+     */
+    getConnectionCount() {
+        return this.endpoints.size;
+    }
+    /**
+     * Get SSE client count
+     */
+    getSSEClientCount() {
+        return this.sseClients.size;
+    }
+}
+//# sourceMappingURL=httpTransport.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Distributed module exports
+ */
+export { DistributedConfigManager } from './configManager.js';
+export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js';
+export { BaseOperationalMode, ReaderMode, WriterMode, HybridMode, OperationalModeFactory } from './operationalModes.js';
+export { DomainDetector } from './domainDetector.js';
+export { HealthMonitor } from './healthMonitor.js';
+export { DistributedCoordinator, createCoordinator } from './coordinator.js';
+export { ShardManager, createShardManager } from './shardManager.js';
+export { CacheSync, createCacheSync } from './cacheSync.js';
+export { ReadWriteSeparation, createReadWriteSeparation } from './readWriteSeparation.js';
+export type { HealthMetrics, HealthStatus } from './healthMonitor.js';
+export type { DomainPattern } from './domainDetector.js';
+export type { NodeInfo, CoordinatorConfig, ConsensusState } from './coordinator.js';
+export type { ShardConfig, Shard, ShardAssignment } from './shardManager.js';
+export type { CacheSyncConfig, CacheEntry, SyncMessage } from './cacheSync.js';
+export type { ReplicationConfig, WriteOperation, ReplicationLog } from './readWriteSeparation.js';

.recovery-workspace/dist-backup-20250910-141917/distributed/index.js

@@ -0,0 +1,14 @@
+/**
+ * Distributed module exports
+ */
+export { DistributedConfigManager } from './configManager.js';
+export { HashPartitioner, AffinityPartitioner } from './hashPartitioner.js';
+export { BaseOperationalMode, ReaderMode, WriterMode, HybridMode, OperationalModeFactory } from './operationalModes.js';
+export { DomainDetector } from './domainDetector.js';
+export { HealthMonitor } from './healthMonitor.js';
+// New distributed scaling components
+export { DistributedCoordinator, createCoordinator } from './coordinator.js';
+export { ShardManager, createShardManager } from './shardManager.js';
+export { CacheSync, createCacheSync } from './cacheSync.js';
+export { ReadWriteSeparation, createReadWriteSeparation } from './readWriteSeparation.js';
+//# sourceMappingURL=index.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/distributed/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC3E,OAAO,EACL,mBAAmB,EACnB,UAAU,EACV,UAAU,EACV,UAAU,EACV,sBAAsB,EACvB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAElD,qCAAqC;AACrC,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAC5E,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACpE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAC3D,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAA"}

.recovery-workspace/dist-backup-20250910-141917/distributed/networkTransport.d.ts

@@ -0,0 +1,132 @@
+/**
+ * Network Transport Layer for Distributed Brainy
+ * Uses WebSocket + HTTP for maximum compatibility
+ */
+import { EventEmitter } from 'events';
+export interface NetworkMessage {
+    type: string;
+    from: string;
+    to?: string;
+    data: any;
+    timestamp: number;
+    id: string;
+}
+export interface NodeEndpoint {
+    nodeId: string;
+    host: string;
+    httpPort: number;
+    wsPort: number;
+    lastSeen: number;
+}
+export interface NetworkConfig {
+    nodeId?: string;
+    host?: string;
+    httpPort?: number;
+    wsPort?: number;
+    seeds?: string[];
+    discoveryMethod?: 'seeds' | 'dns' | 'kubernetes' | 'auto';
+    enableUDP?: boolean;
+}
+/**
+ * Production-ready network transport
+ */
+export declare class NetworkTransport extends EventEmitter {
+    private nodeId;
+    private config;
+    private httpServer?;
+    private wsServer;
+    private peers;
+    private connections;
+    private messageHandlers;
+    private responseHandlers;
+    private isRunning;
+    constructor(config: NetworkConfig);
+    /**
+     * Start network transport
+     */
+    start(): Promise<void>;
+    /**
+     * Stop network transport
+     */
+    stop(): Promise<void>;
+    /**
+     * Start HTTP server for REST API and health checks
+     */
+    private startHTTPServer;
+    /**
+     * Start WebSocket server for real-time communication
+     */
+    private startWebSocketServer;
+    /**
+     * Discover peers based on configuration
+     */
+    private discoverPeers;
+    /**
+     * Connect to seed nodes
+     */
+    private connectToSeeds;
+    /**
+     * Discover peers via DNS
+     */
+    private discoverViaDNS;
+    /**
+     * Discover peers via Kubernetes
+     */
+    private discoverViaKubernetes;
+    /**
+     * Connect to a specific node
+     */
+    private connectToNode;
+    /**
+     * Get node information via HTTP
+     */
+    private getNodeInfo;
+    /**
+     * Connect to peer via WebSocket
+     */
+    private connectWebSocket;
+    /**
+     * Start heartbeat to maintain connections
+     */
+    private startHeartbeat;
+    /**
+     * Send message to specific node
+     */
+    sendToNode(nodeId: string, type: string, data: any): Promise<any>;
+    /**
+     * Send via HTTP
+     */
+    private sendViaHTTP;
+    /**
+     * Broadcast to all peers
+     */
+    broadcast(type: string, data: any): Promise<void>;
+    /**
+     * Register message handler
+     */
+    onMessage(type: string, handler: (msg: NetworkMessage) => Promise<any>): void;
+    /**
+     * Get connected peers
+     */
+    getPeers(): NodeEndpoint[];
+    /**
+     * Check if connected
+     */
+    isConnected(nodeId: string): boolean;
+    /**
+     * Generate node ID
+     */
+    private generateNodeId;
+    /**
+     * Generate message ID
+     */
+    private generateMessageId;
+    /**
+     * Get node ID
+     */
+    getNodeId(): string;
+}
+/**
+ * Create network transport
+ */
+export declare function createNetworkTransport(config: NetworkConfig): NetworkTransport;

.recovery-workspace/dist-backup-20250910-141917/distributed/networkTransport.js

@@ -0,0 +1,633 @@
+/**
+ * Network Transport Layer for Distributed Brainy
+ * Uses WebSocket + HTTP for maximum compatibility
+ */
+import * as http from 'http';
+import { EventEmitter } from 'events';
+import { WebSocket } from 'ws';
+// Use dynamic imports for Node.js specific modules
+let WebSocketServer;
+// Default ports
+const HTTP_PORT = process.env.BRAINY_HTTP_PORT || 7947;
+const WS_PORT = process.env.BRAINY_WS_PORT || 7948;
+/**
+ * Production-ready network transport
+ */
+export class NetworkTransport extends EventEmitter {
+    constructor(config) {
+        super();
+        this.peers = new Map();
+        this.connections = new Map();
+        this.messageHandlers = new Map();
+        this.responseHandlers = new Map();
+        this.isRunning = false;
+        this.nodeId = config.nodeId || this.generateNodeId();
+        this.config = {
+            host: '0.0.0.0',
+            httpPort: Number(HTTP_PORT),
+            wsPort: Number(WS_PORT),
+            discoveryMethod: 'auto',
+            enableUDP: false, // Disabled by default for cloud compatibility
+            ...config
+        };
+    }
+    /**
+     * Start network transport
+     */
+    async start() {
+        if (this.isRunning)
+            return;
+        // Dynamic import for Node.js environment
+        if (typeof window === 'undefined') {
+            const ws = await import('ws');
+            WebSocketServer = ws.WebSocketServer || ws.Server || ws.default;
+        }
+        await this.startHTTPServer();
+        await this.startWebSocketServer();
+        await this.discoverPeers();
+        this.isRunning = true;
+        this.emit('started', { nodeId: this.nodeId });
+        // Start heartbeat
+        this.startHeartbeat();
+    }
+    /**
+     * Stop network transport
+     */
+    async stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        // Close all connections
+        for (const ws of this.connections.values()) {
+            ws.close();
+        }
+        this.connections.clear();
+        // Stop servers
+        if (this.httpServer) {
+            await new Promise((resolve) => {
+                this.httpServer.close(() => resolve());
+            });
+        }
+        if (this.wsServer) {
+            await new Promise((resolve) => {
+                this.wsServer.close(() => resolve());
+            });
+        }
+        this.emit('stopped');
+    }
+    /**
+     * Start HTTP server for REST API and health checks
+     */
+    async startHTTPServer() {
+        return new Promise((resolve, reject) => {
+            this.httpServer = http.createServer(async (req, res) => {
+                // Enable CORS
+                res.setHeader('Access-Control-Allow-Origin', '*');
+                res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+                res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+                if (req.method === 'OPTIONS') {
+                    res.writeHead(200);
+                    res.end();
+                    return;
+                }
+                if (req.url === '/health') {
+                    // Health check endpoint
+                    res.writeHead(200, { 'Content-Type': 'application/json' });
+                    res.end(JSON.stringify({
+                        status: 'healthy',
+                        nodeId: this.nodeId,
+                        peers: Array.from(this.peers.keys()),
+                        connections: Array.from(this.connections.keys())
+                    }));
+                    return;
+                }
+                if (req.url === '/peers') {
+                    // Peer discovery endpoint
+                    res.writeHead(200, { 'Content-Type': 'application/json' });
+                    res.end(JSON.stringify({
+                        nodeId: this.nodeId,
+                        endpoint: {
+                            host: this.config.host,
+                            httpPort: this.config.httpPort,
+                            wsPort: this.config.wsPort
+                        },
+                        peers: Array.from(this.peers.values())
+                    }));
+                    return;
+                }
+                if (req.method === 'POST' && req.url === '/message') {
+                    // Message handling endpoint
+                    let body = '';
+                    req.on('data', (chunk) => {
+                        body += chunk.toString();
+                    });
+                    req.on('end', async () => {
+                        try {
+                            const message = JSON.parse(body);
+                            // Handle message
+                            const handler = this.messageHandlers.get(message.type);
+                            if (handler) {
+                                const response = await handler(message);
+                                res.writeHead(200, { 'Content-Type': 'application/json' });
+                                res.end(JSON.stringify({ success: true, data: response }));
+                            }
+                            else {
+                                res.writeHead(404);
+                                res.end(JSON.stringify({ success: false, error: 'Unknown message type' }));
+                            }
+                        }
+                        catch (err) {
+                            res.writeHead(500);
+                            res.end(JSON.stringify({ success: false, error: err.message }));
+                        }
+                    });
+                }
+                else {
+                    res.writeHead(404);
+                    res.end(JSON.stringify({ error: 'Not found' }));
+                }
+            });
+            this.httpServer.listen(this.config.httpPort, '0.0.0.0', () => {
+                console.log(`[Network] HTTP server listening on :${this.config.httpPort}`);
+                resolve();
+            });
+            this.httpServer.on('error', reject);
+        });
+    }
+    /**
+     * Start WebSocket server for real-time communication
+     */
+    async startWebSocketServer() {
+        if (!WebSocketServer) {
+            console.warn('[Network] WebSocket not available in this environment');
+            return;
+        }
+        this.wsServer = new WebSocketServer({ port: this.config.wsPort });
+        this.wsServer.on('connection', (ws) => {
+            let peerId = null;
+            ws.on('message', async (data) => {
+                try {
+                    const message = JSON.parse(data.toString());
+                    // Handle handshake
+                    if (message.type === 'HANDSHAKE') {
+                        peerId = message.from;
+                        this.connections.set(peerId, ws);
+                        // Add to peers
+                        const endpoint = {
+                            nodeId: peerId,
+                            host: message.data.host || 'unknown',
+                            httpPort: message.data.httpPort || 0,
+                            wsPort: message.data.wsPort || 0,
+                            lastSeen: Date.now()
+                        };
+                        this.peers.set(peerId, endpoint);
+                        // Send handshake response
+                        ws.send(JSON.stringify({
+                            type: 'HANDSHAKE_ACK',
+                            from: this.nodeId,
+                            to: peerId,
+                            data: {
+                                nodeId: this.nodeId,
+                                host: this.config.host,
+                                httpPort: this.config.httpPort,
+                                wsPort: this.config.wsPort
+                            },
+                            timestamp: Date.now(),
+                            id: this.generateMessageId()
+                        }));
+                        this.emit('peerConnected', peerId);
+                        return;
+                    }
+                    // Handle response messages
+                    if (message.type.endsWith('_RESPONSE')) {
+                        const handler = this.responseHandlers.get(message.id);
+                        if (handler) {
+                            handler(message.data);
+                            this.responseHandlers.delete(message.id);
+                        }
+                        return;
+                    }
+                    // Handle regular messages
+                    const handler = this.messageHandlers.get(message.type);
+                    if (handler) {
+                        const response = await handler(message);
+                        if (response !== undefined) {
+                            ws.send(JSON.stringify({
+                                type: `${message.type}_RESPONSE`,
+                                from: this.nodeId,
+                                to: message.from,
+                                data: response,
+                                timestamp: Date.now(),
+                                id: message.id
+                            }));
+                        }
+                    }
+                }
+                catch (err) {
+                    console.error('[Network] WebSocket message error:', err);
+                }
+            });
+            ws.on('close', () => {
+                if (peerId) {
+                    this.connections.delete(peerId);
+                    this.emit('peerDisconnected', peerId);
+                }
+            });
+            ws.on('error', (err) => {
+                console.error(`[Network] WebSocket error:`, err.message);
+            });
+        });
+        console.log(`[Network] WebSocket server listening on :${this.config.wsPort}`);
+    }
+    /**
+     * Discover peers based on configuration
+     */
+    async discoverPeers() {
+        const method = this.config.discoveryMethod;
+        if (method === 'seeds' || (method === 'auto' && this.config.seeds)) {
+            // Use seed nodes
+            await this.connectToSeeds();
+        }
+        else if (method === 'dns' || (method === 'auto' && process.env.BRAINY_DNS)) {
+            // Use DNS discovery
+            await this.discoverViaDNS();
+        }
+        else if (method === 'kubernetes' || (method === 'auto' && process.env.KUBERNETES_SERVICE_HOST)) {
+            // Use Kubernetes service discovery
+            await this.discoverViaKubernetes();
+        }
+        // Fallback: try localhost for development
+        if (this.peers.size === 0 && process.env.NODE_ENV !== 'production') {
+            await this.connectToNode('localhost', this.config.httpPort);
+        }
+    }
+    /**
+     * Connect to seed nodes
+     */
+    async connectToSeeds() {
+        if (!this.config.seeds)
+            return;
+        for (const seed of this.config.seeds) {
+            try {
+                // Parse seed address (host:port or just host)
+                const [host, port] = seed.split(':');
+                await this.connectToNode(host, Number(port) || this.config.httpPort);
+            }
+            catch (err) {
+                console.error(`[Network] Failed to connect to seed ${seed}:`, err);
+            }
+        }
+    }
+    /**
+     * Discover peers via DNS
+     */
+    async discoverViaDNS() {
+        const dnsName = process.env.BRAINY_DNS || 'brainy-cluster.local';
+        try {
+            const dns = await import('dns');
+            const addresses = await new Promise((resolve, reject) => {
+                dns.resolve4(dnsName, (err, addresses) => {
+                    if (err)
+                        reject(err);
+                    else
+                        resolve(addresses || []);
+                });
+            });
+            for (const address of addresses) {
+                await this.connectToNode(address, this.config.httpPort);
+            }
+        }
+        catch (err) {
+            console.log(`[Network] DNS discovery failed for ${dnsName}:`, err);
+        }
+    }
+    /**
+     * Discover peers via Kubernetes
+     */
+    async discoverViaKubernetes() {
+        const serviceName = process.env.BRAINY_SERVICE || 'brainy';
+        const namespace = process.env.BRAINY_NAMESPACE || 'default';
+        const apiServer = 'https://kubernetes.default.svc';
+        const token = process.env.KUBERNETES_TOKEN || '';
+        try {
+            // Query Kubernetes API for pod endpoints
+            const https = await import('https');
+            const response = await new Promise((resolve, reject) => {
+                https.get(`${apiServer}/api/v1/namespaces/${namespace}/endpoints/${serviceName}`, {
+                    headers: {
+                        'Authorization': `Bearer ${token}`
+                    }
+                }, (res) => {
+                    let data = '';
+                    res.on('data', (chunk) => data += chunk);
+                    res.on('end', () => resolve(JSON.parse(data)));
+                }).on('error', reject);
+            });
+            // Connect to each pod
+            if (response.subsets) {
+                for (const subset of response.subsets) {
+                    for (const address of subset.addresses || []) {
+                        await this.connectToNode(address.ip, this.config.httpPort);
+                    }
+                }
+            }
+        }
+        catch (err) {
+            console.log('[Network] Kubernetes discovery failed:', err);
+        }
+    }
+    /**
+     * Connect to a specific node
+     */
+    async connectToNode(host, httpPort) {
+        try {
+            // First, get node info via HTTP
+            const nodeInfo = await this.getNodeInfo(host, httpPort);
+            if (nodeInfo.nodeId === this.nodeId) {
+                return; // Don't connect to self
+            }
+            // Add to peers
+            const endpoint = {
+                nodeId: nodeInfo.nodeId,
+                host,
+                httpPort,
+                wsPort: nodeInfo.endpoint.wsPort,
+                lastSeen: Date.now()
+            };
+            this.peers.set(nodeInfo.nodeId, endpoint);
+            // Connect via WebSocket
+            await this.connectWebSocket(endpoint);
+            // Get their peer list
+            for (const peer of nodeInfo.peers || []) {
+                if (!this.peers.has(peer.nodeId) && peer.nodeId !== this.nodeId) {
+                    this.peers.set(peer.nodeId, peer);
+                    // Optionally connect to them too
+                    if (this.peers.size < 10) { // Limit connections
+                        await this.connectWebSocket(peer);
+                    }
+                }
+            }
+        }
+        catch (err) {
+            // Node might be down or not ready
+            console.debug(`[Network] Could not connect to ${host}:${httpPort}`);
+        }
+    }
+    /**
+     * Get node information via HTTP
+     */
+    async getNodeInfo(host, port) {
+        return new Promise((resolve, reject) => {
+            const req = http.get(`http://${host}:${port}/peers`, (res) => {
+                let data = '';
+                res.on('data', (chunk) => data += chunk);
+                res.on('end', () => {
+                    try {
+                        resolve(JSON.parse(data));
+                    }
+                    catch (err) {
+                        reject(err);
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.setTimeout(2000, () => {
+                req.destroy();
+                reject(new Error('Timeout'));
+            });
+        });
+    }
+    /**
+     * Connect to peer via WebSocket
+     */
+    async connectWebSocket(endpoint) {
+        if (this.connections.has(endpoint.nodeId))
+            return;
+        try {
+            const ws = new WebSocket(`ws://${endpoint.host}:${endpoint.wsPort}`);
+            ws.on('open', () => {
+                // Send handshake
+                ws.send(JSON.stringify({
+                    type: 'HANDSHAKE',
+                    from: this.nodeId,
+                    data: {
+                        nodeId: this.nodeId,
+                        host: this.config.host,
+                        httpPort: this.config.httpPort,
+                        wsPort: this.config.wsPort
+                    },
+                    timestamp: Date.now(),
+                    id: this.generateMessageId()
+                }));
+                this.connections.set(endpoint.nodeId, ws);
+            });
+            ws.on('message', async (data) => {
+                try {
+                    const message = JSON.parse(data.toString());
+                    // Handle responses
+                    if (message.type.endsWith('_RESPONSE')) {
+                        const handler = this.responseHandlers.get(message.id);
+                        if (handler) {
+                            handler(message.data);
+                            this.responseHandlers.delete(message.id);
+                        }
+                        return;
+                    }
+                    // Handle messages
+                    const handler = this.messageHandlers.get(message.type);
+                    if (handler) {
+                        await handler(message);
+                    }
+                }
+                catch (err) {
+                    console.error('[Network] Message handling error:', err);
+                }
+            });
+            ws.on('close', () => {
+                this.connections.delete(endpoint.nodeId);
+            });
+            ws.on('error', (err) => {
+                console.debug(`[Network] WebSocket error with ${endpoint.nodeId}:`, err.message);
+            });
+        }
+        catch (err) {
+            console.debug(`[Network] Failed to connect to ${endpoint.nodeId}`);
+        }
+    }
+    /**
+     * Start heartbeat to maintain connections
+     */
+    startHeartbeat() {
+        setInterval(() => {
+            if (!this.isRunning)
+                return;
+            // Send heartbeat to all connected peers
+            for (const [nodeId, ws] of this.connections.entries()) {
+                if (ws.readyState === WebSocket.OPEN) {
+                    ws.send(JSON.stringify({
+                        type: 'HEARTBEAT',
+                        from: this.nodeId,
+                        to: nodeId,
+                        timestamp: Date.now(),
+                        id: this.generateMessageId()
+                    }));
+                }
+                else {
+                    // Connection lost, remove it
+                    this.connections.delete(nodeId);
+                }
+            }
+            // Clean up stale peers
+            const now = Date.now();
+            for (const [nodeId, peer] of this.peers.entries()) {
+                if (now - peer.lastSeen > 60000) { // 60 seconds
+                    this.peers.delete(nodeId);
+                }
+            }
+        }, 10000); // Every 10 seconds
+    }
+    /**
+     * Send message to specific node
+     */
+    async sendToNode(nodeId, type, data) {
+        const ws = this.connections.get(nodeId);
+        if (ws && ws.readyState === WebSocket.OPEN) {
+            // Use WebSocket
+            return new Promise((resolve, reject) => {
+                const messageId = this.generateMessageId();
+                const timeout = setTimeout(() => {
+                    this.responseHandlers.delete(messageId);
+                    reject(new Error(`Timeout waiting for response from ${nodeId}`));
+                }, 5000);
+                this.responseHandlers.set(messageId, (response) => {
+                    clearTimeout(timeout);
+                    resolve(response);
+                });
+                ws.send(JSON.stringify({
+                    type,
+                    from: this.nodeId,
+                    to: nodeId,
+                    data,
+                    timestamp: Date.now(),
+                    id: messageId
+                }));
+            });
+        }
+        else {
+            // Use HTTP fallback
+            const endpoint = this.peers.get(nodeId);
+            if (!endpoint) {
+                throw new Error(`Unknown node: ${nodeId}`);
+            }
+            return this.sendViaHTTP(endpoint, type, data);
+        }
+    }
+    /**
+     * Send via HTTP
+     */
+    async sendViaHTTP(endpoint, type, data) {
+        return new Promise((resolve, reject) => {
+            const message = {
+                type,
+                from: this.nodeId,
+                to: endpoint.nodeId,
+                data,
+                timestamp: Date.now(),
+                id: this.generateMessageId()
+            };
+            const postData = JSON.stringify(message);
+            const req = http.request({
+                hostname: endpoint.host,
+                port: endpoint.httpPort,
+                path: '/message',
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(postData)
+                }
+            }, (res) => {
+                let responseData = '';
+                res.on('data', (chunk) => {
+                    responseData += chunk;
+                });
+                res.on('end', () => {
+                    try {
+                        const response = JSON.parse(responseData);
+                        if (response.success) {
+                            resolve(response.data);
+                        }
+                        else {
+                            reject(new Error(response.error));
+                        }
+                    }
+                    catch (err) {
+                        reject(err);
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.setTimeout(5000, () => {
+                req.destroy();
+                reject(new Error('HTTP timeout'));
+            });
+            req.write(postData);
+            req.end();
+        });
+    }
+    /**
+     * Broadcast to all peers
+     */
+    async broadcast(type, data) {
+        const promises = [];
+        for (const nodeId of this.peers.keys()) {
+            promises.push(this.sendToNode(nodeId, type, data).catch(() => {
+                // Ignore broadcast failures
+            }));
+        }
+        await Promise.all(promises);
+    }
+    /**
+     * Register message handler
+     */
+    onMessage(type, handler) {
+        this.messageHandlers.set(type, handler);
+    }
+    /**
+     * Get connected peers
+     */
+    getPeers() {
+        return Array.from(this.peers.values());
+    }
+    /**
+     * Check if connected
+     */
+    isConnected(nodeId) {
+        const ws = this.connections.get(nodeId);
+        return ws !== undefined && ws.readyState === WebSocket.OPEN;
+    }
+    /**
+     * Generate node ID
+     */
+    generateNodeId() {
+        return `node-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+    }
+    /**
+     * Generate message ID
+     */
+    generateMessageId() {
+        return `${this.nodeId}-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
+    }
+    /**
+     * Get node ID
+     */
+    getNodeId() {
+        return this.nodeId;
+    }
+}
+/**
+ * Create network transport
+ */
+export function createNetworkTransport(config) {
+    return new NetworkTransport(config);
+}
+//# sourceMappingURL=networkTransport.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/operationalModes.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Operational Modes for Distributed Brainy
+ * Defines different modes with optimized caching strategies
+ */
+import { OperationalMode, CacheStrategy, InstanceRole } from '../types/distributedTypes.js';
+/**
+ * Base operational mode
+ */
+export declare abstract class BaseOperationalMode implements OperationalMode {
+    abstract canRead: boolean;
+    abstract canWrite: boolean;
+    abstract canDelete: boolean;
+    abstract cacheStrategy: CacheStrategy;
+    /**
+     * Validate operation is allowed in this mode
+     */
+    validateOperation(operation: 'read' | 'write' | 'delete'): void;
+}
+/**
+ * Read-only mode optimized for query performance
+ */
+export declare class ReaderMode extends BaseOperationalMode {
+    canRead: boolean;
+    canWrite: boolean;
+    canDelete: boolean;
+    cacheStrategy: CacheStrategy;
+    /**
+     * Get optimized cache configuration for readers
+     */
+    getCacheConfig(): {
+        hotCacheMaxSize: number;
+        hotCacheEvictionThreshold: number;
+        warmCacheTTL: number;
+        batchSize: number;
+        autoTune: boolean;
+        autoTuneInterval: number;
+        readOnly: boolean;
+    };
+}
+/**
+ * Write-only mode optimized for ingestion
+ */
+export declare class WriterMode extends BaseOperationalMode {
+    canRead: boolean;
+    canWrite: boolean;
+    canDelete: boolean;
+    cacheStrategy: CacheStrategy;
+    /**
+     * Get optimized cache configuration for writers
+     */
+    getCacheConfig(): {
+        hotCacheMaxSize: number;
+        hotCacheEvictionThreshold: number;
+        warmCacheTTL: number;
+        batchSize: number;
+        autoTune: boolean;
+        writeOnly: boolean;
+    };
+}
+/**
+ * Hybrid mode that can both read and write
+ */
+export declare class HybridMode extends BaseOperationalMode {
+    canRead: boolean;
+    canWrite: boolean;
+    canDelete: boolean;
+    cacheStrategy: CacheStrategy;
+    private readWriteRatio;
+    /**
+     * Get balanced cache configuration
+     */
+    getCacheConfig(): {
+        hotCacheMaxSize: number;
+        hotCacheEvictionThreshold: number;
+        warmCacheTTL: number;
+        batchSize: number;
+        autoTune: boolean;
+        autoTuneInterval: number;
+    };
+    /**
+     * Update cache strategy based on workload
+     * @param readCount - Number of recent reads
+     * @param writeCount - Number of recent writes
+     */
+    updateWorkloadBalance(readCount: number, writeCount: number): void;
+}
+/**
+ * Factory for creating operational modes
+ */
+export declare class OperationalModeFactory {
+    /**
+     * Create operational mode based on role
+     * @param role - The instance role
+     * @returns The appropriate operational mode
+     */
+    static createMode(role: InstanceRole): BaseOperationalMode;
+    /**
+     * Create mode with custom cache strategy
+     * @param role - The instance role
+     * @param customStrategy - Custom cache strategy overrides
+     * @returns The operational mode with custom strategy
+     */
+    static createModeWithStrategy(role: InstanceRole, customStrategy: Partial<CacheStrategy>): BaseOperationalMode;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/operationalModes.js

@@ -0,0 +1,201 @@
+/**
+ * Operational Modes for Distributed Brainy
+ * Defines different modes with optimized caching strategies
+ */
+/**
+ * Base operational mode
+ */
+export class BaseOperationalMode {
+    /**
+     * Validate operation is allowed in this mode
+     */
+    validateOperation(operation) {
+        switch (operation) {
+            case 'read':
+                if (!this.canRead) {
+                    throw new Error('Read operations are not allowed in write-only mode');
+                }
+                break;
+            case 'write':
+                if (!this.canWrite) {
+                    throw new Error('Write operations are not allowed in read-only mode');
+                }
+                break;
+            case 'delete':
+                if (!this.canDelete) {
+                    throw new Error('Delete operations are not allowed in this mode');
+                }
+                break;
+        }
+    }
+}
+/**
+ * Read-only mode optimized for query performance
+ */
+export class ReaderMode extends BaseOperationalMode {
+    constructor() {
+        super(...arguments);
+        this.canRead = true;
+        this.canWrite = false;
+        this.canDelete = false;
+        this.cacheStrategy = {
+            hotCacheRatio: 0.8, // 80% of memory for read cache
+            prefetchAggressive: true, // Aggressively prefetch related vectors
+            ttl: 3600000, // 1 hour cache TTL
+            compressionEnabled: true, // Trade CPU for more cache capacity
+            writeBufferSize: 0, // No write buffer needed
+            batchWrites: false, // No writes
+            adaptive: true // Adapt to query patterns
+        };
+    }
+    /**
+     * Get optimized cache configuration for readers
+     */
+    getCacheConfig() {
+        return {
+            hotCacheMaxSize: 1000000, // Large hot cache
+            hotCacheEvictionThreshold: 0.9, // Keep cache full
+            warmCacheTTL: 3600000, // 1 hour warm cache
+            batchSize: 100, // Large batch reads
+            autoTune: true, // Auto-tune for read patterns
+            autoTuneInterval: 60000, // Tune every minute
+            readOnly: true // Enable read-only optimizations
+        };
+    }
+}
+/**
+ * Write-only mode optimized for ingestion
+ */
+export class WriterMode extends BaseOperationalMode {
+    constructor() {
+        super(...arguments);
+        this.canRead = false;
+        this.canWrite = true;
+        this.canDelete = true;
+        this.cacheStrategy = {
+            hotCacheRatio: 0.2, // Only 20% for cache, rest for write buffer
+            prefetchAggressive: false, // No prefetching needed
+            ttl: 60000, // Short TTL (1 minute)
+            compressionEnabled: false, // Speed over memory efficiency
+            writeBufferSize: 10000, // Large write buffer for batching
+            batchWrites: true, // Enable write batching
+            adaptive: false // Fixed strategy for consistent writes
+        };
+    }
+    /**
+     * Get optimized cache configuration for writers
+     */
+    getCacheConfig() {
+        return {
+            hotCacheMaxSize: 100000, // Small hot cache
+            hotCacheEvictionThreshold: 0.5, // Aggressive eviction
+            warmCacheTTL: 60000, // 1 minute warm cache
+            batchSize: 1000, // Large batch writes
+            autoTune: false, // Fixed configuration
+            writeOnly: true // Enable write-only optimizations
+        };
+    }
+}
+/**
+ * Hybrid mode that can both read and write
+ */
+export class HybridMode extends BaseOperationalMode {
+    constructor() {
+        super(...arguments);
+        this.canRead = true;
+        this.canWrite = true;
+        this.canDelete = true;
+        this.cacheStrategy = {
+            hotCacheRatio: 0.5, // Balanced cache/buffer allocation
+            prefetchAggressive: false, // Moderate prefetching
+            ttl: 600000, // 10 minute TTL
+            compressionEnabled: true, // Compress when beneficial
+            writeBufferSize: 5000, // Moderate write buffer
+            batchWrites: true, // Batch writes when possible
+            adaptive: true // Adapt to workload mix
+        };
+        this.readWriteRatio = 0.5; // Track read/write ratio
+    }
+    /**
+     * Get balanced cache configuration
+     */
+    getCacheConfig() {
+        return {
+            hotCacheMaxSize: 500000, // Medium cache size
+            hotCacheEvictionThreshold: 0.7, // Balanced eviction
+            warmCacheTTL: 600000, // 10 minute warm cache
+            batchSize: 500, // Medium batch size
+            autoTune: true, // Auto-tune based on workload
+            autoTuneInterval: 300000 // Tune every 5 minutes
+        };
+    }
+    /**
+     * Update cache strategy based on workload
+     * @param readCount - Number of recent reads
+     * @param writeCount - Number of recent writes
+     */
+    updateWorkloadBalance(readCount, writeCount) {
+        const total = readCount + writeCount;
+        if (total === 0)
+            return;
+        this.readWriteRatio = readCount / total;
+        // Adjust cache strategy based on workload
+        if (this.readWriteRatio > 0.8) {
+            // Read-heavy workload
+            this.cacheStrategy.hotCacheRatio = 0.7;
+            this.cacheStrategy.prefetchAggressive = true;
+            this.cacheStrategy.writeBufferSize = 2000;
+        }
+        else if (this.readWriteRatio < 0.2) {
+            // Write-heavy workload
+            this.cacheStrategy.hotCacheRatio = 0.3;
+            this.cacheStrategy.prefetchAggressive = false;
+            this.cacheStrategy.writeBufferSize = 8000;
+        }
+        else {
+            // Balanced workload
+            this.cacheStrategy.hotCacheRatio = 0.5;
+            this.cacheStrategy.prefetchAggressive = false;
+            this.cacheStrategy.writeBufferSize = 5000;
+        }
+    }
+}
+/**
+ * Factory for creating operational modes
+ */
+export class OperationalModeFactory {
+    /**
+     * Create operational mode based on role
+     * @param role - The instance role
+     * @returns The appropriate operational mode
+     */
+    static createMode(role) {
+        switch (role) {
+            case 'reader':
+                return new ReaderMode();
+            case 'writer':
+                return new WriterMode();
+            case 'hybrid':
+                return new HybridMode();
+            default:
+                // Default to reader for safety
+                return new ReaderMode();
+        }
+    }
+    /**
+     * Create mode with custom cache strategy
+     * @param role - The instance role
+     * @param customStrategy - Custom cache strategy overrides
+     * @returns The operational mode with custom strategy
+     */
+    static createModeWithStrategy(role, customStrategy) {
+        const mode = this.createMode(role);
+        // Apply custom strategy overrides
+        mode.cacheStrategy = {
+            ...mode.cacheStrategy,
+            ...customStrategy
+        };
+        return mode;
+    }
+}
+//# sourceMappingURL=operationalModes.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/operationalModes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operationalModes.js","sourceRoot":"","sources":["../../src/distributed/operationalModes.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH;;GAEG;AACH,MAAM,OAAgB,mBAAmB;IAMvC;;OAEG;IACH,iBAAiB,CAAC,SAAsC;QACtD,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,MAAM;gBACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;gBACvE,CAAC;gBACD,MAAK;YACP,KAAK,OAAO;gBACV,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;gBACvE,CAAC;gBACD,MAAK;YACP,KAAK,QAAQ;gBACX,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBACpB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;gBACnE,CAAC;gBACD,MAAK;QACT,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,mBAAmB;IAAnD;;QACE,YAAO,GAAG,IAAI,CAAA;QACd,aAAQ,GAAG,KAAK,CAAA;QAChB,cAAS,GAAG,KAAK,CAAA;QAEjB,kBAAa,GAAkB;YAC7B,aAAa,EAAE,GAAG,EAAY,+BAA+B;YAC7D,kBAAkB,EAAE,IAAI,EAAO,wCAAwC;YACvE,GAAG,EAAE,OAAO,EAAkB,mBAAmB;YACjD,kBAAkB,EAAE,IAAI,EAAO,oCAAoC;YACnE,eAAe,EAAE,CAAC,EAAY,yBAAyB;YACvD,WAAW,EAAE,KAAK,EAAY,YAAY;YAC1C,QAAQ,EAAE,IAAI,CAAgB,0BAA0B;SACzD,CAAA;IAgBH,CAAC;IAdC;;OAEG;IACH,cAAc;QACZ,OAAO;YACL,eAAe,EAAE,OAAO,EAAO,kBAAkB;YACjD,yBAAyB,EAAE,GAAG,EAAE,kBAAkB;YAClD,YAAY,EAAE,OAAO,EAAU,oBAAoB;YACnD,SAAS,EAAE,GAAG,EAAiB,oBAAoB;YACnD,QAAQ,EAAE,IAAI,EAAiB,8BAA8B;YAC7D,gBAAgB,EAAE,KAAK,EAAQ,oBAAoB;YACnD,QAAQ,EAAE,IAAI,CAAiB,iCAAiC;SACjE,CAAA;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,mBAAmB;IAAnD;;QACE,YAAO,GAAG,KAAK,CAAA;QACf,aAAQ,GAAG,IAAI,CAAA;QACf,cAAS,GAAG,IAAI,CAAA;QAEhB,kBAAa,GAAkB;YAC7B,aAAa,EAAE,GAAG,EAAY,4CAA4C;YAC1E,kBAAkB,EAAE,KAAK,EAAM,wBAAwB;YACvD,GAAG,EAAE,KAAK,EAAoB,uBAAuB;YACrD,kBAAkB,EAAE,KAAK,EAAM,+BAA+B;YAC9D,eAAe,EAAE,KAAK,EAAQ,kCAAkC;YAChE,WAAW,EAAE,IAAI,EAAa,wBAAwB;YACtD,QAAQ,EAAE,KAAK,CAAe,uCAAuC;SACtE,CAAA;IAeH,CAAC;IAbC;;OAEG;IACH,cAAc;QACZ,OAAO;YACL,eAAe,EAAE,MAAM,EAAS,kBAAkB;YAClD,yBAAyB,EAAE,GAAG,EAAE,sBAAsB;YACtD,YAAY,EAAE,KAAK,EAAY,sBAAsB;YACrD,SAAS,EAAE,IAAI,EAAgB,qBAAqB;YACpD,QAAQ,EAAE,KAAK,EAAgB,sBAAsB;YACrD,SAAS,EAAE,IAAI,CAAgB,kCAAkC;SAClE,CAAA;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,UAAW,SAAQ,mBAAmB;IAAnD;;QACE,YAAO,GAAG,IAAI,CAAA;QACd,aAAQ,GAAG,IAAI,CAAA;QACf,cAAS,GAAG,IAAI,CAAA;QAEhB,kBAAa,GAAkB;YAC7B,aAAa,EAAE,GAAG,EAAY,mCAAmC;YACjE,kBAAkB,EAAE,KAAK,EAAM,uBAAuB;YACtD,GAAG,EAAE,MAAM,EAAmB,gBAAgB;YAC9C,kBAAkB,EAAE,IAAI,EAAO,2BAA2B;YAC1D,eAAe,EAAE,IAAI,EAAS,wBAAwB;YACtD,WAAW,EAAE,IAAI,EAAa,6BAA6B;YAC3D,QAAQ,EAAE,IAAI,CAAgB,wBAAwB;SACvD,CAAA;QAEO,mBAAc,GAAW,GAAG,CAAA,CAAC,yBAAyB;IA6ChE,CAAC;IA3CC;;OAEG;IACH,cAAc;QACZ,OAAO;YACL,eAAe,EAAE,MAAM,EAAS,oBAAoB;YACpD,yBAAyB,EAAE,GAAG,EAAE,oBAAoB;YACpD,YAAY,EAAE,MAAM,EAAW,uBAAuB;YACtD,SAAS,EAAE,GAAG,EAAiB,oBAAoB;YACnD,QAAQ,EAAE,IAAI,EAAiB,8BAA8B;YAC7D,gBAAgB,EAAE,MAAM,CAAO,uBAAuB;SACvD,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,qBAAqB,CAAC,SAAiB,EAAE,UAAkB;QACzD,MAAM,KAAK,GAAG,SAAS,GAAG,UAAU,CAAA;QACpC,IAAI,KAAK,KAAK,CAAC;YAAE,OAAM;QAEvB,IAAI,CAAC,cAAc,GAAG,SAAS,GAAG,KAAK,CAAA;QAEvC,0CAA0C;QAC1C,IAAI,IAAI,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC;YAC9B,sBAAsB;YACtB,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,GAAG,CAAA;YACtC,IAAI,CAAC,aAAa,CAAC,kBAAkB,GAAG,IAAI,CAAA;YAC5C,IAAI,CAAC,aAAa,CAAC,eAAe,GAAG,IAAI,CAAA;QAC3C,CAAC;aAAM,IAAI,IAAI,CAAC,cAAc,GAAG,GAAG,EAAE,CAAC;YACrC,uBAAuB;YACvB,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,GAAG,CAAA;YACtC,IAAI,CAAC,aAAa,CAAC,kBAAkB,GAAG,KAAK,CAAA;YAC7C,IAAI,CAAC,aAAa,CAAC,eAAe,GAAG,IAAI,CAAA;QAC3C,CAAC;aAAM,CAAC;YACN,oBAAoB;YACpB,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,GAAG,CAAA;YACtC,IAAI,CAAC,aAAa,CAAC,kBAAkB,GAAG,KAAK,CAAA;YAC7C,IAAI,CAAC,aAAa,CAAC,eAAe,GAAG,IAAI,CAAA;QAC3C,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,sBAAsB;IACjC;;;;OAIG;IACH,MAAM,CAAC,UAAU,CAAC,IAAkB;QAClC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,QAAQ;gBACX,OAAO,IAAI,UAAU,EAAE,CAAA;YACzB,KAAK,QAAQ;gBACX,OAAO,IAAI,UAAU,EAAE,CAAA;YACzB,KAAK,QAAQ;gBACX,OAAO,IAAI,UAAU,EAAE,CAAA;YACzB;gBACE,+BAA+B;gBAC/B,OAAO,IAAI,UAAU,EAAE,CAAA;QAC3B,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,sBAAsB,CAC3B,IAAkB,EAClB,cAAsC;QAEtC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,kCAAkC;QAClC,IAAI,CAAC,aAAa,GAAG;YACnB,GAAG,IAAI,CAAC,aAAa;YACrB,GAAG,cAAc;SAClB,CAAA;QAED,OAAO,IAAI,CAAA;IACb,CAAC;CACF"}

.recovery-workspace/dist-backup-20250910-141917/distributed/queryPlanner.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Distributed Query Planner for Brainy 3.0
+ *
+ * Intelligently plans and executes distributed queries across shards
+ * Optimizes for data locality, parallelism, and network efficiency
+ */
+import type { StorageAdapter } from '../coreTypes.js';
+import type { DistributedCoordinator } from './coordinator.js';
+import type { ShardManager } from './shardManager.js';
+import type { HTTPTransport } from './httpTransport.js';
+import type { TripleIntelligenceEngine } from '../triple/TripleIntelligence.js';
+export interface QueryPlan {
+    /**
+     * Shards that need to be queried
+     */
+    shards: string[];
+    /**
+     * Nodes responsible for each shard
+     */
+    nodeAssignments: Map<string, string[]>;
+    /**
+     * Whether query can be parallelized
+     */
+    parallel: boolean;
+    /**
+     * Estimated cost (0-1000)
+     */
+    cost: number;
+    /**
+     * Query strategy
+     */
+    strategy: 'broadcast' | 'targeted' | 'scatter-gather' | 'local-only';
+}
+export interface DistributedQueryResult {
+    results: any[];
+    totalCount: number;
+    executionTime: number;
+    nodeStats: Map<string, {
+        resultsReturned: number;
+        executionTime: number;
+        errors?: string[];
+    }>;
+}
+export declare class DistributedQueryPlanner {
+    private nodeId;
+    private coordinator;
+    private shardManager;
+    private transport;
+    private tripleEngine?;
+    private storage;
+    constructor(nodeId: string, coordinator: DistributedCoordinator, shardManager: ShardManager, transport: HTTPTransport, storage: StorageAdapter, tripleEngine?: TripleIntelligenceEngine);
+    /**
+     * Plan a distributed query
+     */
+    planQuery(query: any): Promise<QueryPlan>;
+    /**
+     * Execute a distributed query based on plan
+     */
+    executeQuery(query: any, plan: QueryPlan): Promise<DistributedQueryResult>;
+    /**
+     * Execute query on local shards
+     */
+    private executeLocalQuery;
+    /**
+     * Execute query on remote node
+     */
+    private executeRemoteQuery;
+    /**
+     * Get data from a specific shard
+     */
+    private getShardData;
+    /**
+     * Filter data based on query criteria
+     */
+    private filterData;
+    /**
+     * Check if item matches filter
+     */
+    private matchesFilter;
+    /**
+     * Merge results from multiple nodes using Triple Intelligence
+     */
+    private mergeResults;
+    /**
+     * Simple deduplication of results
+     */
+    private deduplicateResults;
+    /**
+     * Get unique key for result
+     */
+    private getResultKey;
+    /**
+     * Determine query type
+     */
+    private getQueryType;
+    /**
+     * Determine which shards are affected by query
+     */
+    private determineAffectedShards;
+    /**
+     * Optimize query plan based on statistics
+     */
+    optimizePlan(plan: QueryPlan): Promise<QueryPlan>;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/queryPlanner.js

@@ -0,0 +1,327 @@
+/**
+ * Distributed Query Planner for Brainy 3.0
+ *
+ * Intelligently plans and executes distributed queries across shards
+ * Optimizes for data locality, parallelism, and network efficiency
+ */
+export class DistributedQueryPlanner {
+    constructor(nodeId, coordinator, shardManager, transport, storage, tripleEngine) {
+        this.nodeId = nodeId;
+        this.coordinator = coordinator;
+        this.shardManager = shardManager;
+        this.transport = transport;
+        this.storage = storage;
+        this.tripleEngine = tripleEngine;
+    }
+    /**
+     * Plan a distributed query
+     */
+    async planQuery(query) {
+        // Determine query type and scope
+        const queryType = this.getQueryType(query);
+        const affectedShards = await this.determineAffectedShards(query);
+        // Get current shard assignments
+        const assignments = new Map();
+        for (const shardId of affectedShards) {
+            const nodes = await this.shardManager.getNodesForShard(shardId);
+            assignments.set(shardId, nodes);
+        }
+        // Determine strategy based on query characteristics
+        let strategy = 'scatter-gather';
+        let cost = 0;
+        if (affectedShards.length === 0) {
+            // Local only query
+            strategy = 'local-only';
+            cost = 1;
+        }
+        else if (affectedShards.length === this.shardManager.getTotalShards()) {
+            // Full table scan
+            strategy = 'broadcast';
+            cost = 1000;
+        }
+        else if (affectedShards.length <= 3) {
+            // Targeted query
+            strategy = 'targeted';
+            cost = affectedShards.length * 10;
+        }
+        else {
+            // Scatter-gather for medium queries
+            strategy = 'scatter-gather';
+            cost = affectedShards.length * 50;
+        }
+        // Add network cost
+        const remoteShards = affectedShards.filter(shardId => {
+            const nodes = assignments.get(shardId) || [];
+            return !nodes.includes(this.nodeId);
+        });
+        cost += remoteShards.length * 20;
+        return {
+            shards: affectedShards,
+            nodeAssignments: assignments,
+            parallel: affectedShards.length > 1,
+            cost,
+            strategy
+        };
+    }
+    /**
+     * Execute a distributed query based on plan
+     */
+    async executeQuery(query, plan) {
+        const startTime = Date.now();
+        const nodeStats = new Map();
+        // Group shards by node for efficient batching
+        const nodeToShards = new Map();
+        for (const [shardId, nodes] of plan.nodeAssignments) {
+            // Pick the first available node (could be optimized)
+            const targetNode = nodes[0];
+            if (!nodeToShards.has(targetNode)) {
+                nodeToShards.set(targetNode, []);
+            }
+            nodeToShards.get(targetNode).push(shardId);
+        }
+        // Execute queries in parallel
+        const promises = [];
+        for (const [targetNode, shards] of nodeToShards) {
+            if (targetNode === this.nodeId) {
+                // Local execution
+                promises.push(this.executeLocalQuery(query, shards));
+            }
+            else {
+                // Remote execution
+                promises.push(this.executeRemoteQuery(targetNode, query, shards));
+            }
+        }
+        // Wait for all results
+        const results = await Promise.allSettled(promises);
+        // Aggregate results
+        const allResults = [];
+        let totalCount = 0;
+        let nodeIndex = 0;
+        for (const [targetNode, shards] of nodeToShards) {
+            const result = results[nodeIndex];
+            const nodeTime = Date.now() - startTime;
+            if (result.status === 'fulfilled') {
+                const nodeResult = result.value;
+                allResults.push(...(nodeResult.results || []));
+                totalCount += nodeResult.count || 0;
+                nodeStats.set(targetNode, {
+                    resultsReturned: nodeResult.count || 0,
+                    executionTime: nodeTime,
+                    shards
+                });
+            }
+            else {
+                nodeStats.set(targetNode, {
+                    resultsReturned: 0,
+                    executionTime: nodeTime,
+                    errors: [result.reason?.message || 'Unknown error'],
+                    shards
+                });
+            }
+            nodeIndex++;
+        }
+        // Merge and rank results using Triple Intelligence if available
+        const mergedResults = await this.mergeResults(allResults, query);
+        return {
+            results: mergedResults,
+            totalCount,
+            executionTime: Date.now() - startTime,
+            nodeStats
+        };
+    }
+    /**
+     * Execute query on local shards
+     */
+    async executeLocalQuery(query, shards) {
+        const results = [];
+        for (const shardId of shards) {
+            // Get data from storage for this shard
+            const shardData = await this.getShardData(shardId, query);
+            results.push(...shardData);
+        }
+        return {
+            results,
+            count: results.length
+        };
+    }
+    /**
+     * Execute query on remote node
+     */
+    async executeRemoteQuery(targetNode, query, shards) {
+        try {
+            const response = await this.transport.call(targetNode, 'query', {
+                query,
+                shards
+            });
+            return response || { results: [], count: 0 };
+        }
+        catch (error) {
+            console.error(`Failed to query node ${targetNode}:`, error);
+            throw error;
+        }
+    }
+    /**
+     * Get data from a specific shard
+     */
+    async getShardData(shardId, query) {
+        // This would interact with the actual storage adapter
+        // For now, return empty array since storage adapters don't have direct shard access
+        // In a real implementation, this would use storage-specific methods
+        try {
+            // Would need to implement shard-aware storage methods
+            // For now, return empty to allow compilation
+            return [];
+        }
+        catch (error) {
+            console.error(`Failed to get shard ${shardId} data:`, error);
+            return [];
+        }
+    }
+    /**
+     * Filter data based on query criteria
+     */
+    filterData(data, query) {
+        if (!Array.isArray(data))
+            return [];
+        // Apply query filters
+        let filtered = data;
+        if (query.filter) {
+            filtered = filtered.filter((item) => {
+                // Apply filter logic
+                return this.matchesFilter(item, query.filter);
+            });
+        }
+        if (query.limit) {
+            filtered = filtered.slice(0, query.limit);
+        }
+        return filtered;
+    }
+    /**
+     * Check if item matches filter
+     */
+    matchesFilter(item, filter) {
+        // Simple filter matching
+        for (const [key, value] of Object.entries(filter)) {
+            if (item[key] !== value) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Merge results from multiple nodes using Triple Intelligence
+     */
+    async mergeResults(results, query) {
+        if (!this.tripleEngine) {
+            // Simple merge without Triple Intelligence
+            return this.deduplicateResults(results);
+        }
+        // Use Triple Intelligence for intelligent merging
+        // Merge results by combining scores and maintaining order
+        const mergedMap = new Map();
+        for (const result of results) {
+            const id = result.id || result.entity?.id;
+            if (!id)
+                continue;
+            if (mergedMap.has(id)) {
+                // Merge duplicate results by averaging scores
+                const existing = mergedMap.get(id);
+                existing.score = (existing.score + (result.score || 0)) / 2;
+                // Preserve highest confidence metadata
+                if (result.confidence > existing.confidence) {
+                    existing.metadata = { ...existing.metadata, ...result.metadata };
+                }
+            }
+            else {
+                mergedMap.set(id, { ...result });
+            }
+        }
+        // Sort by score and return
+        return Array.from(mergedMap.values())
+            .sort((a, b) => (b.score || 0) - (a.score || 0));
+    }
+    /**
+     * Simple deduplication of results
+     */
+    deduplicateResults(results) {
+        const seen = new Set();
+        const deduplicated = [];
+        for (const result of results) {
+            const key = this.getResultKey(result);
+            if (!seen.has(key)) {
+                seen.add(key);
+                deduplicated.push(result);
+            }
+        }
+        return deduplicated;
+    }
+    /**
+     * Get unique key for result
+     */
+    getResultKey(result) {
+        if (result.id)
+            return result.id;
+        if (result.uuid)
+            return result.uuid;
+        return JSON.stringify(result);
+    }
+    /**
+     * Determine query type
+     */
+    getQueryType(query) {
+        if (query.vector)
+            return 'vector';
+        if (query.triple)
+            return 'triple';
+        if (query.filter)
+            return 'filter';
+        return 'scan';
+    }
+    /**
+     * Determine which shards are affected by query
+     */
+    async determineAffectedShards(query) {
+        const totalShards = this.shardManager.getTotalShards();
+        const affectedShards = [];
+        // If query has specific entity/key, determine shard
+        if (query.entity || query.key) {
+            const key = query.entity || query.key;
+            const assignment = this.shardManager.getShardForKey(key);
+            if (assignment) {
+                return [assignment.shardId];
+            }
+        }
+        // If query has partition hint
+        if (query.partition) {
+            return [query.partition];
+        }
+        // Otherwise, query all shards (broadcast)
+        for (let i = 0; i < totalShards; i++) {
+            affectedShards.push(`shard-${i}`);
+        }
+        return affectedShards;
+    }
+    /**
+     * Optimize query plan based on statistics
+     */
+    async optimizePlan(plan) {
+        // Get node health and latency stats
+        const nodeHealth = await this.coordinator.getHealth();
+        // Re-assign shards to healthier nodes if needed
+        const optimizedAssignments = new Map();
+        for (const [shardId, nodes] of plan.nodeAssignments) {
+            // Sort nodes by health score (simple heuristic)
+            const sortedNodes = nodes.sort((a, b) => {
+                // Higher health score = better node
+                // For now, use a simple approach
+                return 0;
+            });
+            optimizedAssignments.set(shardId, sortedNodes);
+        }
+        return {
+            ...plan,
+            nodeAssignments: optimizedAssignments
+        };
+    }
+}
+//# sourceMappingURL=queryPlanner.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/readWriteSeparation.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Read/Write Separation for Distributed Scaling
+ * Implements primary-replica architecture for scalable reads
+ */
+import { EventEmitter } from 'events';
+import { DistributedCoordinator } from './coordinator.js';
+import { ShardManager } from './shardManager.js';
+import { CacheSync } from './cacheSync.js';
+export interface ReplicationConfig {
+    nodeId: string;
+    role?: 'primary' | 'replica' | 'auto';
+    primaryUrl?: string;
+    replicaUrls?: string[];
+    syncInterval?: number;
+    readPreference?: 'primary' | 'replica' | 'nearest';
+    consistencyLevel?: 'eventual' | 'strong' | 'bounded';
+    maxStaleness?: number;
+}
+export interface WriteOperation {
+    id: string;
+    type: 'add' | 'update' | 'delete';
+    data: any;
+    timestamp: number;
+    version: number;
+}
+export interface ReplicationLog {
+    operations: WriteOperation[];
+    lastSequence: number;
+    primaryVersion: number;
+}
+/**
+ * Read/Write Separation Manager
+ */
+export declare class ReadWriteSeparation extends EventEmitter {
+    private nodeId;
+    private role;
+    private coordinator;
+    private cacheSync;
+    private replicationLog;
+    private replicas;
+    private primaryConnection?;
+    private config;
+    private syncTimer?;
+    private isRunning;
+    constructor(config: ReplicationConfig, coordinator: DistributedCoordinator, _shardManager: ShardManager, cacheSync: CacheSync);
+    /**
+     * Start read/write separation
+     */
+    start(): Promise<void>;
+    /**
+     * Stop read/write separation
+     */
+    stop(): Promise<void>;
+    /**
+     * Execute a write operation (primary only)
+     */
+    write(operation: Omit<WriteOperation, 'id' | 'timestamp' | 'version'>): Promise<string>;
+    /**
+     * Execute a read operation
+     */
+    read(key: string, options?: {
+        consistency?: 'eventual' | 'strong';
+    }): Promise<any>;
+    /**
+     * Get replication lag (replica only)
+     */
+    getReplicationLag(): number;
+    /**
+     * Setup connections based on role
+     */
+    private setupConnections;
+    /**
+     * Start as primary node
+     */
+    private startAsPrimary;
+    /**
+     * Start as replica node
+     */
+    private startAsReplica;
+    /**
+     * Sync replicas (primary only)
+     */
+    private syncReplicas;
+    /**
+     * Sync from primary (replica only)
+     */
+    private syncFromPrimary;
+    /**
+     * Apply a replicated operation
+     */
+    private applyOperation;
+    /**
+     * Propagate operation to replicas
+     */
+    private propagateToReplicas;
+    /**
+     * Determine role automatically
+     */
+    private determineRole;
+    /**
+     * Read from local storage
+     */
+    private readLocal;
+    /**
+     * Generate unique operation ID
+     */
+    private generateOperationId;
+    /**
+     * Get replication statistics
+     */
+    getStats(): {
+        role: string;
+        replicas: number;
+        replicationLag: number;
+        operationsInLog: number;
+        primaryVersion: number;
+    };
+    /**
+     * Check if node can accept writes
+     */
+    canWrite(): boolean;
+    /**
+     * Check if node can serve reads
+     */
+    canRead(): boolean;
+    /**
+     * Set whether this node is primary (for leader election integration)
+     */
+    setPrimary(isPrimary: boolean): void;
+}
+/**
+ * Create read/write separation manager
+ */
+export declare function createReadWriteSeparation(config: ReplicationConfig, coordinator: DistributedCoordinator, shardManager: ShardManager, cacheSync: CacheSync): ReadWriteSeparation;

.recovery-workspace/dist-backup-20250910-141917/distributed/readWriteSeparation.js

@@ -0,0 +1,350 @@
+/**
+ * Read/Write Separation for Distributed Scaling
+ * Implements primary-replica architecture for scalable reads
+ */
+import { EventEmitter } from 'events';
+/**
+ * Read/Write Separation Manager
+ */
+export class ReadWriteSeparation extends EventEmitter {
+    constructor(config, coordinator, _shardManager, cacheSync) {
+        super();
+        this.replicas = new Map();
+        this.isRunning = false;
+        this.config = config;
+        this.nodeId = config.nodeId;
+        this.role = config.role === 'auto' ? this.determineRole() : (config.role || 'replica');
+        this.coordinator = coordinator;
+        this.cacheSync = cacheSync;
+        this.replicationLog = {
+            operations: [],
+            lastSequence: 0,
+            primaryVersion: 0
+        };
+        // Setup connections based on role
+        this.setupConnections();
+    }
+    /**
+     * Start read/write separation
+     */
+    async start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        // Start components
+        await this.coordinator.start();
+        // Setup role-specific behavior
+        if (this.role === 'primary') {
+            this.startAsPrimary();
+        }
+        else {
+            this.startAsReplica();
+        }
+        this.emit('started', { nodeId: this.nodeId, role: this.role });
+    }
+    /**
+     * Stop read/write separation
+     */
+    async stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.syncTimer) {
+            clearInterval(this.syncTimer);
+            this.syncTimer = undefined;
+        }
+        await this.coordinator.stop();
+        this.emit('stopped', { nodeId: this.nodeId });
+    }
+    /**
+     * Execute a write operation (primary only)
+     */
+    async write(operation) {
+        if (this.role !== 'primary') {
+            // Forward to primary
+            if (this.primaryConnection) {
+                return this.primaryConnection.forwardWrite(operation);
+            }
+            throw new Error('Cannot write: not connected to primary');
+        }
+        // Generate operation metadata
+        const writeOp = {
+            id: this.generateOperationId(),
+            ...operation,
+            timestamp: Date.now(),
+            version: ++this.replicationLog.primaryVersion
+        };
+        // Add to replication log
+        this.replicationLog.operations.push(writeOp);
+        this.replicationLog.lastSequence++;
+        // Propagate to replicas
+        this.propagateToReplicas(writeOp);
+        // Update cache
+        if (operation.type !== 'delete') {
+            this.cacheSync.set(writeOp.id, operation.data);
+        }
+        else {
+            this.cacheSync.delete(writeOp.id);
+        }
+        this.emit('write', writeOp);
+        return writeOp.id;
+    }
+    /**
+     * Execute a read operation
+     */
+    async read(key, options) {
+        const consistency = options?.consistency || this.config.consistencyLevel || 'eventual';
+        if (consistency === 'strong' && this.role === 'replica') {
+            // For strong consistency, read from primary
+            if (this.primaryConnection) {
+                return this.primaryConnection.read(key);
+            }
+            throw new Error('Cannot guarantee strong consistency: not connected to primary');
+        }
+        // Check cache first
+        const cached = this.cacheSync.get(key);
+        if (cached !== undefined) {
+            return cached;
+        }
+        // Read from appropriate source based on preference
+        if (this.config.readPreference === 'primary' && this.primaryConnection) {
+            return this.primaryConnection.read(key);
+        }
+        // Read locally (replica or primary)
+        return this.readLocal(key);
+    }
+    /**
+     * Get replication lag (replica only)
+     */
+    getReplicationLag() {
+        if (this.role === 'primary')
+            return 0;
+        if (this.primaryConnection) {
+            return Date.now() - this.primaryConnection.lastSync;
+        }
+        return -1; // Unknown
+    }
+    /**
+     * Setup connections based on role
+     */
+    setupConnections() {
+        if (this.role === 'primary') {
+            // Setup replica connections
+            if (this.config.replicaUrls) {
+                for (const url of this.config.replicaUrls) {
+                    const replica = new ReplicaConnection(url);
+                    this.replicas.set(url, replica);
+                }
+            }
+        }
+        else {
+            // Setup primary connection
+            if (this.config.primaryUrl) {
+                this.primaryConnection = new PrimaryConnection(this.config.primaryUrl);
+            }
+        }
+    }
+    /**
+     * Start as primary node
+     */
+    startAsPrimary() {
+        // Start accepting writes
+        this.emit('roleEstablished', { role: 'primary' });
+        // Start replication timer
+        this.syncTimer = setInterval(() => {
+            this.syncReplicas();
+        }, this.config.syncInterval || 1000);
+    }
+    /**
+     * Start as replica node
+     */
+    startAsReplica() {
+        // Start syncing from primary
+        this.emit('roleEstablished', { role: 'replica' });
+        // Start sync timer
+        this.syncTimer = setInterval(() => {
+            this.syncFromPrimary();
+        }, this.config.syncInterval || 1000);
+    }
+    /**
+     * Sync replicas (primary only)
+     */
+    async syncReplicas() {
+        const batch = this.replicationLog.operations.slice(-100); // Last 100 ops
+        for (const [url, replica] of this.replicas) {
+            try {
+                await replica.sync(batch, this.replicationLog.primaryVersion);
+            }
+            catch (error) {
+                this.emit('replicaSyncError', { url, error });
+            }
+        }
+    }
+    /**
+     * Sync from primary (replica only)
+     */
+    async syncFromPrimary() {
+        if (!this.primaryConnection)
+            return;
+        try {
+            const updates = await this.primaryConnection.getUpdates(this.replicationLog.lastSequence);
+            // Apply updates
+            for (const op of updates) {
+                await this.applyOperation(op);
+            }
+            this.emit('synced', { operations: updates.length });
+        }
+        catch (error) {
+            this.emit('primarySyncError', { error });
+        }
+    }
+    /**
+     * Apply a replicated operation
+     */
+    async applyOperation(op) {
+        // Update local state
+        this.replicationLog.operations.push(op);
+        this.replicationLog.lastSequence = Math.max(this.replicationLog.lastSequence, op.version);
+        // Update cache
+        switch (op.type) {
+            case 'add':
+            case 'update':
+                this.cacheSync.set(op.id, op.data);
+                break;
+            case 'delete':
+                this.cacheSync.delete(op.id);
+                break;
+        }
+        this.emit('operationApplied', op);
+    }
+    /**
+     * Propagate operation to replicas
+     */
+    propagateToReplicas(op) {
+        for (const replica of this.replicas.values()) {
+            replica.sendOperation(op).catch(error => {
+                this.emit('replicationError', { replica, error });
+            });
+        }
+    }
+    /**
+     * Determine role automatically
+     */
+    determineRole() {
+        // Use coordinator's leader election
+        return this.coordinator.isLeader() ? 'primary' : 'replica';
+    }
+    /**
+     * Read from local storage
+     */
+    async readLocal(key) {
+        // This would connect to actual storage
+        // For now, return from cache or undefined
+        return this.cacheSync.get(key);
+    }
+    /**
+     * Generate unique operation ID
+     */
+    generateOperationId() {
+        return `${this.nodeId}-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+    }
+    /**
+     * Get replication statistics
+     */
+    getStats() {
+        return {
+            role: this.role,
+            replicas: this.replicas.size,
+            replicationLag: this.getReplicationLag(),
+            operationsInLog: this.replicationLog.operations.length,
+            primaryVersion: this.replicationLog.primaryVersion
+        };
+    }
+    /**
+     * Check if node can accept writes
+     */
+    canWrite() {
+        return this.role === 'primary';
+    }
+    /**
+     * Check if node can serve reads
+     */
+    canRead() {
+        if (this.config.consistencyLevel === 'strong') {
+            return this.role === 'primary' || this.primaryConnection !== undefined;
+        }
+        return true;
+    }
+    /**
+     * Set whether this node is primary (for leader election integration)
+     */
+    setPrimary(isPrimary) {
+        const newRole = isPrimary ? 'primary' : 'replica';
+        if (this.role !== newRole) {
+            this.role = newRole;
+            this.emit('roleChange', { oldRole: this.role, newRole });
+            if (isPrimary) {
+                // Became primary - stop syncing from old primary
+                this.primaryConnection = undefined;
+            }
+            else {
+                // Became replica - connect to new primary if URL is known
+                if (this.config.primaryUrl) {
+                    this.primaryConnection = new PrimaryConnection(this.config.primaryUrl);
+                }
+            }
+        }
+    }
+}
+/**
+ * Connection to a replica (used by primary)
+ */
+class ReplicaConnection {
+    constructor(url) {
+        this.url = url;
+        // Store URL for connection
+        void this.url;
+    }
+    async sync(_operations, _version) {
+        // In real implementation, this would use HTTP/gRPC to this.url
+        // For now, simulate network call
+        await new Promise(resolve => setTimeout(resolve, 10));
+    }
+    async sendOperation(_op) {
+        // Send single operation to replica at this.url
+        await new Promise(resolve => setTimeout(resolve, 5));
+    }
+}
+/**
+ * Connection to primary (used by replicas)
+ */
+class PrimaryConnection {
+    constructor(url) {
+        this.url = url;
+        this.lastSync = Date.now();
+        // Store URL for connection
+        void this.url;
+    }
+    async getUpdates(_fromSequence) {
+        // In real implementation, fetch from primary at this.url
+        this.lastSync = Date.now();
+        return [];
+    }
+    async forwardWrite(_operation) {
+        // Forward write to primary at this.url
+        await new Promise(resolve => setTimeout(resolve, 20));
+        return `forwarded-${Date.now()}`;
+    }
+    async read(_key) {
+        // Read from primary at this.url for strong consistency
+        await new Promise(resolve => setTimeout(resolve, 10));
+        return undefined;
+    }
+}
+/**
+ * Create read/write separation manager
+ */
+export function createReadWriteSeparation(config, coordinator, shardManager, cacheSync) {
+    return new ReadWriteSeparation(config, coordinator, shardManager, cacheSync);
+}
+//# sourceMappingURL=readWriteSeparation.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/shardManager.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Shard Manager for Horizontal Scaling
+ * Implements consistent hashing for data distribution across shards
+ */
+import { EventEmitter } from 'events';
+export interface ShardConfig {
+    shardCount?: number;
+    replicationFactor?: number;
+    virtualNodes?: number;
+    autoRebalance?: boolean;
+}
+export interface Shard {
+    id: string;
+    nodeId: string;
+    virtualNodes: string[];
+    itemCount: number;
+    sizeBytes: number;
+    status: 'active' | 'rebalancing' | 'offline';
+}
+export interface ShardAssignment {
+    shardId: string;
+    nodeId: string;
+    replicas: string[];
+}
+/**
+ * Shard Manager for distributing data across multiple nodes
+ */
+export declare class ShardManager extends EventEmitter {
+    private hashRing;
+    private shards;
+    private nodeToShards;
+    private shardCount;
+    private replicationFactor;
+    private autoRebalance;
+    constructor(config?: ShardConfig);
+    /**
+     * Initialize shard configuration
+     */
+    private initializeShards;
+    /**
+     * Add a node to the cluster
+     */
+    addNode(nodeId: string): void;
+    /**
+     * Remove a node from the cluster
+     */
+    removeNode(nodeId: string): void;
+    /**
+     * Get shard assignment for a key
+     */
+    getShardForKey(key: string): ShardAssignment | null;
+    /**
+     * Get nodes responsible for a shard
+     */
+    getNodesForShard(shardId: string): string[];
+    /**
+     * Get total number of shards
+     */
+    getTotalShards(): number;
+    /**
+     * Update shard assignment to a new node
+     */
+    updateShardAssignment(shardId: string, newNodeId: string): void;
+    /**
+     * Get shard ID for a key
+     */
+    private getShardId;
+    /**
+     * Rebalance shards across nodes
+     */
+    private rebalanceShards;
+    /**
+     * Get shard assignment for all shards
+     */
+    getShardAssignments(): ShardAssignment[];
+    /**
+     * Get shard statistics
+     */
+    getShardStats(): {
+        totalShards: number;
+        activeShards: number;
+        rebalancingShards: number;
+        offlineShards: number;
+        averageItemsPerShard: number;
+    };
+    /**
+     * Update shard metrics
+     */
+    updateShardMetrics(shardId: string, itemCount: number, sizeBytes: number): void;
+    /**
+     * Get replication nodes for a shard
+     */
+    getReplicationNodes(shardId: string): string[];
+    /**
+     * Check if rebalancing is needed
+     */
+    needsRebalancing(): boolean;
+    /**
+     * Get cluster health
+     */
+    getHealth(): {
+        healthy: boolean;
+        nodes: number;
+        shards: {
+            total: number;
+            active: number;
+            inactive: number;
+        };
+    };
+}
+/**
+ * Create a shard manager instance
+ */
+export declare function createShardManager(config?: ShardConfig): ShardManager;

.recovery-workspace/dist-backup-20250910-141917/distributed/shardManager.js

@@ -0,0 +1,357 @@
+/**
+ * Shard Manager for Horizontal Scaling
+ * Implements consistent hashing for data distribution across shards
+ */
+import { createHash } from 'crypto';
+import { EventEmitter } from 'events';
+/**
+ * Consistent Hash Ring for shard distribution
+ */
+class ConsistentHashRing {
+    constructor(virtualNodes = 150) {
+        this.ring = new Map();
+        this.sortedKeys = [];
+        this.virtualNodes = virtualNodes;
+    }
+    /**
+     * Add a node to the hash ring
+     */
+    addNode(nodeId) {
+        for (let i = 0; i < this.virtualNodes; i++) {
+            const virtualNodeId = `${nodeId}:${i}`;
+            const hash = this.hash(virtualNodeId);
+            this.ring.set(hash, nodeId);
+        }
+        this.updateSortedKeys();
+    }
+    /**
+     * Remove a node from the hash ring
+     */
+    removeNode(nodeId) {
+        const keysToRemove = [];
+        for (const [hash, node] of this.ring) {
+            if (node === nodeId) {
+                keysToRemove.push(hash);
+            }
+        }
+        for (const key of keysToRemove) {
+            this.ring.delete(key);
+        }
+        this.updateSortedKeys();
+    }
+    /**
+     * Get the node responsible for a given key
+     */
+    getNode(key) {
+        if (this.ring.size === 0)
+            return null;
+        const hash = this.hash(key);
+        // Find the first node with hash >= key hash
+        for (const nodeHash of this.sortedKeys) {
+            if (nodeHash >= hash) {
+                return this.ring.get(nodeHash) || null;
+            }
+        }
+        // Wrap around to the first node
+        return this.ring.get(this.sortedKeys[0]) || null;
+    }
+    /**
+     * Get N nodes for replication
+     */
+    getNodes(key, count) {
+        if (this.ring.size === 0)
+            return [];
+        const nodes = new Set();
+        const hash = this.hash(key);
+        // Start from the primary node position
+        let startIdx = 0;
+        for (let i = 0; i < this.sortedKeys.length; i++) {
+            if (this.sortedKeys[i] >= hash) {
+                startIdx = i;
+                break;
+            }
+        }
+        // Collect unique nodes
+        let idx = startIdx;
+        while (nodes.size < count && nodes.size < this.getUniqueNodeCount()) {
+            const nodeHash = this.sortedKeys[idx % this.sortedKeys.length];
+            const node = this.ring.get(nodeHash);
+            if (node) {
+                nodes.add(node);
+            }
+            idx++;
+        }
+        return Array.from(nodes);
+    }
+    /**
+     * Get unique node count
+     */
+    getUniqueNodeCount() {
+        return new Set(this.ring.values()).size;
+    }
+    /**
+     * Update sorted keys for efficient lookup
+     */
+    updateSortedKeys() {
+        this.sortedKeys = Array.from(this.ring.keys()).sort((a, b) => a - b);
+    }
+    /**
+     * Hash function for consistent hashing
+     */
+    hash(key) {
+        const hash = createHash('md5').update(key).digest();
+        return hash.readUInt32BE(0);
+    }
+    /**
+     * Get all nodes in the ring
+     */
+    getAllNodes() {
+        return Array.from(new Set(this.ring.values()));
+    }
+}
+/**
+ * Shard Manager for distributing data across multiple nodes
+ */
+export class ShardManager extends EventEmitter {
+    constructor(config = {}) {
+        super();
+        this.shards = new Map();
+        this.nodeToShards = new Map();
+        this.shardCount = config.shardCount || 64;
+        this.replicationFactor = config.replicationFactor || 3;
+        this.autoRebalance = config.autoRebalance ?? true;
+        this.hashRing = new ConsistentHashRing(config.virtualNodes || 150);
+        // Initialize shards
+        this.initializeShards();
+    }
+    /**
+     * Initialize shard configuration
+     */
+    initializeShards() {
+        for (let i = 0; i < this.shardCount; i++) {
+            const shardId = `shard-${i.toString().padStart(3, '0')}`;
+            this.shards.set(shardId, {
+                id: shardId,
+                nodeId: '',
+                virtualNodes: [],
+                itemCount: 0,
+                sizeBytes: 0,
+                status: 'offline'
+            });
+        }
+    }
+    /**
+     * Add a node to the cluster
+     */
+    addNode(nodeId) {
+        this.hashRing.addNode(nodeId);
+        this.nodeToShards.set(nodeId, new Set());
+        // Assign shards to the new node
+        this.rebalanceShards();
+        this.emit('nodeAdded', { nodeId });
+    }
+    /**
+     * Remove a node from the cluster
+     */
+    removeNode(nodeId) {
+        const affectedShards = this.nodeToShards.get(nodeId) || new Set();
+        this.hashRing.removeNode(nodeId);
+        this.nodeToShards.delete(nodeId);
+        // Reassign affected shards
+        for (const shardId of affectedShards) {
+            const shard = this.shards.get(shardId);
+            if (shard) {
+                shard.status = 'rebalancing';
+            }
+        }
+        this.rebalanceShards();
+        this.emit('nodeRemoved', { nodeId, affectedShards: Array.from(affectedShards) });
+    }
+    /**
+     * Get shard assignment for a key
+     */
+    getShardForKey(key) {
+        const shardId = this.getShardId(key);
+        const nodes = this.hashRing.getNodes(shardId, this.replicationFactor);
+        if (nodes.length === 0)
+            return null;
+        return {
+            shardId,
+            nodeId: nodes[0],
+            replicas: nodes.slice(1)
+        };
+    }
+    /**
+     * Get nodes responsible for a shard
+     */
+    getNodesForShard(shardId) {
+        const shard = this.shards.get(shardId);
+        if (!shard)
+            return [];
+        // Return primary node and replicas
+        const nodes = this.hashRing.getNodes(shardId, this.replicationFactor);
+        return nodes;
+    }
+    /**
+     * Get total number of shards
+     */
+    getTotalShards() {
+        return this.shardCount;
+    }
+    /**
+     * Update shard assignment to a new node
+     */
+    updateShardAssignment(shardId, newNodeId) {
+        const shard = this.shards.get(shardId);
+        if (!shard) {
+            throw new Error(`Shard ${shardId} not found`);
+        }
+        // Remove from old node
+        if (shard.nodeId) {
+            const oldNodeShards = this.nodeToShards.get(shard.nodeId);
+            if (oldNodeShards) {
+                oldNodeShards.delete(shardId);
+            }
+        }
+        // Add to new node
+        shard.nodeId = newNodeId;
+        const newNodeShards = this.nodeToShards.get(newNodeId);
+        if (newNodeShards) {
+            newNodeShards.add(shardId);
+        }
+        else {
+            this.nodeToShards.set(newNodeId, new Set([shardId]));
+        }
+        this.emit('shardReassigned', { shardId, newNodeId });
+    }
+    /**
+     * Get shard ID for a key
+     */
+    getShardId(key) {
+        const hash = createHash('md5').update(key).digest();
+        const shardIndex = hash.readUInt16BE(0) % this.shardCount;
+        return `shard-${shardIndex.toString().padStart(3, '0')}`;
+    }
+    /**
+     * Rebalance shards across nodes
+     */
+    rebalanceShards() {
+        if (!this.autoRebalance)
+            return;
+        const nodes = this.hashRing.getAllNodes();
+        if (nodes.length === 0)
+            return;
+        // Clear current assignments
+        for (const nodeSet of this.nodeToShards.values()) {
+            nodeSet.clear();
+        }
+        // Reassign each shard
+        for (const [shardId, shard] of this.shards) {
+            const assignedNodes = this.hashRing.getNodes(shardId, 1);
+            if (assignedNodes.length > 0) {
+                shard.nodeId = assignedNodes[0];
+                shard.status = 'active';
+                const nodeShards = this.nodeToShards.get(assignedNodes[0]);
+                if (nodeShards) {
+                    nodeShards.add(shardId);
+                }
+            }
+            else {
+                shard.status = 'offline';
+            }
+        }
+        this.emit('rebalanced', { nodes, shardCount: this.shardCount });
+    }
+    /**
+     * Get shard assignment for all shards
+     */
+    getShardAssignments() {
+        const assignments = [];
+        for (const [shardId, shard] of this.shards) {
+            if (shard.nodeId) {
+                assignments.push({
+                    shardId,
+                    nodeId: shard.nodeId,
+                    replicas: this.hashRing.getNodes(shardId, this.replicationFactor).slice(1)
+                });
+            }
+        }
+        return assignments;
+    }
+    /**
+     * Get shard statistics
+     */
+    getShardStats() {
+        let activeShards = 0;
+        let rebalancingShards = 0;
+        let offlineShards = 0;
+        let totalItems = 0;
+        for (const shard of this.shards.values()) {
+            switch (shard.status) {
+                case 'active':
+                    activeShards++;
+                    break;
+                case 'rebalancing':
+                    rebalancingShards++;
+                    break;
+                case 'offline':
+                    offlineShards++;
+                    break;
+            }
+            totalItems += shard.itemCount;
+        }
+        return {
+            totalShards: this.shardCount,
+            activeShards,
+            rebalancingShards,
+            offlineShards,
+            averageItemsPerShard: this.shardCount > 0 ? Math.floor(totalItems / this.shardCount) : 0
+        };
+    }
+    /**
+     * Update shard metrics
+     */
+    updateShardMetrics(shardId, itemCount, sizeBytes) {
+        const shard = this.shards.get(shardId);
+        if (shard) {
+            shard.itemCount = itemCount;
+            shard.sizeBytes = sizeBytes;
+        }
+    }
+    /**
+     * Get replication nodes for a shard
+     */
+    getReplicationNodes(shardId) {
+        return this.hashRing.getNodes(shardId, this.replicationFactor);
+    }
+    /**
+     * Check if rebalancing is needed
+     */
+    needsRebalancing() {
+        const stats = this.getShardStats();
+        return stats.offlineShards > 0 || stats.rebalancingShards > 0;
+    }
+    /**
+     * Get cluster health
+     */
+    getHealth() {
+        const nodes = this.hashRing.getAllNodes();
+        const stats = this.getShardStats();
+        return {
+            healthy: stats.activeShards >= this.shardCount * 0.9, // 90% shards active
+            nodes: nodes.length,
+            shards: {
+                total: this.shardCount,
+                active: stats.activeShards,
+                inactive: stats.offlineShards + stats.rebalancingShards
+            }
+        };
+    }
+}
+/**
+ * Create a shard manager instance
+ */
+export function createShardManager(config) {
+    return new ShardManager(config);
+}
+//# sourceMappingURL=shardManager.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/shardMigration.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Shard Migration System for Brainy 3.0
+ *
+ * Handles zero-downtime migration of data between nodes
+ * Uses streaming for efficient transfer of large datasets
+ */
+import { EventEmitter } from 'events';
+import type { StorageAdapter } from '../coreTypes.js';
+import type { ShardManager } from './shardManager.js';
+import type { HTTPTransport } from './httpTransport.js';
+import type { DistributedCoordinator } from './coordinator.js';
+export interface MigrationTask {
+    id: string;
+    shardId: string;
+    sourceNode: string;
+    targetNode: string;
+    status: 'pending' | 'transferring' | 'validating' | 'switching' | 'completed' | 'failed';
+    progress: number;
+    itemsTransferred: number;
+    totalItems: number;
+    startTime: number;
+    endTime?: number;
+    error?: string;
+}
+export interface MigrationOptions {
+    batchSize?: number;
+    validateData?: boolean;
+    maxRetries?: number;
+    timeout?: number;
+}
+export declare class ShardMigrationManager extends EventEmitter {
+    private storage;
+    private shardManager;
+    private transport;
+    private coordinator;
+    private nodeId;
+    private activeMigrations;
+    private migrationQueue;
+    private maxConcurrentMigrations;
+    constructor(nodeId: string, storage: StorageAdapter, shardManager: ShardManager, transport: HTTPTransport, coordinator: DistributedCoordinator);
+    /**
+     * Initiate migration of a shard to a new node
+     */
+    migrateShard(shardId: string, targetNode: string, options?: MigrationOptions): Promise<MigrationTask>;
+    /**
+     * Process migration queue
+     */
+    private processMigrationQueue;
+    /**
+     * Execute a single migration task
+     */
+    private executeMigration;
+    /**
+     * Transfer data from source to target node
+     */
+    private transferData;
+    /**
+     * Get items from a shard
+     */
+    private getShardItems;
+    /**
+     * Get count of items in a shard
+     */
+    private getShardItemCount;
+    /**
+     * Validate transferred data
+     */
+    private validateData;
+    /**
+     * Switch shard ownership atomically
+     */
+    private switchOwnership;
+    /**
+     * Wait for consensus on migration
+     */
+    private waitForConsensus;
+    /**
+     * Cleanup local shard data after migration
+     */
+    private cleanupShardData;
+    /**
+     * Handle incoming migration batch (when we're the target)
+     */
+    receiveMigrationBatch(data: {
+        migrationId: string;
+        shardId: string;
+        items: any[];
+        offset: number;
+        total: number;
+    }): Promise<void>;
+    /**
+     * Validate received migration data
+     */
+    validateMigration(data: {
+        migrationId: string;
+        shardId: string;
+        expectedCount: number;
+    }): Promise<{
+        valid: boolean;
+        error?: string;
+    }>;
+    /**
+     * Get status of all active migrations
+     */
+    getActiveMigrations(): MigrationTask[];
+    /**
+     * Cancel a migration
+     */
+    cancelMigration(migrationId: string): Promise<void>;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/shardMigration.js

@@ -0,0 +1,289 @@
+/**
+ * Shard Migration System for Brainy 3.0
+ *
+ * Handles zero-downtime migration of data between nodes
+ * Uses streaming for efficient transfer of large datasets
+ */
+import { EventEmitter } from 'events';
+export class ShardMigrationManager extends EventEmitter {
+    constructor(nodeId, storage, shardManager, transport, coordinator) {
+        super();
+        this.activeMigrations = new Map();
+        this.migrationQueue = [];
+        this.maxConcurrentMigrations = 2;
+        this.nodeId = nodeId;
+        this.storage = storage;
+        this.shardManager = shardManager;
+        this.transport = transport;
+        this.coordinator = coordinator;
+    }
+    /**
+     * Initiate migration of a shard to a new node
+     */
+    async migrateShard(shardId, targetNode, options = {}) {
+        const task = {
+            id: `migration-${Date.now()}-${Math.random().toString(36).substring(7)}`,
+            shardId,
+            sourceNode: this.nodeId,
+            targetNode,
+            status: 'pending',
+            progress: 0,
+            itemsTransferred: 0,
+            totalItems: 0,
+            startTime: Date.now()
+        };
+        // Add to queue
+        this.migrationQueue.push(task);
+        this.processMigrationQueue();
+        return task;
+    }
+    /**
+     * Process migration queue
+     */
+    async processMigrationQueue() {
+        while (this.migrationQueue.length > 0 &&
+            this.activeMigrations.size < this.maxConcurrentMigrations) {
+            const task = this.migrationQueue.shift();
+            this.activeMigrations.set(task.id, task);
+            // Execute migration in background
+            this.executeMigration(task).catch(error => {
+                console.error(`Migration ${task.id} failed:`, error);
+                task.status = 'failed';
+                task.error = error.message;
+                this.emit('migrationFailed', task);
+            });
+        }
+    }
+    /**
+     * Execute a single migration task
+     */
+    async executeMigration(task) {
+        try {
+            this.emit('migrationStarted', task);
+            // Phase 1: Start transferring data
+            task.status = 'transferring';
+            await this.transferData(task);
+            // Phase 2: Validate transferred data
+            task.status = 'validating';
+            await this.validateData(task);
+            // Phase 3: Switch ownership atomically
+            task.status = 'switching';
+            await this.switchOwnership(task);
+            // Phase 4: Cleanup source
+            task.status = 'completed';
+            task.endTime = Date.now();
+            task.progress = 100;
+            this.activeMigrations.delete(task.id);
+            this.emit('migrationCompleted', task);
+            // Process next in queue
+            this.processMigrationQueue();
+        }
+        catch (error) {
+            task.status = 'failed';
+            task.error = error.message;
+            this.activeMigrations.delete(task.id);
+            throw error;
+        }
+    }
+    /**
+     * Transfer data from source to target node
+     */
+    async transferData(task) {
+        const batchSize = 1000;
+        let offset = 0;
+        // Get total count
+        const totalItems = await this.getShardItemCount(task.shardId);
+        task.totalItems = totalItems;
+        while (offset < totalItems) {
+            // Get batch of items
+            const items = await this.getShardItems(task.shardId, offset, batchSize);
+            if (items.length === 0)
+                break;
+            // Send batch to target node
+            await this.transport.call(task.targetNode, 'receiveMigrationBatch', {
+                migrationId: task.id,
+                shardId: task.shardId,
+                items,
+                offset,
+                total: totalItems
+            });
+            offset += items.length;
+            task.itemsTransferred = offset;
+            task.progress = Math.floor((offset / totalItems) * 80); // 80% for transfer
+            this.emit('migrationProgress', task);
+        }
+    }
+    /**
+     * Get items from a shard
+     */
+    async getShardItems(shardId, offset, limit) {
+        // Get all noun IDs for this shard
+        const nounKey = `shard:${shardId}:nouns`;
+        const verbKey = `shard:${shardId}:verbs`;
+        const items = [];
+        try {
+            // Get nouns
+            const nouns = await this.storage.getNounsByNounType('*');
+            const shardNouns = nouns.filter(n => {
+                const assignment = this.shardManager.getShardForKey(n.id);
+                return assignment?.shardId === shardId;
+            }).slice(offset, offset + limit);
+            items.push(...shardNouns.map(n => ({ type: 'noun', data: n })));
+            // Get verbs if we have room
+            if (items.length < limit) {
+                const verbs = await this.storage.getVerbsByType('*');
+                const shardVerbs = verbs.filter(v => {
+                    const assignment = this.shardManager.getShardForKey(v.id);
+                    return assignment?.shardId === shardId;
+                }).slice(0, limit - items.length);
+                items.push(...shardVerbs.map(v => ({ type: 'verb', data: v })));
+            }
+        }
+        catch (error) {
+            console.error(`Failed to get shard items for ${shardId}:`, error);
+        }
+        return items;
+    }
+    /**
+     * Get count of items in a shard
+     */
+    async getShardItemCount(shardId) {
+        // For now, estimate based on total items / shard count
+        // In production, maintain accurate per-shard counts
+        const status = await this.storage.getStorageStatus();
+        const totalShards = this.shardManager.getTotalShards();
+        return Math.ceil(status.used / totalShards);
+    }
+    /**
+     * Validate transferred data
+     */
+    async validateData(task) {
+        // Request validation from target node
+        const response = await this.transport.call(task.targetNode, 'validateMigration', {
+            migrationId: task.id,
+            shardId: task.shardId,
+            expectedCount: task.totalItems
+        });
+        if (!response.valid) {
+            throw new Error(`Validation failed: ${response.error}`);
+        }
+        task.progress = 90; // 90% after validation
+        this.emit('migrationProgress', task);
+    }
+    /**
+     * Switch shard ownership atomically
+     */
+    async switchOwnership(task) {
+        // Coordinate with all nodes to update shard assignment
+        await this.coordinator.proposeMigration({
+            shardId: task.shardId,
+            fromNode: task.sourceNode,
+            toNode: task.targetNode,
+            migrationId: task.id
+        });
+        // Wait for consensus
+        await this.waitForConsensus(task.id);
+        // Update local shard manager
+        this.shardManager.updateShardAssignment(task.shardId, task.targetNode);
+        task.progress = 95; // 95% after ownership switch
+        this.emit('migrationProgress', task);
+        // Cleanup local data
+        await this.cleanupShardData(task.shardId);
+    }
+    /**
+     * Wait for consensus on migration
+     */
+    async waitForConsensus(migrationId) {
+        const maxWait = 30000; // 30 seconds
+        const startTime = Date.now();
+        while (Date.now() - startTime < maxWait) {
+            const status = await this.coordinator.getMigrationStatus(migrationId);
+            if (status === 'committed') {
+                return;
+            }
+            else if (status === 'rejected') {
+                throw new Error('Migration rejected by cluster');
+            }
+            // Wait a bit before checking again
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+        throw new Error('Consensus timeout');
+    }
+    /**
+     * Cleanup local shard data after migration
+     */
+    async cleanupShardData(shardId) {
+        // Mark shard data for deletion
+        // Don't delete immediately in case of rollback
+        const cleanupKey = `cleanup:${shardId}:${Date.now()}`;
+        await this.storage.saveMetadata(cleanupKey, {
+            shardId,
+            scheduledFor: Date.now() + 3600000 // Delete after 1 hour
+        });
+    }
+    /**
+     * Handle incoming migration batch (when we're the target)
+     */
+    async receiveMigrationBatch(data) {
+        // Store items
+        for (const item of data.items) {
+            if (item.type === 'noun') {
+                await this.storage.saveNoun(item.data);
+            }
+            else if (item.type === 'verb') {
+                await this.storage.saveVerb(item.data);
+            }
+        }
+        // Track progress
+        const progress = {
+            migrationId: data.migrationId,
+            shardId: data.shardId,
+            received: data.offset + data.items.length,
+            total: data.total
+        };
+        await this.storage.saveMetadata(`migration:${data.migrationId}:progress`, progress);
+    }
+    /**
+     * Validate received migration data
+     */
+    async validateMigration(data) {
+        // Check if we received all expected items
+        const progressKey = `migration:${data.migrationId}:progress`;
+        const progress = await this.storage.getMetadata(progressKey);
+        if (!progress) {
+            return { valid: false, error: 'No migration progress found' };
+        }
+        if (progress.received !== data.expectedCount) {
+            return {
+                valid: false,
+                error: `Expected ${data.expectedCount} items, received ${progress.received}`
+            };
+        }
+        // Verify data integrity (could add checksums)
+        return { valid: true };
+    }
+    /**
+     * Get status of all active migrations
+     */
+    getActiveMigrations() {
+        return Array.from(this.activeMigrations.values());
+    }
+    /**
+     * Cancel a migration
+     */
+    async cancelMigration(migrationId) {
+        const task = this.activeMigrations.get(migrationId);
+        if (!task) {
+            throw new Error(`Migration ${migrationId} not found`);
+        }
+        task.status = 'failed';
+        task.error = 'Cancelled by user';
+        this.activeMigrations.delete(migrationId);
+        // Notify target node
+        await this.transport.call(task.targetNode, 'cancelMigration', {
+            migrationId
+        });
+        this.emit('migrationCancelled', task);
+    }
+}
+//# sourceMappingURL=shardMigration.js.map

.recovery-workspace/dist-backup-20250910-141917/distributed/storageDiscovery.d.ts

@@ -0,0 +1,160 @@
+/**
+ * Storage-based Discovery for Zero-Config Distributed Brainy
+ * Uses shared storage (S3/GCS/R2) as coordination point
+ * REAL PRODUCTION CODE - No mocks, no stubs!
+ */
+import { EventEmitter } from 'events';
+import { StorageAdapter } from '../coreTypes.js';
+export interface NodeInfo {
+    id: string;
+    endpoint: string;
+    hostname: string;
+    started: number;
+    lastSeen: number;
+    role: 'primary' | 'replica' | 'candidate';
+    shards: string[];
+    capacity: {
+        cpu: number;
+        memory: number;
+        storage: number;
+    };
+    stats: {
+        nouns: number;
+        verbs: number;
+        queries: number;
+        latency: number;
+    };
+}
+export interface ClusterConfig {
+    version: number;
+    created: number;
+    updated: number;
+    leader: string | null;
+    nodes: Record<string, NodeInfo>;
+    shards: {
+        count: number;
+        assignments: Record<string, string[]>;
+    };
+    settings: {
+        replicationFactor: number;
+        shardCount: number;
+        autoRebalance: boolean;
+        minNodes: number;
+        maxNodesPerShard: number;
+    };
+}
+export declare class StorageDiscovery extends EventEmitter {
+    private nodeId;
+    private storage;
+    private nodeInfo;
+    private clusterConfig;
+    private heartbeatInterval;
+    private discoveryInterval;
+    private endpoint;
+    private isRunning;
+    private readonly HEARTBEAT_INTERVAL;
+    private readonly DISCOVERY_INTERVAL;
+    private readonly NODE_TIMEOUT;
+    private readonly CLUSTER_PATH;
+    constructor(storage: StorageAdapter, nodeId?: string);
+    /**
+     * Start discovery and registration
+     */
+    start(httpPort: number): Promise<ClusterConfig>;
+    /**
+     * Stop discovery and unregister
+     */
+    stop(): Promise<void>;
+    /**
+     * Initialize a new cluster (we're the first node)
+     */
+    private initializeCluster;
+    /**
+     * Join an existing cluster
+     */
+    private joinCluster;
+    /**
+     * Leave cluster cleanly
+     */
+    private leaveCluster;
+    /**
+     * Register node in storage
+     */
+    private registerNode;
+    /**
+     * Heartbeat to keep node alive
+     */
+    private startHeartbeat;
+    /**
+     * Discover other nodes and monitor health
+     */
+    private startDiscovery;
+    /**
+     * Discover nodes from storage
+     */
+    private discoverNodes;
+    /**
+     * Load node registry from storage
+     */
+    private loadNodeRegistry;
+    /**
+     * Update node registry in storage
+     */
+    private updateNodeRegistry;
+    /**
+     * Check health of known nodes
+     */
+    private checkNodeHealth;
+    /**
+     * Load cluster configuration from storage
+     */
+    private loadClusterConfig;
+    /**
+     * Save cluster configuration to storage
+     */
+    private saveClusterConfig;
+    /**
+     * Trigger leader election (simplified - not full Raft)
+     */
+    private triggerLeaderElection;
+    /**
+     * Request shard assignment for this node
+     */
+    private requestShardAssignment;
+    /**
+     * Check if rebalancing is needed
+     */
+    private shouldRebalance;
+    /**
+     * Trigger shard rebalancing
+     */
+    private triggerRebalance;
+    /**
+     * Redistribute shards among active nodes
+     */
+    private redistributeShards;
+    /**
+     * Detect our public endpoint
+     */
+    private detectEndpoint;
+    /**
+     * Generate unique node ID
+     */
+    private generateNodeId;
+    /**
+     * Get current cluster configuration
+     */
+    getClusterConfig(): ClusterConfig | null;
+    /**
+     * Get active nodes
+     */
+    getActiveNodes(): NodeInfo[];
+    /**
+     * Get shards assigned to this node
+     */
+    getMyShards(): string[];
+    /**
+     * Update node statistics
+     */
+    updateStats(stats: Partial<NodeInfo['stats']>): void;
+}

.recovery-workspace/dist-backup-20250910-141917/distributed/storageDiscovery.js

@@ -0,0 +1,551 @@
+/**
+ * Storage-based Discovery for Zero-Config Distributed Brainy
+ * Uses shared storage (S3/GCS/R2) as coordination point
+ * REAL PRODUCTION CODE - No mocks, no stubs!
+ */
+import { EventEmitter } from 'events';
+import * as os from 'os';
+export class StorageDiscovery extends EventEmitter {
+    constructor(storage, nodeId) {
+        super();
+        this.clusterConfig = null;
+        this.heartbeatInterval = null;
+        this.discoveryInterval = null;
+        this.endpoint = '';
+        this.isRunning = false;
+        this.HEARTBEAT_INTERVAL = 5000; // 5 seconds
+        this.DISCOVERY_INTERVAL = 2000; // 2 seconds  
+        this.NODE_TIMEOUT = 30000; // 30 seconds until node considered dead
+        this.CLUSTER_PATH = '_cluster';
+        this.storage = storage;
+        this.nodeId = nodeId || this.generateNodeId();
+        // Initialize node info with REAL system data
+        this.nodeInfo = {
+            id: this.nodeId,
+            endpoint: '', // Will be set when HTTP server starts
+            hostname: os.hostname(),
+            started: Date.now(),
+            lastSeen: Date.now(),
+            role: 'candidate',
+            shards: [],
+            capacity: {
+                cpu: os.cpus().length,
+                memory: Math.floor(os.totalmem() / 1024 / 1024), // MB
+                storage: 0 // Will be updated based on actual usage
+            },
+            stats: {
+                nouns: 0,
+                verbs: 0,
+                queries: 0,
+                latency: 0
+            }
+        };
+    }
+    /**
+     * Start discovery and registration
+     */
+    async start(httpPort) {
+        if (this.isRunning)
+            return this.clusterConfig;
+        this.isRunning = true;
+        // Set our endpoint
+        this.endpoint = await this.detectEndpoint(httpPort);
+        this.nodeInfo.endpoint = this.endpoint;
+        // Try to load existing cluster config
+        this.clusterConfig = await this.loadClusterConfig();
+        if (!this.clusterConfig) {
+            // We're the first node - initialize cluster
+            await this.initializeCluster();
+        }
+        else {
+            // Join existing cluster
+            await this.joinCluster();
+        }
+        // Start heartbeat to keep our node alive
+        this.startHeartbeat();
+        // Start discovery to find other nodes
+        this.startDiscovery();
+        this.emit('started', this.nodeInfo);
+        return this.clusterConfig;
+    }
+    /**
+     * Stop discovery and unregister
+     */
+    async stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        // Stop intervals
+        if (this.heartbeatInterval) {
+            clearInterval(this.heartbeatInterval);
+            this.heartbeatInterval = null;
+        }
+        if (this.discoveryInterval) {
+            clearInterval(this.discoveryInterval);
+            this.discoveryInterval = null;
+        }
+        // Remove ourselves from cluster
+        await this.leaveCluster();
+        this.emit('stopped');
+    }
+    /**
+     * Initialize a new cluster (we're the first node)
+     */
+    async initializeCluster() {
+        console.log(`[${this.nodeId}] Initializing new cluster as first node`);
+        this.nodeInfo.role = 'primary';
+        this.clusterConfig = {
+            version: 1,
+            created: Date.now(),
+            updated: Date.now(),
+            leader: this.nodeId,
+            nodes: {
+                [this.nodeId]: this.nodeInfo
+            },
+            shards: {
+                count: 64, // Default shard count
+                assignments: {}
+            },
+            settings: {
+                replicationFactor: 3,
+                shardCount: 64,
+                autoRebalance: true,
+                minNodes: 1,
+                maxNodesPerShard: 5
+            }
+        };
+        // Assign all shards to ourselves initially
+        for (let i = 0; i < this.clusterConfig.shards.count; i++) {
+            const shardId = `shard-${i.toString().padStart(3, '0')}`;
+            this.clusterConfig.shards.assignments[shardId] = [this.nodeId];
+            this.nodeInfo.shards.push(shardId);
+        }
+        // Save cluster config
+        await this.saveClusterConfig();
+        // Register ourselves
+        await this.registerNode();
+        this.emit('clusterInitialized', this.clusterConfig);
+    }
+    /**
+     * Join an existing cluster
+     */
+    async joinCluster() {
+        console.log(`[${this.nodeId}] Joining existing cluster`);
+        if (!this.clusterConfig)
+            throw new Error('No cluster config');
+        // Add ourselves to the cluster
+        this.clusterConfig.nodes[this.nodeId] = this.nodeInfo;
+        // Determine our role based on cluster state
+        const nodeCount = Object.keys(this.clusterConfig.nodes).length;
+        if (!this.clusterConfig.leader || !this.clusterConfig.nodes[this.clusterConfig.leader]) {
+            // No leader or leader is gone - trigger election
+            await this.triggerLeaderElection();
+        }
+        else {
+            // Become replica
+            this.nodeInfo.role = 'replica';
+        }
+        // Register ourselves
+        await this.registerNode();
+        // Request shard assignment if auto-rebalance is enabled
+        if (this.clusterConfig.settings.autoRebalance) {
+            await this.requestShardAssignment();
+        }
+        this.emit('clusterJoined', this.clusterConfig);
+    }
+    /**
+     * Leave cluster cleanly
+     */
+    async leaveCluster() {
+        if (!this.clusterConfig)
+            return;
+        console.log(`[${this.nodeId}] Leaving cluster`);
+        // Remove ourselves from node registry
+        try {
+            // Mark as deleted rather than actually deleting
+            const deadNode = { ...this.nodeInfo, lastSeen: 0, status: 'inactive' };
+            await this.storage.saveMetadata(`${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`, deadNode);
+        }
+        catch (err) {
+            // Ignore errors during shutdown
+        }
+        // If we're the leader, trigger new election
+        if (this.clusterConfig.leader === this.nodeId) {
+            this.clusterConfig.leader = null;
+            await this.saveClusterConfig();
+        }
+        this.emit('clusterLeft');
+    }
+    /**
+     * Register node in storage
+     */
+    async registerNode() {
+        const path = `${this.CLUSTER_PATH}/nodes/${this.nodeId}.json`;
+        await this.storage.saveMetadata(path, this.nodeInfo);
+        // Also update registry
+        await this.updateNodeRegistry(this.nodeId);
+    }
+    /**
+     * Heartbeat to keep node alive
+     */
+    startHeartbeat() {
+        this.heartbeatInterval = setInterval(async () => {
+            try {
+                this.nodeInfo.lastSeen = Date.now();
+                await this.registerNode();
+                // Also update cluster config if we're the leader
+                if (this.clusterConfig && this.clusterConfig.leader === this.nodeId) {
+                    await this.saveClusterConfig();
+                }
+            }
+            catch (err) {
+                console.error(`[${this.nodeId}] Heartbeat failed:`, err);
+            }
+        }, this.HEARTBEAT_INTERVAL);
+    }
+    /**
+     * Discover other nodes and monitor health
+     */
+    startDiscovery() {
+        this.discoveryInterval = setInterval(async () => {
+            try {
+                await this.discoverNodes();
+                await this.checkNodeHealth();
+                // Check if we need to rebalance
+                if (this.shouldRebalance()) {
+                    await this.triggerRebalance();
+                }
+            }
+            catch (err) {
+                console.error(`[${this.nodeId}] Discovery failed:`, err);
+            }
+        }, this.DISCOVERY_INTERVAL);
+    }
+    /**
+     * Discover nodes from storage
+     */
+    async discoverNodes() {
+        try {
+            // Since we can't list arbitrary paths, we'll use a registry approach
+            // Each node registers in a central registry file
+            const registry = await this.loadNodeRegistry();
+            const now = Date.now();
+            let updated = false;
+            for (const nodeId of registry) {
+                if (nodeId === this.nodeId)
+                    continue;
+                try {
+                    const nodeInfo = await this.storage.getMetadata(`${this.CLUSTER_PATH}/nodes/${nodeId}.json`);
+                    // Check if node is alive
+                    if (now - nodeInfo.lastSeen < this.NODE_TIMEOUT) {
+                        if (!this.clusterConfig.nodes[nodeId]) {
+                            // New node discovered!
+                            console.log(`[${this.nodeId}] Discovered new node: ${nodeId}`);
+                            this.clusterConfig.nodes[nodeId] = nodeInfo;
+                            updated = true;
+                            this.emit('nodeDiscovered', nodeInfo);
+                        }
+                        else {
+                            // Update existing node info
+                            this.clusterConfig.nodes[nodeId] = nodeInfo;
+                        }
+                    }
+                }
+                catch (err) {
+                    // Node file might be corrupted or deleted
+                    console.warn(`[${this.nodeId}] Failed to read node ${nodeId}:`, err);
+                }
+            }
+            if (updated) {
+                this.clusterConfig.version++;
+                this.clusterConfig.updated = Date.now();
+            }
+        }
+        catch (err) {
+            // Storage might be unavailable
+            console.error(`[${this.nodeId}] Failed to discover nodes:`, err);
+        }
+    }
+    /**
+     * Load node registry from storage
+     */
+    async loadNodeRegistry() {
+        try {
+            const registry = await this.storage.getMetadata(`${this.CLUSTER_PATH}/registry.json`);
+            return registry?.nodes || [];
+        }
+        catch (err) {
+            return [];
+        }
+    }
+    /**
+     * Update node registry in storage
+     */
+    async updateNodeRegistry(add, remove) {
+        try {
+            let registry = await this.loadNodeRegistry();
+            if (add && !registry.includes(add)) {
+                registry.push(add);
+            }
+            if (remove) {
+                registry = registry.filter(id => id !== remove);
+            }
+            await this.storage.saveMetadata(`${this.CLUSTER_PATH}/registry.json`, {
+                nodes: registry,
+                updated: Date.now()
+            });
+        }
+        catch (err) {
+            console.error(`[${this.nodeId}] Failed to update registry:`, err);
+        }
+    }
+    /**
+     * Check health of known nodes
+     */
+    async checkNodeHealth() {
+        if (!this.clusterConfig)
+            return;
+        const now = Date.now();
+        const deadNodes = [];
+        for (const [nodeId, nodeInfo] of Object.entries(this.clusterConfig.nodes)) {
+            if (nodeId === this.nodeId)
+                continue;
+            if (now - nodeInfo.lastSeen > this.NODE_TIMEOUT) {
+                console.log(`[${this.nodeId}] Node ${nodeId} is dead (last seen ${now - nodeInfo.lastSeen}ms ago)`);
+                deadNodes.push(nodeId);
+            }
+        }
+        // Remove dead nodes
+        for (const nodeId of deadNodes) {
+            delete this.clusterConfig.nodes[nodeId];
+            this.emit('nodeLost', nodeId);
+            // If dead node was leader, trigger election
+            if (this.clusterConfig.leader === nodeId) {
+                await this.triggerLeaderElection();
+            }
+        }
+        if (deadNodes.length > 0) {
+            // Trigger rebalance to reassign shards from dead nodes
+            await this.triggerRebalance();
+        }
+    }
+    /**
+     * Load cluster configuration from storage
+     */
+    async loadClusterConfig() {
+        try {
+            const config = await this.storage.getMetadata(`${this.CLUSTER_PATH}/config.json`);
+            return config;
+        }
+        catch (err) {
+            // No cluster config exists yet
+            return null;
+        }
+    }
+    /**
+     * Save cluster configuration to storage
+     */
+    async saveClusterConfig() {
+        if (!this.clusterConfig)
+            return;
+        await this.storage.saveMetadata(`${this.CLUSTER_PATH}/config.json`, this.clusterConfig);
+    }
+    /**
+     * Trigger leader election (simplified - not full Raft)
+     */
+    async triggerLeaderElection() {
+        console.log(`[${this.nodeId}] Triggering leader election`);
+        // Simple election: node with lowest ID wins
+        // In production, use proper Raft consensus
+        const activeNodes = Object.entries(this.clusterConfig.nodes)
+            .filter(([_, info]) => Date.now() - info.lastSeen < this.NODE_TIMEOUT)
+            .sort(([a], [b]) => a.localeCompare(b));
+        if (activeNodes.length > 0) {
+            const [leaderId, leaderInfo] = activeNodes[0];
+            this.clusterConfig.leader = leaderId;
+            if (leaderId === this.nodeId) {
+                console.log(`[${this.nodeId}] Became leader`);
+                this.nodeInfo.role = 'primary';
+                this.emit('becameLeader');
+            }
+            else {
+                console.log(`[${this.nodeId}] Node ${leaderId} is the new leader`);
+                this.nodeInfo.role = 'replica';
+                this.emit('leaderElected', leaderId);
+            }
+            await this.saveClusterConfig();
+        }
+    }
+    /**
+     * Request shard assignment for this node
+     */
+    async requestShardAssignment() {
+        if (!this.clusterConfig)
+            return;
+        // Calculate how many shards each node should have
+        const nodeCount = Object.keys(this.clusterConfig.nodes).length;
+        const shardsPerNode = Math.ceil(this.clusterConfig.shards.count / nodeCount);
+        // Find shards that need assignment
+        const unassignedShards = [];
+        for (let i = 0; i < this.clusterConfig.shards.count; i++) {
+            const shardId = `shard-${i.toString().padStart(3, '0')}`;
+            if (!this.clusterConfig.shards.assignments[shardId] ||
+                this.clusterConfig.shards.assignments[shardId].length === 0) {
+                unassignedShards.push(shardId);
+            }
+        }
+        // Assign some shards to ourselves
+        const ourShare = unassignedShards.slice(0, shardsPerNode);
+        for (const shardId of ourShare) {
+            this.clusterConfig.shards.assignments[shardId] = [this.nodeId];
+            this.nodeInfo.shards.push(shardId);
+        }
+        if (ourShare.length > 0) {
+            console.log(`[${this.nodeId}] Assigned ${ourShare.length} shards`);
+            await this.saveClusterConfig();
+        }
+    }
+    /**
+     * Check if rebalancing is needed
+     */
+    shouldRebalance() {
+        if (!this.clusterConfig || !this.clusterConfig.settings.autoRebalance) {
+            return false;
+        }
+        // Check if shards are evenly distributed
+        const nodeCount = Object.keys(this.clusterConfig.nodes).length;
+        if (nodeCount <= 1)
+            return false;
+        const targetShardsPerNode = Math.ceil(this.clusterConfig.shards.count / nodeCount);
+        const variance = 2; // Allow some variance
+        for (const nodeInfo of Object.values(this.clusterConfig.nodes)) {
+            const shardCount = nodeInfo.shards.length;
+            if (Math.abs(shardCount - targetShardsPerNode) > variance) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Trigger shard rebalancing
+     */
+    async triggerRebalance() {
+        // Only leader can trigger rebalance
+        if (this.clusterConfig?.leader !== this.nodeId)
+            return;
+        console.log(`[${this.nodeId}] Triggering shard rebalance`);
+        // This will be implemented with actual data migration
+        // For now, just redistribute shard assignments
+        await this.redistributeShards();
+        this.emit('rebalanceTriggered');
+    }
+    /**
+     * Redistribute shards among active nodes
+     */
+    async redistributeShards() {
+        if (!this.clusterConfig)
+            return;
+        const activeNodes = Object.keys(this.clusterConfig.nodes)
+            .filter(id => Date.now() - this.clusterConfig.nodes[id].lastSeen < this.NODE_TIMEOUT);
+        if (activeNodes.length === 0)
+            return;
+        const shardsPerNode = Math.ceil(this.clusterConfig.shards.count / activeNodes.length);
+        const newAssignments = {};
+        // Clear current shard assignments from nodes
+        for (const nodeInfo of Object.values(this.clusterConfig.nodes)) {
+            nodeInfo.shards = [];
+        }
+        // Redistribute shards
+        let nodeIndex = 0;
+        for (let i = 0; i < this.clusterConfig.shards.count; i++) {
+            const shardId = `shard-${i.toString().padStart(3, '0')}`;
+            const primaryNode = activeNodes[nodeIndex % activeNodes.length];
+            // Assign primary
+            newAssignments[shardId] = [primaryNode];
+            this.clusterConfig.nodes[primaryNode].shards.push(shardId);
+            // Assign replicas
+            const replicas = [];
+            for (let r = 1; r < Math.min(this.clusterConfig.settings.replicationFactor, activeNodes.length); r++) {
+                const replicaNode = activeNodes[(nodeIndex + r) % activeNodes.length];
+                if (replicaNode !== primaryNode) {
+                    replicas.push(replicaNode);
+                }
+            }
+            if (replicas.length > 0) {
+                newAssignments[shardId].push(...replicas);
+            }
+            nodeIndex++;
+        }
+        this.clusterConfig.shards.assignments = newAssignments;
+        this.clusterConfig.version++;
+        this.clusterConfig.updated = Date.now();
+        await this.saveClusterConfig();
+        console.log(`[${this.nodeId}] Rebalanced ${this.clusterConfig.shards.count} shards across ${activeNodes.length} nodes`);
+    }
+    /**
+     * Detect our public endpoint
+     */
+    async detectEndpoint(port) {
+        // Try to detect public IP
+        const interfaces = os.networkInterfaces();
+        let ip = '127.0.0.1';
+        // Find first non-internal IPv4 address
+        for (const iface of Object.values(interfaces)) {
+            if (!iface)
+                continue;
+            for (const addr of iface) {
+                if (addr.family === 'IPv4' && !addr.internal) {
+                    ip = addr.address;
+                    break;
+                }
+            }
+        }
+        // In cloud environments, might need to detect public IP differently
+        if (process.env.PUBLIC_IP) {
+            ip = process.env.PUBLIC_IP;
+        }
+        else if (process.env.KUBERNETES_SERVICE_HOST) {
+            // In Kubernetes, use pod IP
+            ip = process.env.POD_IP || ip;
+        }
+        return `http://${ip}:${port}`;
+    }
+    /**
+     * Generate unique node ID
+     */
+    generateNodeId() {
+        const hostname = os.hostname();
+        const pid = process.pid;
+        const random = Math.random().toString(36).substring(2, 8);
+        return `${hostname}-${pid}-${random}`;
+    }
+    /**
+     * Get current cluster configuration
+     */
+    getClusterConfig() {
+        return this.clusterConfig;
+    }
+    /**
+     * Get active nodes
+     */
+    getActiveNodes() {
+        if (!this.clusterConfig)
+            return [];
+        const now = Date.now();
+        return Object.values(this.clusterConfig.nodes)
+            .filter(node => now - node.lastSeen < this.NODE_TIMEOUT);
+    }
+    /**
+     * Get shards assigned to this node
+     */
+    getMyShards() {
+        return this.nodeInfo.shards;
+    }
+    /**
+     * Update node statistics
+     */
+    updateStats(stats) {
+        Object.assign(this.nodeInfo.stats, stats);
+    }
+}
+//# sourceMappingURL=storageDiscovery.js.map