chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,357 @@
|
|||
/**
|
||||
* Shard Manager for Horizontal Scaling
|
||||
* Implements consistent hashing for data distribution across shards
|
||||
*/
|
||||
import { createHash } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
/**
|
||||
* Consistent Hash Ring for shard distribution
|
||||
*/
|
||||
class ConsistentHashRing {
|
||||
constructor(virtualNodes = 150) {
|
||||
this.ring = new Map();
|
||||
this.sortedKeys = [];
|
||||
this.virtualNodes = virtualNodes;
|
||||
}
|
||||
/**
|
||||
* Add a node to the hash ring
|
||||
*/
|
||||
addNode(nodeId) {
|
||||
for (let i = 0; i < this.virtualNodes; i++) {
|
||||
const virtualNodeId = `${nodeId}:${i}`;
|
||||
const hash = this.hash(virtualNodeId);
|
||||
this.ring.set(hash, nodeId);
|
||||
}
|
||||
this.updateSortedKeys();
|
||||
}
|
||||
/**
|
||||
* Remove a node from the hash ring
|
||||
*/
|
||||
removeNode(nodeId) {
|
||||
const keysToRemove = [];
|
||||
for (const [hash, node] of this.ring) {
|
||||
if (node === nodeId) {
|
||||
keysToRemove.push(hash);
|
||||
}
|
||||
}
|
||||
for (const key of keysToRemove) {
|
||||
this.ring.delete(key);
|
||||
}
|
||||
this.updateSortedKeys();
|
||||
}
|
||||
/**
|
||||
* Get the node responsible for a given key
|
||||
*/
|
||||
getNode(key) {
|
||||
if (this.ring.size === 0)
|
||||
return null;
|
||||
const hash = this.hash(key);
|
||||
// Find the first node with hash >= key hash
|
||||
for (const nodeHash of this.sortedKeys) {
|
||||
if (nodeHash >= hash) {
|
||||
return this.ring.get(nodeHash) || null;
|
||||
}
|
||||
}
|
||||
// Wrap around to the first node
|
||||
return this.ring.get(this.sortedKeys[0]) || null;
|
||||
}
|
||||
/**
|
||||
* Get N nodes for replication
|
||||
*/
|
||||
getNodes(key, count) {
|
||||
if (this.ring.size === 0)
|
||||
return [];
|
||||
const nodes = new Set();
|
||||
const hash = this.hash(key);
|
||||
// Start from the primary node position
|
||||
let startIdx = 0;
|
||||
for (let i = 0; i < this.sortedKeys.length; i++) {
|
||||
if (this.sortedKeys[i] >= hash) {
|
||||
startIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Collect unique nodes
|
||||
let idx = startIdx;
|
||||
while (nodes.size < count && nodes.size < this.getUniqueNodeCount()) {
|
||||
const nodeHash = this.sortedKeys[idx % this.sortedKeys.length];
|
||||
const node = this.ring.get(nodeHash);
|
||||
if (node) {
|
||||
nodes.add(node);
|
||||
}
|
||||
idx++;
|
||||
}
|
||||
return Array.from(nodes);
|
||||
}
|
||||
/**
|
||||
* Get unique node count
|
||||
*/
|
||||
getUniqueNodeCount() {
|
||||
return new Set(this.ring.values()).size;
|
||||
}
|
||||
/**
|
||||
* Update sorted keys for efficient lookup
|
||||
*/
|
||||
updateSortedKeys() {
|
||||
this.sortedKeys = Array.from(this.ring.keys()).sort((a, b) => a - b);
|
||||
}
|
||||
/**
|
||||
* Hash function for consistent hashing
|
||||
*/
|
||||
hash(key) {
|
||||
const hash = createHash('md5').update(key).digest();
|
||||
return hash.readUInt32BE(0);
|
||||
}
|
||||
/**
|
||||
* Get all nodes in the ring
|
||||
*/
|
||||
getAllNodes() {
|
||||
return Array.from(new Set(this.ring.values()));
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Shard Manager for distributing data across multiple nodes
|
||||
*/
|
||||
export class ShardManager extends EventEmitter {
|
||||
constructor(config = {}) {
|
||||
super();
|
||||
this.shards = new Map();
|
||||
this.nodeToShards = new Map();
|
||||
this.shardCount = config.shardCount || 64;
|
||||
this.replicationFactor = config.replicationFactor || 3;
|
||||
this.autoRebalance = config.autoRebalance ?? true;
|
||||
this.hashRing = new ConsistentHashRing(config.virtualNodes || 150);
|
||||
// Initialize shards
|
||||
this.initializeShards();
|
||||
}
|
||||
/**
|
||||
* Initialize shard configuration
|
||||
*/
|
||||
initializeShards() {
|
||||
for (let i = 0; i < this.shardCount; i++) {
|
||||
const shardId = `shard-${i.toString().padStart(3, '0')}`;
|
||||
this.shards.set(shardId, {
|
||||
id: shardId,
|
||||
nodeId: '',
|
||||
virtualNodes: [],
|
||||
itemCount: 0,
|
||||
sizeBytes: 0,
|
||||
status: 'offline'
|
||||
});
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Add a node to the cluster
|
||||
*/
|
||||
addNode(nodeId) {
|
||||
this.hashRing.addNode(nodeId);
|
||||
this.nodeToShards.set(nodeId, new Set());
|
||||
// Assign shards to the new node
|
||||
this.rebalanceShards();
|
||||
this.emit('nodeAdded', { nodeId });
|
||||
}
|
||||
/**
|
||||
* Remove a node from the cluster
|
||||
*/
|
||||
removeNode(nodeId) {
|
||||
const affectedShards = this.nodeToShards.get(nodeId) || new Set();
|
||||
this.hashRing.removeNode(nodeId);
|
||||
this.nodeToShards.delete(nodeId);
|
||||
// Reassign affected shards
|
||||
for (const shardId of affectedShards) {
|
||||
const shard = this.shards.get(shardId);
|
||||
if (shard) {
|
||||
shard.status = 'rebalancing';
|
||||
}
|
||||
}
|
||||
this.rebalanceShards();
|
||||
this.emit('nodeRemoved', { nodeId, affectedShards: Array.from(affectedShards) });
|
||||
}
|
||||
/**
|
||||
* Get shard assignment for a key
|
||||
*/
|
||||
getShardForKey(key) {
|
||||
const shardId = this.getShardId(key);
|
||||
const nodes = this.hashRing.getNodes(shardId, this.replicationFactor);
|
||||
if (nodes.length === 0)
|
||||
return null;
|
||||
return {
|
||||
shardId,
|
||||
nodeId: nodes[0],
|
||||
replicas: nodes.slice(1)
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get nodes responsible for a shard
|
||||
*/
|
||||
getNodesForShard(shardId) {
|
||||
const shard = this.shards.get(shardId);
|
||||
if (!shard)
|
||||
return [];
|
||||
// Return primary node and replicas
|
||||
const nodes = this.hashRing.getNodes(shardId, this.replicationFactor);
|
||||
return nodes;
|
||||
}
|
||||
/**
|
||||
* Get total number of shards
|
||||
*/
|
||||
getTotalShards() {
|
||||
return this.shardCount;
|
||||
}
|
||||
/**
|
||||
* Update shard assignment to a new node
|
||||
*/
|
||||
updateShardAssignment(shardId, newNodeId) {
|
||||
const shard = this.shards.get(shardId);
|
||||
if (!shard) {
|
||||
throw new Error(`Shard ${shardId} not found`);
|
||||
}
|
||||
// Remove from old node
|
||||
if (shard.nodeId) {
|
||||
const oldNodeShards = this.nodeToShards.get(shard.nodeId);
|
||||
if (oldNodeShards) {
|
||||
oldNodeShards.delete(shardId);
|
||||
}
|
||||
}
|
||||
// Add to new node
|
||||
shard.nodeId = newNodeId;
|
||||
const newNodeShards = this.nodeToShards.get(newNodeId);
|
||||
if (newNodeShards) {
|
||||
newNodeShards.add(shardId);
|
||||
}
|
||||
else {
|
||||
this.nodeToShards.set(newNodeId, new Set([shardId]));
|
||||
}
|
||||
this.emit('shardReassigned', { shardId, newNodeId });
|
||||
}
|
||||
/**
|
||||
* Get shard ID for a key
|
||||
*/
|
||||
getShardId(key) {
|
||||
const hash = createHash('md5').update(key).digest();
|
||||
const shardIndex = hash.readUInt16BE(0) % this.shardCount;
|
||||
return `shard-${shardIndex.toString().padStart(3, '0')}`;
|
||||
}
|
||||
/**
|
||||
* Rebalance shards across nodes
|
||||
*/
|
||||
rebalanceShards() {
|
||||
if (!this.autoRebalance)
|
||||
return;
|
||||
const nodes = this.hashRing.getAllNodes();
|
||||
if (nodes.length === 0)
|
||||
return;
|
||||
// Clear current assignments
|
||||
for (const nodeSet of this.nodeToShards.values()) {
|
||||
nodeSet.clear();
|
||||
}
|
||||
// Reassign each shard
|
||||
for (const [shardId, shard] of this.shards) {
|
||||
const assignedNodes = this.hashRing.getNodes(shardId, 1);
|
||||
if (assignedNodes.length > 0) {
|
||||
shard.nodeId = assignedNodes[0];
|
||||
shard.status = 'active';
|
||||
const nodeShards = this.nodeToShards.get(assignedNodes[0]);
|
||||
if (nodeShards) {
|
||||
nodeShards.add(shardId);
|
||||
}
|
||||
}
|
||||
else {
|
||||
shard.status = 'offline';
|
||||
}
|
||||
}
|
||||
this.emit('rebalanced', { nodes, shardCount: this.shardCount });
|
||||
}
|
||||
/**
|
||||
* Get shard assignment for all shards
|
||||
*/
|
||||
getShardAssignments() {
|
||||
const assignments = [];
|
||||
for (const [shardId, shard] of this.shards) {
|
||||
if (shard.nodeId) {
|
||||
assignments.push({
|
||||
shardId,
|
||||
nodeId: shard.nodeId,
|
||||
replicas: this.hashRing.getNodes(shardId, this.replicationFactor).slice(1)
|
||||
});
|
||||
}
|
||||
}
|
||||
return assignments;
|
||||
}
|
||||
/**
|
||||
* Get shard statistics
|
||||
*/
|
||||
getShardStats() {
|
||||
let activeShards = 0;
|
||||
let rebalancingShards = 0;
|
||||
let offlineShards = 0;
|
||||
let totalItems = 0;
|
||||
for (const shard of this.shards.values()) {
|
||||
switch (shard.status) {
|
||||
case 'active':
|
||||
activeShards++;
|
||||
break;
|
||||
case 'rebalancing':
|
||||
rebalancingShards++;
|
||||
break;
|
||||
case 'offline':
|
||||
offlineShards++;
|
||||
break;
|
||||
}
|
||||
totalItems += shard.itemCount;
|
||||
}
|
||||
return {
|
||||
totalShards: this.shardCount,
|
||||
activeShards,
|
||||
rebalancingShards,
|
||||
offlineShards,
|
||||
averageItemsPerShard: this.shardCount > 0 ? Math.floor(totalItems / this.shardCount) : 0
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Update shard metrics
|
||||
*/
|
||||
updateShardMetrics(shardId, itemCount, sizeBytes) {
|
||||
const shard = this.shards.get(shardId);
|
||||
if (shard) {
|
||||
shard.itemCount = itemCount;
|
||||
shard.sizeBytes = sizeBytes;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get replication nodes for a shard
|
||||
*/
|
||||
getReplicationNodes(shardId) {
|
||||
return this.hashRing.getNodes(shardId, this.replicationFactor);
|
||||
}
|
||||
/**
|
||||
* Check if rebalancing is needed
|
||||
*/
|
||||
needsRebalancing() {
|
||||
const stats = this.getShardStats();
|
||||
return stats.offlineShards > 0 || stats.rebalancingShards > 0;
|
||||
}
|
||||
/**
|
||||
* Get cluster health
|
||||
*/
|
||||
getHealth() {
|
||||
const nodes = this.hashRing.getAllNodes();
|
||||
const stats = this.getShardStats();
|
||||
return {
|
||||
healthy: stats.activeShards >= this.shardCount * 0.9, // 90% shards active
|
||||
nodes: nodes.length,
|
||||
shards: {
|
||||
total: this.shardCount,
|
||||
active: stats.activeShards,
|
||||
inactive: stats.offlineShards + stats.rebalancingShards
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Create a shard manager instance
|
||||
*/
|
||||
export function createShardManager(config) {
|
||||
return new ShardManager(config);
|
||||
}
|
||||
//# sourceMappingURL=shardManager.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue