refactor(8.0)!: remove distributed clustering subsystem — inert/orphaned, scale is single-process + native provider
The distributed-clustering subsystem never ran in production: it was inert, orphaned dead code (faked consensus, stub replication, no live wiring, and it did not interoperate with the 8.0 Db API). Brainy 8.0 is a single-process library. Scale is single-process + the optional native provider (@soulcraft/cortex, on-disk DiskANN to 10B+ vectors) + per-tenant pools + horizontal read scaling (many reader processes, one writer). Removed: - src/distributed/ entirely (coordinator, shardManager, cacheSync, readWriteSeparation, queryPlanner, healthMonitor, configManager, hashPartitioner, shardMigration, domainDetector, storageDiscovery, http/network transports). ReaderMode/HybridMode relocated to src/storage/operationalModes.ts (slimmed to the live surface). - src/types/distributedTypes.ts; config.distributed field + JSDoc; coreTypes distributedConfig; memoryStorage distributedConfig persistence. - DistributedRole enum + src/config/distributedPresets.ts and the orphaned src/config/extensibleConfig.ts (config/augmentation registry built on removed cloud adapters + distributed presets), plus their src/index.ts re-exports. - 13 BRAINY_* cluster env vars; the storage setDistributedComponents hook; enableDistributedSearch (dead config flag); the metadata partition field; the distributed_ reserved key prefix. - Orphaned src/storage/readOnlyOptimizations.ts (zero importers). - Tests targeting the subsystem: distributed-demo, distributed-cluster helper, distributed-transactions, sharding-transactions. - Docs: EXTENDING_STORAGE.md (deleted); scrubbed distributed/cluster/Raft/ shard-manager/multi-node prose from v3-features, enterprise-for-everyone, augmentations-actual, complete-feature-list, vfs/README, vfs/ROADMAP, vfs/VFS_CORE, capacity-planning, transactions, MIGRATION-V3-TO-V4, storage-architecture; reframed scale prose to the 8.0 model. Kept: src/storage/sharding.ts (local-disk 256-bucket directory sharding via getShardIdFromUuid — used live by baseStorage, unrelated to clustering); the multi-process mode: 'reader' | 'writer' roles; semantic/HNSW clustering. RELEASES.md: added a removed-surfaces row documenting the cut and the 8.0 scale model.
This commit is contained in:
parent
f8e0079d3f
commit
00d3203d68
51 changed files with 153 additions and 9889 deletions
|
|
@ -1,300 +0,0 @@
|
|||
import { GenericContainer, StartedTestContainer, Wait } from 'testcontainers';
|
||||
import { Brainy } from '../../src/brainy';
|
||||
|
||||
export interface DistributedTestNode {
|
||||
id: string;
|
||||
brain: Brainy;
|
||||
port: number;
|
||||
}
|
||||
|
||||
export interface DistributedClusterConfig {
|
||||
nodeCount: number;
|
||||
redisHost?: string;
|
||||
redisPort?: number;
|
||||
useTestContainers?: boolean;
|
||||
}
|
||||
|
||||
export class DistributedTestCluster {
|
||||
private nodes: DistributedTestNode[] = [];
|
||||
private redisContainer?: StartedTestContainer;
|
||||
private config: DistributedClusterConfig;
|
||||
private redisEndpoint?: { host: string; port: number };
|
||||
|
||||
constructor(config: DistributedClusterConfig) {
|
||||
this.config = {
|
||||
useTestContainers: config.useTestContainers !== false,
|
||||
...config,
|
||||
nodeCount: config.nodeCount || 3
|
||||
};
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
console.log(`Starting distributed test cluster with ${this.config.nodeCount} nodes...`);
|
||||
|
||||
// Start Redis if using test containers
|
||||
if (this.config.useTestContainers && !this.config.redisHost) {
|
||||
await this.startRedis();
|
||||
} else {
|
||||
this.redisEndpoint = {
|
||||
host: this.config.redisHost || 'localhost',
|
||||
port: this.config.redisPort || 6379
|
||||
};
|
||||
}
|
||||
|
||||
// Start cluster nodes
|
||||
await this.startNodes();
|
||||
|
||||
// Wait for nodes to discover each other
|
||||
await this.waitForClusterFormation();
|
||||
|
||||
console.log(`Distributed cluster ready with ${this.nodes.length} nodes`);
|
||||
}
|
||||
|
||||
private async startRedis(): Promise<void> {
|
||||
console.log('Starting Redis test container...');
|
||||
|
||||
this.redisContainer = await new GenericContainer('redis:7-alpine')
|
||||
.withExposedPorts(6379)
|
||||
.withCommand(['redis-server', '--appendonly', 'yes'])
|
||||
.withWaitStrategy(Wait.forLogMessage('Ready to accept connections'))
|
||||
.start();
|
||||
|
||||
this.redisEndpoint = {
|
||||
host: this.redisContainer.getHost(),
|
||||
port: this.redisContainer.getMappedPort(6379)
|
||||
};
|
||||
|
||||
console.log(`Redis started on ${this.redisEndpoint.host}:${this.redisEndpoint.port}`);
|
||||
}
|
||||
|
||||
private async startNodes(): Promise<void> {
|
||||
const basePort = 8000;
|
||||
|
||||
for (let i = 0; i < this.config.nodeCount; i++) {
|
||||
const nodeId = `node-${i + 1}`;
|
||||
const port = basePort + i;
|
||||
|
||||
console.log(`Starting ${nodeId} on port ${port}...`);
|
||||
|
||||
const brain = new Brainy({ requireSubtype: false,
|
||||
storage: {
|
||||
type: 'memory'
|
||||
}
|
||||
// distributed config is not part of BrainyConfig
|
||||
// distributed: {
|
||||
// enabled: true,
|
||||
// nodeId: nodeId,
|
||||
// redis: {
|
||||
// host: this.redisEndpoint!.host,
|
||||
// port: this.redisEndpoint!.port
|
||||
// },
|
||||
// server: {
|
||||
// port: port,
|
||||
// host: '0.0.0.0'
|
||||
// },
|
||||
// discovery: {
|
||||
// interval: 1000,
|
||||
// timeout: 500
|
||||
// },
|
||||
// sharding: {
|
||||
// enabled: true,
|
||||
// replicas: 2
|
||||
// },
|
||||
// consensus: {
|
||||
// enabled: true,
|
||||
// quorum: Math.floor(this.config.nodeCount / 2) + 1
|
||||
// }
|
||||
// }
|
||||
});
|
||||
|
||||
await brain.init();
|
||||
|
||||
this.nodes.push({
|
||||
id: nodeId,
|
||||
brain: brain,
|
||||
port: port
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async waitForClusterFormation(): Promise<void> {
|
||||
console.log('Waiting for cluster formation...');
|
||||
|
||||
const maxAttempts = 30;
|
||||
const delayMs = 1000;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const allNodesDiscovered = await this.checkClusterHealth();
|
||||
|
||||
if (allNodesDiscovered) {
|
||||
console.log('All nodes have discovered each other');
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
|
||||
throw new Error('Cluster formation timeout - nodes failed to discover each other');
|
||||
}
|
||||
|
||||
private async checkClusterHealth(): Promise<boolean> {
|
||||
for (const node of this.nodes) {
|
||||
const health = await node.brain.health();
|
||||
|
||||
if (!health.distributed || !health.distributed.connectedNodes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Each node should see all other nodes
|
||||
const expectedNodeCount = this.config.nodeCount - 1;
|
||||
if (health.distributed.connectedNodes.length < expectedNodeCount) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
console.log('Stopping distributed test cluster...');
|
||||
|
||||
// Stop all nodes
|
||||
for (const node of this.nodes) {
|
||||
try {
|
||||
if (node.brain.close) {
|
||||
await node.brain.close();
|
||||
}
|
||||
console.log(`Stopped ${node.id}`);
|
||||
} catch (error) {
|
||||
console.warn(`Failed to stop ${node.id}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Stop Redis container
|
||||
if (this.redisContainer) {
|
||||
await this.redisContainer.stop();
|
||||
console.log('Redis container stopped');
|
||||
}
|
||||
|
||||
this.nodes = [];
|
||||
console.log('Distributed cluster stopped');
|
||||
}
|
||||
|
||||
getNodes(): DistributedTestNode[] {
|
||||
return this.nodes;
|
||||
}
|
||||
|
||||
getNode(index: number): DistributedTestNode {
|
||||
if (index < 0 || index >= this.nodes.length) {
|
||||
throw new Error(`Invalid node index: ${index}`);
|
||||
}
|
||||
return this.nodes[index];
|
||||
}
|
||||
|
||||
getPrimaryNode(): DistributedTestNode {
|
||||
return this.nodes[0];
|
||||
}
|
||||
|
||||
getSecondaryNodes(): DistributedTestNode[] {
|
||||
return this.nodes.slice(1);
|
||||
}
|
||||
|
||||
async executeOnAllNodes<T>(fn: (brain: Brainy) => Promise<T>): Promise<T[]> {
|
||||
return Promise.all(this.nodes.map(node => fn(node.brain)));
|
||||
}
|
||||
|
||||
async executeOnNode<T>(index: number, fn: (brain: Brainy) => Promise<T>): Promise<T> {
|
||||
const node = this.getNode(index);
|
||||
return fn(node.brain);
|
||||
}
|
||||
|
||||
async simulateNodeFailure(index: number): Promise<void> {
|
||||
const node = this.getNode(index);
|
||||
console.log(`Simulating failure of ${node.id}...`);
|
||||
|
||||
if (node.brain.close) {
|
||||
await node.brain.close();
|
||||
}
|
||||
|
||||
// Remove from active nodes
|
||||
this.nodes.splice(index, 1);
|
||||
}
|
||||
|
||||
async addNode(): Promise<DistributedTestNode> {
|
||||
const nodeId = `node-${this.nodes.length + 1}`;
|
||||
const port = 8000 + this.nodes.length;
|
||||
|
||||
console.log(`Adding new node ${nodeId} on port ${port}...`);
|
||||
|
||||
const brain = new Brainy({ requireSubtype: false,
|
||||
storage: {
|
||||
type: 'memory'
|
||||
}
|
||||
// distributed config is not part of BrainyConfig
|
||||
// distributed: {
|
||||
// enabled: true,
|
||||
// nodeId: nodeId,
|
||||
// redis: {
|
||||
// host: this.redisEndpoint!.host,
|
||||
// port: this.redisEndpoint!.port
|
||||
// },
|
||||
// server: {
|
||||
// port: port,
|
||||
// host: '0.0.0.0'
|
||||
// },
|
||||
// discovery: {
|
||||
// interval: 1000,
|
||||
// timeout: 500
|
||||
// },
|
||||
// sharding: {
|
||||
// enabled: true,
|
||||
// replicas: 2
|
||||
// }
|
||||
// }
|
||||
});
|
||||
|
||||
await brain.init();
|
||||
|
||||
const newNode = { id: nodeId, brain, port };
|
||||
this.nodes.push(newNode);
|
||||
|
||||
// Wait for new node to join cluster
|
||||
await this.waitForNodeDiscovery(nodeId);
|
||||
|
||||
return newNode;
|
||||
}
|
||||
|
||||
private async waitForNodeDiscovery(nodeId: string): Promise<void> {
|
||||
console.log(`Waiting for ${nodeId} to join cluster...`);
|
||||
|
||||
const maxAttempts = 10;
|
||||
const delayMs = 1000;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
const discovered = await this.isNodeDiscovered(nodeId);
|
||||
|
||||
if (discovered) {
|
||||
console.log(`${nodeId} has joined the cluster`);
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
|
||||
throw new Error(`Timeout waiting for ${nodeId} to join cluster`);
|
||||
}
|
||||
|
||||
private async isNodeDiscovered(nodeId: string): Promise<boolean> {
|
||||
// Check if other nodes can see the new node
|
||||
for (const node of this.nodes) {
|
||||
if (node.id === nodeId) continue;
|
||||
|
||||
const health = await node.brain.health();
|
||||
if (health.distributed?.connectedNodes?.includes(nodeId)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue